toolroll 0.9.47 → 0.9.48
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/dist/browser/workspace.js +16 -16
- package/dist/cli.d.ts +1 -1
- package/dist/cli.js +4 -0
- package/dist/contracts/cli.d.ts +87 -0
- package/dist/contracts/cli.js +2 -0
- package/dist/contracts/health.d.ts +84 -0
- package/dist/contracts/health.js +11 -0
- package/dist/desktop-update.d.ts +1 -1
- package/dist/flow-live.d.ts +5 -13
- package/dist/flow-live.js +11 -110
- package/dist/health-cli.d.ts +3 -0
- package/dist/health-cli.js +79 -0
- package/dist/live-bus.d.ts +89 -0
- package/dist/live-bus.js +321 -0
- package/dist/metrics.d.ts +3 -0
- package/dist/metrics.js +53 -1
- package/dist/operate.js +37 -24
- package/dist/read-executor-worker.d.ts +1 -0
- package/dist/read-executor-worker.js +26 -0
- package/dist/read-executor.d.ts +49 -0
- package/dist/read-executor.js +119 -0
- package/dist/request-budget.d.ts +1 -0
- package/dist/request-budget.js +6 -0
- package/dist/serve.d.ts +2 -0
- package/dist/serve.js +28 -4
- package/dist/server-telemetry.d.ts +86 -0
- package/dist/server-telemetry.js +263 -0
- package/dist/sqlite-telemetry.d.ts +9 -0
- package/dist/sqlite-telemetry.js +98 -0
- package/dist/store.d.ts +6 -0
- package/dist/store.js +25 -3
- package/dist/surface.js +2 -1
- package/dist/task-live.d.ts +5 -14
- package/dist/task-live.js +13 -110
- package/dist/work-index.js +12 -1
- package/dist/workspace-revision.d.ts +2 -0
- package/dist/workspace-revision.js +3 -1
- package/package.json +1 -1
package/dist/live-bus.js
ADDED
|
@@ -0,0 +1,321 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Live views push on write. Nothing here polls once a second.
|
|
3
|
+
*
|
|
4
|
+
* - The bus. The database's own revision triggers (workspace-revision.ts)
|
|
5
|
+
* move one counter on every meaningful write. After each in-process
|
|
6
|
+
* transact() COMMIT, and whenever another process (the worker, the CLI)
|
|
7
|
+
* writes the database's write-ahead log, the revision is read once; when
|
|
8
|
+
* it moved, every subscriber hears `{ revision }`. A rolled-back or quiet
|
|
9
|
+
* write moves nothing and says nothing. A page left open with nothing
|
|
10
|
+
* being written costs no database work at all.
|
|
11
|
+
* - The rooms. Each open task or flow is a room keyed by what it shows.
|
|
12
|
+
* When the bus speaks, a room rechecks every viewer's access (a signed-out
|
|
13
|
+
* or narrowed account is dropped), takes its fingerprint once, and only if
|
|
14
|
+
* that moved tells its pages `change` with the fingerprint and revision —
|
|
15
|
+
* an invalidation, never content. The page reads itself again under its
|
|
16
|
+
* own checks, so redaction stays the page's.
|
|
17
|
+
* - The safety net. Every room also checks itself about every 30 s, so a
|
|
18
|
+
* missed signal heals, and sends a keep-alive between.
|
|
19
|
+
* - Back-pressure. Each page's stream is bounded: a page that falls behind
|
|
20
|
+
* gets one `reload` and ends so its EventSource can reconnect.
|
|
21
|
+
*
|
|
22
|
+
* Transport seam: a future WebSocket (live cursors) subscribes to the same
|
|
23
|
+
* LiveBus beside these SSE rooms; nothing in the Store or the triggers changes.
|
|
24
|
+
*/
|
|
25
|
+
import { watch } from "node:fs";
|
|
26
|
+
import { WORKSPACE_REVISION_KEY } from "./workspace-revision.js";
|
|
27
|
+
export function createLiveBus() {
|
|
28
|
+
const listeners = new Set();
|
|
29
|
+
let last = null;
|
|
30
|
+
return {
|
|
31
|
+
subscribe(listener) { listeners.add(listener); return () => { listeners.delete(listener); }; },
|
|
32
|
+
publish(change) {
|
|
33
|
+
last = change.revision;
|
|
34
|
+
for (const listener of [...listeners]) {
|
|
35
|
+
try {
|
|
36
|
+
listener(change);
|
|
37
|
+
}
|
|
38
|
+
catch { /* one room never stops another */ }
|
|
39
|
+
}
|
|
40
|
+
},
|
|
41
|
+
revision: () => last,
|
|
42
|
+
listeners: () => listeners.size,
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
const SIGNAL = "toolroll_live_written";
|
|
46
|
+
/**
|
|
47
|
+
* Publish the workspace revision when it moves: after this process commits,
|
|
48
|
+
* and when the database's write-ahead log changes under another process.
|
|
49
|
+
* Signals in one turn of the event loop coalesce into one read.
|
|
50
|
+
*/
|
|
51
|
+
export function followWorkspace(store, current, bus, options = {}) {
|
|
52
|
+
let last = null;
|
|
53
|
+
try {
|
|
54
|
+
last = current();
|
|
55
|
+
}
|
|
56
|
+
catch { /* the first signal reads it */ }
|
|
57
|
+
let scheduled = false, closed = false;
|
|
58
|
+
let timer = null;
|
|
59
|
+
const check = () => {
|
|
60
|
+
scheduled = false;
|
|
61
|
+
timer = null;
|
|
62
|
+
if (closed)
|
|
63
|
+
return;
|
|
64
|
+
// Never read inside an open transaction: what it wrote isn't committed yet.
|
|
65
|
+
if (store.handle.isTransaction === true) {
|
|
66
|
+
schedule(10);
|
|
67
|
+
return;
|
|
68
|
+
}
|
|
69
|
+
let now;
|
|
70
|
+
try {
|
|
71
|
+
now = current();
|
|
72
|
+
}
|
|
73
|
+
catch {
|
|
74
|
+
schedule(250);
|
|
75
|
+
return; /* busy: read again shortly */
|
|
76
|
+
}
|
|
77
|
+
if (now === last)
|
|
78
|
+
return;
|
|
79
|
+
last = now;
|
|
80
|
+
bus.publish({ revision: now });
|
|
81
|
+
};
|
|
82
|
+
const schedule = (delay = 0) => {
|
|
83
|
+
// With no page open, nobody needs the read.
|
|
84
|
+
if (scheduled || closed || bus.listeners() === 0)
|
|
85
|
+
return;
|
|
86
|
+
scheduled = true;
|
|
87
|
+
timer = setTimeout(check, delay);
|
|
88
|
+
timer.unref?.();
|
|
89
|
+
};
|
|
90
|
+
const unhook = store.onCommit(() => schedule());
|
|
91
|
+
// A write this process makes outside transact() (an autocommit statement)
|
|
92
|
+
// moves the revision too: a connection-local TEMP trigger on the revision
|
|
93
|
+
// row calls back here. Other processes never see it. The read waits for
|
|
94
|
+
// the statement's own commit, and a rollback leaves the revision unmoved.
|
|
95
|
+
let signalled = false;
|
|
96
|
+
const db = store.handle;
|
|
97
|
+
if (typeof db.function === "function") {
|
|
98
|
+
try {
|
|
99
|
+
db.function(SIGNAL, { deterministic: false }, () => { if (!closed)
|
|
100
|
+
schedule(); return null; });
|
|
101
|
+
db.exec(`CREATE TEMP TRIGGER IF NOT EXISTS ${SIGNAL}_update AFTER UPDATE ON main.service_cursor WHEN NEW.key = '${WORKSPACE_REVISION_KEY}' BEGIN SELECT ${SIGNAL}(); END;
|
|
102
|
+
CREATE TEMP TRIGGER IF NOT EXISTS ${SIGNAL}_insert AFTER INSERT ON main.service_cursor WHEN NEW.key = '${WORKSPACE_REVISION_KEY}' BEGIN SELECT ${SIGNAL}(); END;`);
|
|
103
|
+
signalled = true;
|
|
104
|
+
}
|
|
105
|
+
catch { /* commits and the safety net still reach the rooms */ }
|
|
106
|
+
}
|
|
107
|
+
const file = options.file ?? null;
|
|
108
|
+
let watcher = null;
|
|
109
|
+
if (file !== null) {
|
|
110
|
+
const start = options.watchFile ?? ((path, listener) => { try {
|
|
111
|
+
return watch(path, { persistent: false }, listener);
|
|
112
|
+
}
|
|
113
|
+
catch {
|
|
114
|
+
return null;
|
|
115
|
+
} });
|
|
116
|
+
// The worker and CLI write from their own processes; their commits land in the WAL.
|
|
117
|
+
watcher = start(`${file}-wal`, () => schedule());
|
|
118
|
+
watcher?.on?.("error", () => { watcher?.close(); watcher = null; });
|
|
119
|
+
}
|
|
120
|
+
return {
|
|
121
|
+
check,
|
|
122
|
+
close() {
|
|
123
|
+
closed = true;
|
|
124
|
+
if (timer !== null)
|
|
125
|
+
clearTimeout(timer);
|
|
126
|
+
unhook();
|
|
127
|
+
watcher?.close();
|
|
128
|
+
watcher = null;
|
|
129
|
+
if (signalled) {
|
|
130
|
+
try {
|
|
131
|
+
db.exec(`DROP TRIGGER IF EXISTS temp.${SIGNAL}_update; DROP TRIGGER IF EXISTS temp.${SIGNAL}_insert;`);
|
|
132
|
+
}
|
|
133
|
+
catch { /* the database is closing */ }
|
|
134
|
+
}
|
|
135
|
+
},
|
|
136
|
+
};
|
|
137
|
+
}
|
|
138
|
+
/** About every 30 s a room checks itself; a keep-alive goes between. */
|
|
139
|
+
export const SAFETY_NET_MS = 30_000;
|
|
140
|
+
export const KEEP_ALIVE_MS = 15_000;
|
|
141
|
+
/** What a page's stream may hold unsent before it is "behind". */
|
|
142
|
+
export const STREAM_LIMIT_BYTES = 32 * 1024;
|
|
143
|
+
/** One page's stream, bounded. Overflow is terminal, even below Node's drain threshold. */
|
|
144
|
+
export class LiveStream {
|
|
145
|
+
response;
|
|
146
|
+
limit;
|
|
147
|
+
onOverflow;
|
|
148
|
+
ended = false;
|
|
149
|
+
constructor(response, limit = STREAM_LIMIT_BYTES, onOverflow = () => { }) {
|
|
150
|
+
this.response = response;
|
|
151
|
+
this.limit = limit;
|
|
152
|
+
this.onOverflow = onOverflow;
|
|
153
|
+
}
|
|
154
|
+
get open() { return !this.ended && !this.response.writableEnded && !this.response.destroyed; }
|
|
155
|
+
overflow() {
|
|
156
|
+
if (!this.open)
|
|
157
|
+
return;
|
|
158
|
+
this.ended = true;
|
|
159
|
+
// end(data) queues at most this final frame; it never waits for a drain
|
|
160
|
+
// that Node may not emit when our limit is below its high-water mark.
|
|
161
|
+
try {
|
|
162
|
+
this.response.end("event: reload\ndata: {}\n\n");
|
|
163
|
+
}
|
|
164
|
+
finally {
|
|
165
|
+
this.onOverflow();
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
write(frame) {
|
|
169
|
+
if (!this.open)
|
|
170
|
+
return;
|
|
171
|
+
if (this.response.writableLength > this.limit) {
|
|
172
|
+
this.overflow();
|
|
173
|
+
return;
|
|
174
|
+
}
|
|
175
|
+
this.response.write(frame);
|
|
176
|
+
if (this.response.writableLength > this.limit)
|
|
177
|
+
this.overflow();
|
|
178
|
+
}
|
|
179
|
+
send(event, data) {
|
|
180
|
+
this.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
|
|
181
|
+
}
|
|
182
|
+
keepAlive() {
|
|
183
|
+
this.write(": keep-alive\n\n");
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
const PRESENCE_SETTLE_MS = 150;
|
|
187
|
+
/** Rooms of open pages that hear the bus. `presence` says who else is here, as each page should see it. */
|
|
188
|
+
export function createLiveRooms(fingerprint, presence, options = {}) {
|
|
189
|
+
const bus = options.bus ?? null;
|
|
190
|
+
const keepAliveMs = options.keepAliveMs ?? KEEP_ALIVE_MS;
|
|
191
|
+
const checkEvery = Math.max(1, Math.round((options.safetyNetMs ?? SAFETY_NET_MS) / keepAliveMs));
|
|
192
|
+
const rooms = new Map();
|
|
193
|
+
let unsubscribe = null;
|
|
194
|
+
const here = (room, viewer) => presence([...room.viewers.keys()], viewer.name);
|
|
195
|
+
const announce = (room) => {
|
|
196
|
+
// A page that moves from one card to the next leaves and joins at once: say it once, after it settles.
|
|
197
|
+
if (room.settling !== null)
|
|
198
|
+
clearTimeout(room.settling);
|
|
199
|
+
room.settling = setTimeout(() => {
|
|
200
|
+
room.settling = null;
|
|
201
|
+
for (const [viewer, stream] of room.viewers)
|
|
202
|
+
stream.send("here", { people: here(room, viewer) });
|
|
203
|
+
}, PRESENCE_SETTLE_MS);
|
|
204
|
+
room.settling.unref();
|
|
205
|
+
};
|
|
206
|
+
const leave = (key, viewer) => {
|
|
207
|
+
const room = rooms.get(key);
|
|
208
|
+
if (room === undefined || !room.viewers.delete(viewer))
|
|
209
|
+
return;
|
|
210
|
+
if (!viewer.response.writableEnded)
|
|
211
|
+
viewer.response.end();
|
|
212
|
+
if (room.viewers.size > 0) {
|
|
213
|
+
announce(room);
|
|
214
|
+
return;
|
|
215
|
+
}
|
|
216
|
+
clearInterval(room.timer);
|
|
217
|
+
if (room.settling !== null)
|
|
218
|
+
clearTimeout(room.settling);
|
|
219
|
+
if (room.retry !== null)
|
|
220
|
+
clearTimeout(room.retry);
|
|
221
|
+
rooms.delete(key);
|
|
222
|
+
if (rooms.size === 0 && unsubscribe !== null) {
|
|
223
|
+
unsubscribe();
|
|
224
|
+
unsubscribe = null;
|
|
225
|
+
}
|
|
226
|
+
};
|
|
227
|
+
const drop = (key, viewer) => {
|
|
228
|
+
rooms.get(key)?.viewers.get(viewer)?.send("gone", {});
|
|
229
|
+
leave(key, viewer);
|
|
230
|
+
};
|
|
231
|
+
/** Recheck who may still see the room, then whether what it shows moved. */
|
|
232
|
+
const refresh = (key, room, revision) => {
|
|
233
|
+
// A signed-out or narrowed account stops hearing about it, at the first write after.
|
|
234
|
+
for (const viewer of [...room.viewers.keys()]) {
|
|
235
|
+
let ok = false;
|
|
236
|
+
try {
|
|
237
|
+
ok = viewer.valid();
|
|
238
|
+
}
|
|
239
|
+
catch {
|
|
240
|
+
ok = false;
|
|
241
|
+
}
|
|
242
|
+
if (!ok)
|
|
243
|
+
drop(key, viewer);
|
|
244
|
+
}
|
|
245
|
+
if (rooms.get(key) !== room)
|
|
246
|
+
return;
|
|
247
|
+
let now;
|
|
248
|
+
try {
|
|
249
|
+
now = fingerprint(key);
|
|
250
|
+
}
|
|
251
|
+
catch {
|
|
252
|
+
// A busy database: read again shortly rather than wait for the safety net.
|
|
253
|
+
if (room.retry === null) {
|
|
254
|
+
room.retry = setTimeout(() => { room.retry = null; if (rooms.get(key) === room)
|
|
255
|
+
refresh(key, room, revision); }, 1_000);
|
|
256
|
+
room.retry.unref();
|
|
257
|
+
}
|
|
258
|
+
return;
|
|
259
|
+
}
|
|
260
|
+
if (now === null) {
|
|
261
|
+
for (const viewer of [...room.viewers.keys()])
|
|
262
|
+
drop(key, viewer);
|
|
263
|
+
return;
|
|
264
|
+
}
|
|
265
|
+
if (now === room.last)
|
|
266
|
+
return;
|
|
267
|
+
room.last = now;
|
|
268
|
+
for (const stream of room.viewers.values())
|
|
269
|
+
stream.send("change", { at: now, revision: revision ?? bus?.revision() ?? null });
|
|
270
|
+
};
|
|
271
|
+
const tick = (key, room) => {
|
|
272
|
+
room.ticks += 1;
|
|
273
|
+
if (room.ticks % checkEvery === 0) {
|
|
274
|
+
const before = room.last;
|
|
275
|
+
refresh(key, room, null);
|
|
276
|
+
if (rooms.get(key) === room && room.last !== before)
|
|
277
|
+
return;
|
|
278
|
+
}
|
|
279
|
+
for (const stream of room.viewers.values())
|
|
280
|
+
stream.keepAlive();
|
|
281
|
+
};
|
|
282
|
+
const heard = (change) => {
|
|
283
|
+
for (const [key, room] of [...rooms])
|
|
284
|
+
refresh(key, room, change.revision);
|
|
285
|
+
};
|
|
286
|
+
return {
|
|
287
|
+
join(key, viewer) {
|
|
288
|
+
let room = rooms.get(key);
|
|
289
|
+
if (room === undefined) {
|
|
290
|
+
let last = null;
|
|
291
|
+
try {
|
|
292
|
+
last = fingerprint(key);
|
|
293
|
+
}
|
|
294
|
+
catch { /* the next signal or the safety net reads it */ }
|
|
295
|
+
const created = { viewers: new Map(), last, timer: setInterval(() => tick(key, created), keepAliveMs), ticks: 0, settling: null, retry: null };
|
|
296
|
+
created.timer.unref();
|
|
297
|
+
room = created;
|
|
298
|
+
rooms.set(key, room);
|
|
299
|
+
if (bus !== null && unsubscribe === null)
|
|
300
|
+
unsubscribe = bus.subscribe(heard);
|
|
301
|
+
}
|
|
302
|
+
// Remove an overflowed page immediately, even while its final bytes
|
|
303
|
+
// wait for the client. Peers must not keep seeing stale presence.
|
|
304
|
+
const stream = new LiveStream(viewer.response, options.limitBytes, () => leave(key, viewer));
|
|
305
|
+
room.viewers.set(viewer, stream);
|
|
306
|
+
viewer.response.once("close", () => leave(key, viewer));
|
|
307
|
+
stream.send("change", { at: room.last, revision: bus?.revision() ?? null });
|
|
308
|
+
if (!stream.open)
|
|
309
|
+
return;
|
|
310
|
+
stream.send("here", { people: here(room, viewer) });
|
|
311
|
+
if (stream.open)
|
|
312
|
+
announce(room);
|
|
313
|
+
},
|
|
314
|
+
close() {
|
|
315
|
+
for (const [key, room] of [...rooms])
|
|
316
|
+
for (const viewer of [...room.viewers.keys()])
|
|
317
|
+
leave(key, viewer);
|
|
318
|
+
},
|
|
319
|
+
size: () => [...rooms.values()].reduce((sum, room) => sum + room.viewers.size, 0),
|
|
320
|
+
};
|
|
321
|
+
}
|
package/dist/metrics.d.ts
CHANGED
|
@@ -1,3 +1,6 @@
|
|
|
1
1
|
import type { Store } from "./store.js";
|
|
2
|
+
import { type ServerTelemetry } from "./server-telemetry.js";
|
|
2
3
|
/** `repos`: the projects this console serves (null: all); task, run and worker figures count only theirs. */
|
|
3
4
|
export declare function prometheusMetrics(store: Store, now: Date, repos?: readonly string[] | null, destinations?: ReadonlyMap<string, string>): string;
|
|
5
|
+
/** Performance labels are fixed vocabularies only, independent of the store's project labels. */
|
|
6
|
+
export declare function performanceMetrics(telemetry: ServerTelemetry): string;
|
package/dist/metrics.js
CHANGED
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
*/
|
|
7
7
|
import { createHash } from "node:crypto";
|
|
8
8
|
import { basename } from "node:path";
|
|
9
|
+
import { LATENCY_BUCKETS, QUEUE_BUCKETS, STREAM_FAMILIES } from "./server-telemetry.js";
|
|
9
10
|
const label = (value) => value.replace(/\\/g, "\\\\").replace(/"/g, '\\"').replace(/\n/g, "\\n");
|
|
10
11
|
const line = (name, sample) => {
|
|
11
12
|
const labels = Object.entries(sample.labels ?? {});
|
|
@@ -52,5 +53,56 @@ export function prometheusMetrics(store, now, repos = null, destinations) {
|
|
|
52
53
|
add("toolroll_monitoring_last_ok_timestamp_seconds", "When a destination last took a delivery.", "gauge", status.filter(one => one.lastOkAt !== null).map(one => ({ labels: { destination: one.sink }, value: Math.floor(Date.parse(one.lastOkAt) / 1000) })));
|
|
53
54
|
add("toolroll_worker_heartbeat_age_seconds", "Seconds since each project's worker last checked in.", "gauge", rows(`SELECT repo, MAX(heartbeat_at) AS at FROM watch_lease WHERE ${inProjects("repo")} GROUP BY repo`, ...scoped).map(row => ({ labels: { project: project(String(row["repo"])) }, value: Math.max(0, Math.round((now.getTime() - Date.parse(String(row["at"]))) / 1000)) })));
|
|
54
55
|
add("toolroll_checkouts", "Build checkouts on disk, by state.", "gauge", rows(`SELECT CASE WHEN runner IS NOT NULL AND released_at IS NULL THEN 'leased' ELSE 'released' END AS state, COUNT(*) AS n FROM worktree WHERE ${inProjects("repo")} GROUP BY 1`, ...scoped).map(row => ({ labels: { state: String(row["state"]) }, value: Number(row["n"]) })));
|
|
55
|
-
return `${metrics.map(metric => [`# HELP ${metric.name} ${metric.help}`, `# TYPE ${metric.name} ${metric.type}`, ...metric.samples.map(sample => line(metric.name, sample))].join("\n")).join("\n")}\n`;
|
|
56
|
+
return `${metrics.map(metric => [`# HELP ${metric.name} ${metric.help}`, `# TYPE ${metric.name} ${metric.type}`, ...metric.samples.map(sample => line(metric.name, sample))].join("\n")).join("\n")}\n${performanceMetrics(store.telemetry)}`;
|
|
57
|
+
}
|
|
58
|
+
/** Performance labels are fixed vocabularies only, independent of the store's project labels. */
|
|
59
|
+
export function performanceMetrics(telemetry) {
|
|
60
|
+
const out = [];
|
|
61
|
+
const header = (name, help, type) => out.push(`# HELP ${name} ${help}`, `# TYPE ${name} ${type}`);
|
|
62
|
+
const histogram = (name, value, labels = {}) => {
|
|
63
|
+
let cumulative = 0;
|
|
64
|
+
for (const [i, n] of value.total.bins.entries()) {
|
|
65
|
+
cumulative += n;
|
|
66
|
+
out.push(line(`${name}_bucket`, { labels: { ...labels, le: String(LATENCY_BUCKETS[i] ?? '+Inf') }, value: cumulative }));
|
|
67
|
+
}
|
|
68
|
+
out.push(line(`${name}_sum`, { labels, value: value.total.sum }), line(`${name}_count`, { labels, value: value.total.count }));
|
|
69
|
+
};
|
|
70
|
+
header('toolroll_http_request_duration_seconds', 'Request duration; SSE measures opening headers, not stream lifetime.', 'histogram');
|
|
71
|
+
for (const [route, value] of telemetry.routes)
|
|
72
|
+
histogram('toolroll_http_request_duration_seconds', value, { route });
|
|
73
|
+
for (const [name, help, value] of [
|
|
74
|
+
['toolroll_sqlite_write_wait_seconds', 'Elapsed acquisition of an explicit write lock, including failed waits and all retry slices.', telemetry.writeWait],
|
|
75
|
+
['toolroll_sqlite_write_hold_seconds', 'Time holding an explicit write transaction, through commit or rollback.', telemetry.writeHold],
|
|
76
|
+
['toolroll_sqlite_write_statement_seconds', 'Standalone write duration, including native busy waits that SQLite cannot separate.', telemetry.writeStatement],
|
|
77
|
+
]) {
|
|
78
|
+
header(name, help, 'histogram');
|
|
79
|
+
histogram(name, value);
|
|
80
|
+
}
|
|
81
|
+
const snapshot = telemetry.snapshot();
|
|
82
|
+
header('toolroll_event_loop_delay_seconds', 'Approximate event-loop delay over the last five minutes; 20 ms sampling interval.', 'gauge');
|
|
83
|
+
for (const [quantile, value] of [['0.5', snapshot.eventLoop.p50Ms], ['0.99', snapshot.eventLoop.p99Ms]]) {
|
|
84
|
+
if (value !== null)
|
|
85
|
+
out.push(line('toolroll_event_loop_delay_seconds', { labels: { quantile }, value: value / 1000 }));
|
|
86
|
+
}
|
|
87
|
+
header('toolroll_event_loop_delay_samples', 'Event-loop samples in the recent window; zero means no measurement yet.', 'gauge');
|
|
88
|
+
out.push(line('toolroll_event_loop_delay_samples', { value: snapshot.eventLoop.count }));
|
|
89
|
+
header('toolroll_sse_connections', 'Currently open live streams.', 'gauge');
|
|
90
|
+
for (const route of STREAM_FAMILIES)
|
|
91
|
+
out.push(line('toolroll_sse_connections', { labels: { route }, value: telemetry.streamQueues(route).length }));
|
|
92
|
+
header('toolroll_sse_queue_bytes', 'Bytes currently waiting to send across open streams.', 'gauge');
|
|
93
|
+
header('toolroll_sse_queue_max_bytes', 'Largest current per-stream send queue.', 'gauge');
|
|
94
|
+
header('toolroll_sse_queue_bytes_bucket', 'Current per-stream queue depth distribution, not a lifetime histogram; each stream contributes once.', 'gauge');
|
|
95
|
+
for (const route of STREAM_FAMILIES) {
|
|
96
|
+
const queues = telemetry.streamQueues(route);
|
|
97
|
+
out.push(line('toolroll_sse_queue_bytes', { labels: { route }, value: queues.reduce((sum, n) => sum + n, 0) }));
|
|
98
|
+
out.push(line('toolroll_sse_queue_max_bytes', { labels: { route }, value: queues.reduce((max, n) => Math.max(max, n), 0) }));
|
|
99
|
+
for (const bound of [...QUEUE_BUCKETS, Infinity])
|
|
100
|
+
out.push(line('toolroll_sse_queue_bytes_bucket', { labels: { route, le: Number.isFinite(bound) ? String(bound) : '+Inf' }, value: queues.filter(n => n <= bound).length }));
|
|
101
|
+
}
|
|
102
|
+
header('toolroll_request_budget_refusals_total', 'Requests refused by the shared CLI and MCP request budget, including unavailable budget checks.', 'counter');
|
|
103
|
+
for (const route of ['api', 'mcp'])
|
|
104
|
+
for (const reason of ['read-per-minute', 'act-per-minute', 'per-day', 'unavailable']) {
|
|
105
|
+
out.push(line('toolroll_request_budget_refusals_total', { labels: { route, reason }, value: telemetry.refusals.get(`${route}:${reason}`) ?? 0 }));
|
|
106
|
+
}
|
|
107
|
+
return `${out.join('\n')}\n`;
|
|
56
108
|
}
|
package/dist/operate.js
CHANGED
|
@@ -181,6 +181,7 @@ import { WorktreePool } from "./worktree.js";
|
|
|
181
181
|
import { requestTaskStop, resumeTaskStop, taskControlOf } from "./task-control.js";
|
|
182
182
|
import { worktreeAdoptionNotice } from "./worktree-notices.js";
|
|
183
183
|
import { workIndexPage, WorkIndexCursorError } from "./work-index.js";
|
|
184
|
+
import { readExecutorOf } from "./read-executor.js";
|
|
184
185
|
import { parseWorkView } from "./workspace-ui.js";
|
|
185
186
|
import { taskWorkSummaryOf } from "./work-summary.js";
|
|
186
187
|
import { assignmentOf, assignmentBrief, syncAssignmentHandoffs } from "./assignment.js";
|
|
@@ -10756,35 +10757,47 @@ function listTasks(flags, context) {
|
|
|
10756
10757
|
!Number.isSafeInteger(limit) || limit < 1 || limit > 100) {
|
|
10757
10758
|
return fail(write, json, 'task list', 'usage', '--view takes all, needs-you, running or completed; --limit takes 1–100', EXIT.usage);
|
|
10758
10759
|
}
|
|
10759
|
-
|
|
10760
|
-
|
|
10761
|
-
|
|
10762
|
-
|
|
10763
|
-
|
|
10764
|
-
|
|
10765
|
-
|
|
10766
|
-
|
|
10767
|
-
|
|
10768
|
-
}
|
|
10769
|
-
catch (error) {
|
|
10760
|
+
// A remote person limited to some projects (remoteLensOf) pages and counts only theirs.
|
|
10761
|
+
const lens = context.principal?.lens ?? null;
|
|
10762
|
+
const access = { principal: 'operator', repos: lens, includeUnplaced: lens === null };
|
|
10763
|
+
const options = {
|
|
10764
|
+
view: parseWorkView(rawView ?? null), limit, cursor: text(flags, 'cursor') ?? null,
|
|
10765
|
+
...(wanted === undefined ? {} : { state: wanted }),
|
|
10766
|
+
...(text(flags, 'repo') === undefined ? {} : { project: text(flags, 'repo') }),
|
|
10767
|
+
};
|
|
10768
|
+
const refused = (error) => {
|
|
10770
10769
|
if (!(error instanceof WorkIndexCursorError))
|
|
10771
10770
|
throw error;
|
|
10772
10771
|
return fail(write, json, 'task list', 'usage', 'Invalid page cursor. Start again without --cursor.', EXIT.usage);
|
|
10773
|
-
}
|
|
10774
|
-
const
|
|
10775
|
-
|
|
10776
|
-
|
|
10777
|
-
|
|
10772
|
+
};
|
|
10773
|
+
const show = (page) => {
|
|
10774
|
+
const tasks = page.items.map(item => ({ ...item, id: item.rootId }));
|
|
10775
|
+
if (json) {
|
|
10776
|
+
write(envelopeJson({ ok: true, command: 'task list', count: tasks.length, tasks,
|
|
10777
|
+
totals: page.totals, nextCursor: page.nextCursor, limit: page.limit, view: page.view, evidence: 'recorded' }));
|
|
10778
|
+
return EXIT.ok;
|
|
10779
|
+
}
|
|
10780
|
+
if (tasks.length === 0)
|
|
10781
|
+
write(wanted === undefined ? 'The task list is empty.' : `Nothing is ${wanted}.`);
|
|
10782
|
+
const width = Math.max(0, ...tasks.map(task => task.id.length));
|
|
10783
|
+
for (const task of tasks)
|
|
10784
|
+
write(` ${task.id.padEnd(width)} ${task.status.label.padEnd(16)} ${task.title}`);
|
|
10785
|
+
if (page.nextCursor !== null)
|
|
10786
|
+
write(`Next page: --cursor ${page.nextCursor}`);
|
|
10778
10787
|
return EXIT.ok;
|
|
10788
|
+
};
|
|
10789
|
+
// On the server, the same page is read on a worker's read-only connection so the request loop stays free.
|
|
10790
|
+
const executor = readExecutorOf(store);
|
|
10791
|
+
if (executor !== null)
|
|
10792
|
+
return executor.workIndexPage(now, access, options).then(show, refused);
|
|
10793
|
+
let page;
|
|
10794
|
+
try {
|
|
10795
|
+
page = workIndexPage(store, now, access, options);
|
|
10779
10796
|
}
|
|
10780
|
-
|
|
10781
|
-
|
|
10782
|
-
|
|
10783
|
-
|
|
10784
|
-
write(` ${task.id.padEnd(width)} ${task.status.label.padEnd(16)} ${task.title}`);
|
|
10785
|
-
if (page.nextCursor !== null)
|
|
10786
|
-
write(`Next page: --cursor ${page.nextCursor}`);
|
|
10787
|
-
return EXIT.ok;
|
|
10797
|
+
catch (error) {
|
|
10798
|
+
return refused(error);
|
|
10799
|
+
}
|
|
10800
|
+
return show(page);
|
|
10788
10801
|
}
|
|
10789
10802
|
function showTask(positional, context) {
|
|
10790
10803
|
const { store, write, json, now } = context;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import { parentPort, workerData } from 'node:worker_threads';
|
|
2
|
+
import { openStoreReadOnly } from './store.js';
|
|
3
|
+
import { workIndexPage } from './work-index.js';
|
|
4
|
+
/** One read-only, query-only connection per worker (read-executor.ts). Jobs arrive one at a time. */
|
|
5
|
+
let store = null;
|
|
6
|
+
function connection() {
|
|
7
|
+
if (store !== null)
|
|
8
|
+
return store;
|
|
9
|
+
const opened = openStoreReadOnly(String(workerData.file));
|
|
10
|
+
if (opened === null)
|
|
11
|
+
throw new Error('The read worker could not open the database read-only at the current schema.');
|
|
12
|
+
opened.handle.exec('PRAGMA query_only=1');
|
|
13
|
+
return store = opened;
|
|
14
|
+
}
|
|
15
|
+
parentPort.on('message', ({ id, job }) => {
|
|
16
|
+
let reply;
|
|
17
|
+
try {
|
|
18
|
+
if (job.kind !== 'work-index-page')
|
|
19
|
+
throw new Error('Unknown read.');
|
|
20
|
+
reply = { id, ok: true, value: workIndexPage(connection(), new Date(job.now), job.access, job.options) };
|
|
21
|
+
}
|
|
22
|
+
catch (error) {
|
|
23
|
+
reply = { id, ok: false, error: { name: error instanceof Error ? error.name : 'Error', message: error instanceof Error ? error.message : String(error) } };
|
|
24
|
+
}
|
|
25
|
+
parentPort.postMessage(reply);
|
|
26
|
+
});
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import type { Store } from './store.js';
|
|
2
|
+
import { type WorkIndexOptions, type WorkIndexPage } from './work-index.js';
|
|
3
|
+
import type { WorkSummaryAccess } from './work-summary.js';
|
|
4
|
+
/**
|
|
5
|
+
* Heavy, read-only database work for the server, run on worker threads (docs/server-performance.md).
|
|
6
|
+
* Each worker owns one read-only, query-only connection to the server's own database file, so the
|
|
7
|
+
* measured hot read (the work index behind `task list`) never holds the HTTP event loop. Callers
|
|
8
|
+
* authorize first and send only the already-admitted access; a worker never widens it, writes, or
|
|
9
|
+
* retries. Without a file-backed database (tests, the demo) nothing is attached and callers keep
|
|
10
|
+
* running the same builder in-process.
|
|
11
|
+
*/
|
|
12
|
+
export type ReadJob = {
|
|
13
|
+
kind: 'work-index-page';
|
|
14
|
+
now: string;
|
|
15
|
+
access: WorkSummaryAccess;
|
|
16
|
+
options: WorkIndexOptions;
|
|
17
|
+
};
|
|
18
|
+
export type ReadReply = {
|
|
19
|
+
id: number;
|
|
20
|
+
ok: true;
|
|
21
|
+
value: unknown;
|
|
22
|
+
} | {
|
|
23
|
+
id: number;
|
|
24
|
+
ok: false;
|
|
25
|
+
error: {
|
|
26
|
+
name: string;
|
|
27
|
+
message: string;
|
|
28
|
+
};
|
|
29
|
+
};
|
|
30
|
+
/** More queued reads than this fail at once instead of growing without bound. */
|
|
31
|
+
export declare const READ_QUEUE_MAX = 1000;
|
|
32
|
+
export declare class ReadExecutorError extends Error {
|
|
33
|
+
constructor(message: string);
|
|
34
|
+
}
|
|
35
|
+
export declare class ReadExecutor {
|
|
36
|
+
#private;
|
|
37
|
+
constructor(file: string, size?: number);
|
|
38
|
+
get size(): number;
|
|
39
|
+
/** workIndexPage(store, now, access, options) on a worker's read-only connection: same rows, same errors. */
|
|
40
|
+
workIndexPage(now: Date, access: WorkSummaryAccess, options: WorkIndexOptions): Promise<WorkIndexPage>;
|
|
41
|
+
/** Fails every queued and running read once, then stops the workers. */
|
|
42
|
+
close(): Promise<void>;
|
|
43
|
+
}
|
|
44
|
+
/** Half the cores, at most eight: the measured 20-engineer load needs over four (docs/server-performance.md),
|
|
45
|
+
* and the rest stay with the event loop and local agents. Workers start only when reads arrive. */
|
|
46
|
+
export declare function defaultReadWorkers(): number;
|
|
47
|
+
/** The server attaches its executor to its own Store; a command run on that Store reads through it. */
|
|
48
|
+
export declare function attachReadExecutor(store: Store, executor: ReadExecutor): () => void;
|
|
49
|
+
export declare function readExecutorOf(store: Store): ReadExecutor | null;
|