toolroll 0.9.60 → 0.9.62
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +32 -0
- package/dist/agent-board.d.ts +208 -0
- package/dist/agent-board.js +454 -0
- package/dist/agent-fence.d.ts +19 -0
- package/dist/agent-fence.js +100 -8
- package/dist/agent-pulse.d.ts +43 -0
- package/dist/agent-pulse.js +214 -0
- package/dist/agent-turns.d.ts +52 -0
- package/dist/agent-turns.js +137 -0
- package/dist/agents-board.d.ts +101 -0
- package/dist/agents-board.js +225 -0
- package/dist/assignment-adapters.d.ts +1 -0
- package/dist/assignment-status.d.ts +1 -1
- package/dist/assignment.d.ts +4 -0
- package/dist/assignment.js +8 -6
- package/dist/browser/workspace.css +1 -1
- package/dist/browser/workspace.js +29 -25
- package/dist/browser-crew.js +2 -0
- package/dist/browser-workspace.d.ts +79 -1
- package/dist/browser-workspace.js +1 -0
- package/dist/builder.d.ts +8 -0
- package/dist/builder.js +113 -13
- package/dist/chat-channel.js +9 -0
- package/dist/chat-core.d.ts +6 -0
- package/dist/chat-core.js +8 -0
- package/dist/chat-delivery.js +22 -0
- package/dist/chat-messages.js +8 -1
- package/dist/chat-quiet.js +1 -0
- package/dist/chat-voice.d.ts +2 -0
- package/dist/chat-voice.js +2 -0
- package/dist/checkout-cleanup.d.ts +3 -0
- package/dist/checkout-cleanup.js +11 -2
- package/dist/cli.js +28 -11
- package/dist/contracts/console-api.d.ts +3 -1
- package/dist/contracts/console-api.js +5 -1
- package/dist/contracts/lead-tools.d.ts +16 -2
- package/dist/contracts/lead-tools.js +5 -1
- package/dist/desktop-update.d.ts +1 -1
- package/dist/evidence-pack.js +1 -1
- package/dist/evidence.d.ts +6 -0
- package/dist/evidence.js +14 -3
- package/dist/exec.d.ts +32 -0
- package/dist/exec.js +229 -10
- package/dist/invoke.js +110 -18
- package/dist/landed-cleanup.d.ts +170 -0
- package/dist/landed-cleanup.js +814 -0
- package/dist/lead-cli.js +22 -19
- package/dist/lead-contract.d.ts +1 -1
- package/dist/lead-contract.js +4 -4
- package/dist/lead-doors.js +45 -1
- package/dist/lead-tools.js +40 -7
- package/dist/live-input.d.ts +68 -0
- package/dist/live-input.js +254 -0
- package/dist/live.d.ts +1 -1
- package/dist/live.js +4 -2
- package/dist/maintenance.d.ts +2 -0
- package/dist/maintenance.js +5 -2
- package/dist/operate-remote-arguments.js +5 -1
- package/dist/operate-remote.js +1 -1
- package/dist/operate.d.ts +1 -1
- package/dist/operate.js +249 -26
- package/dist/orphan-sweep.d.ts +2 -0
- package/dist/orphan-sweep.js +5 -3
- package/dist/project-admission.d.ts +10 -6
- package/dist/project-admission.js +33 -11
- package/dist/project-delete-ui.d.ts +12 -0
- package/dist/project-delete-ui.js +32 -2
- package/dist/project-tools.d.ts +7 -11
- package/dist/project-tools.js +13 -37
- package/dist/provider.d.ts +65 -8
- package/dist/provider.js +80 -21
- package/dist/remote-main.d.ts +57 -0
- package/dist/remote-main.js +179 -0
- package/dist/result-completion.d.ts +7 -0
- package/dist/result-completion.js +9 -0
- package/dist/runner.d.ts +4 -2
- package/dist/runner.js +3 -3
- package/dist/serve.js +19 -1
- package/dist/server/live.d.ts +3 -0
- package/dist/server/live.js +35 -2
- package/dist/server/pages.js +59 -35
- package/dist/server/render-chat.js +12 -0
- package/dist/server/render-tasks.d.ts +15 -2
- package/dist/server/render-tasks.js +30 -7
- package/dist/server/route-table.js +4 -1
- package/dist/server/settings.js +5 -3
- package/dist/server/task-views.js +11 -0
- package/dist/server/tasks.js +63 -0
- package/dist/shared-deps.d.ts +9 -4
- package/dist/shared-deps.js +89 -13
- package/dist/staged-runtimes.d.ts +16 -4
- package/dist/staged-runtimes.js +160 -38
- package/dist/steer-live.d.ts +19 -0
- package/dist/steer-live.js +93 -0
- package/dist/steer-status.d.ts +8 -2
- package/dist/steer-status.js +19 -5
- package/dist/storage-sweep-child.d.ts +1 -0
- package/dist/storage-sweep-child.js +76 -0
- package/dist/storage-sweep-process.d.ts +48 -0
- package/dist/storage-sweep-process.js +204 -0
- package/dist/storage-sweep-schedule.d.ts +30 -0
- package/dist/storage-sweep-schedule.js +164 -0
- package/dist/storage-sweep.d.ts +31 -6
- package/dist/storage-sweep.js +165 -39
- package/dist/storage-ui.d.ts +4 -0
- package/dist/storage-ui.js +22 -7
- package/dist/storage.d.ts +6 -0
- package/dist/storage.js +35 -0
- package/dist/store.d.ts +200 -3
- package/dist/store.js +443 -17
- package/dist/surface.js +8 -4
- package/dist/sweep-quarantine.d.ts +28 -0
- package/dist/sweep-quarantine.js +153 -0
- package/dist/task-base.d.ts +35 -0
- package/dist/task-base.js +51 -0
- package/dist/task-status.d.ts +2 -0
- package/dist/task-status.js +5 -1
- package/dist/test-temp.d.ts +6 -3
- package/dist/test-temp.js +15 -9
- package/dist/toolroll-update.js +4 -2
- package/dist/work-index.d.ts +1 -0
- package/dist/work-index.js +32 -12
- package/dist/work-summary.d.ts +2 -2
- package/dist/workspace-revision.js +3 -1
- package/dist/worktree.d.ts +57 -3
- package/dist/worktree.js +201 -23
- package/package.json +1 -1
- package/scripts/provider-canary.mjs +235 -147
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
/** Launched only by storage-sweep-process. All potentially expensive sweep work stays in this process. */
|
|
2
|
+
import { Worker } from "node:worker_threads";
|
|
3
|
+
import { openStoreNoMigrate } from "./store.js";
|
|
4
|
+
import { WorktreePool } from "./worktree.js";
|
|
5
|
+
import { executeSweep } from "./storage-sweep.js";
|
|
6
|
+
import { sweepRequest } from "./storage-sweep-process.js";
|
|
7
|
+
const parent = process.ppid;
|
|
8
|
+
let store;
|
|
9
|
+
let finished = false;
|
|
10
|
+
const closeStore = () => { const opened = store; store = undefined; opened?.close(); };
|
|
11
|
+
const abandon = () => {
|
|
12
|
+
if (finished)
|
|
13
|
+
return;
|
|
14
|
+
finished = true;
|
|
15
|
+
try {
|
|
16
|
+
closeStore();
|
|
17
|
+
}
|
|
18
|
+
finally {
|
|
19
|
+
// This detached child owns the group, including any blocked rm/du descendants.
|
|
20
|
+
try {
|
|
21
|
+
if (process.platform !== "win32")
|
|
22
|
+
process.kill(-process.pid, "SIGKILL");
|
|
23
|
+
}
|
|
24
|
+
catch { /* no group */ }
|
|
25
|
+
process.exit(1);
|
|
26
|
+
}
|
|
27
|
+
};
|
|
28
|
+
process.once("disconnect", abandon);
|
|
29
|
+
process.once("SIGTERM", abandon);
|
|
30
|
+
// A separate thread also detects parent loss while synchronous filesystem work blocks the child's event loop.
|
|
31
|
+
const watchdog = new Worker(`
|
|
32
|
+
const { workerData } = require('node:worker_threads');
|
|
33
|
+
setInterval(() => {
|
|
34
|
+
let lost = process.ppid !== workerData.parent || workerData.parent <= 1;
|
|
35
|
+
try { process.kill(workerData.parent, 0); } catch (error) { if (error.code === 'ESRCH') lost = true; }
|
|
36
|
+
if (!lost) return;
|
|
37
|
+
try { if (process.platform !== 'win32') process.kill(-process.pid, 'SIGKILL'); } catch {}
|
|
38
|
+
process.kill(process.pid, 'SIGKILL');
|
|
39
|
+
}, 1000);
|
|
40
|
+
`, { eval: true, workerData: { parent } });
|
|
41
|
+
watchdog.unref();
|
|
42
|
+
watchdog.once("error", abandon);
|
|
43
|
+
const finish = (code) => {
|
|
44
|
+
if (finished)
|
|
45
|
+
return;
|
|
46
|
+
finished = true;
|
|
47
|
+
closeStore();
|
|
48
|
+
void watchdog.terminate();
|
|
49
|
+
process.exitCode = code;
|
|
50
|
+
if (process.connected)
|
|
51
|
+
process.disconnect?.();
|
|
52
|
+
};
|
|
53
|
+
if (parent <= 1 || !process.connected)
|
|
54
|
+
abandon();
|
|
55
|
+
process.once("message", async (request) => {
|
|
56
|
+
if (!sweepRequest(request)) {
|
|
57
|
+
finish(1);
|
|
58
|
+
return;
|
|
59
|
+
}
|
|
60
|
+
try {
|
|
61
|
+
const opened = openStoreNoMigrate(request.databaseFile);
|
|
62
|
+
if (!opened.ok)
|
|
63
|
+
throw new Error("Sweep database unavailable");
|
|
64
|
+
store = opened.store;
|
|
65
|
+
const started = Date.now();
|
|
66
|
+
const now = () => new Date(Date.parse(request.at) + Date.now() - started);
|
|
67
|
+
const pool = request.poolRoot === null ? null : new WorktreePool(store, { root: request.poolRoot });
|
|
68
|
+
const record = await executeSweep(store, { databaseFile: request.databaseFile, pool, tempRoots: request.tempRoots, actor: request.actor }, now);
|
|
69
|
+
closeStore();
|
|
70
|
+
if (!finished)
|
|
71
|
+
process.send?.({ version: 1, record }, error => finish(error === null ? 0 : 1));
|
|
72
|
+
}
|
|
73
|
+
catch {
|
|
74
|
+
finish(1);
|
|
75
|
+
}
|
|
76
|
+
});
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
/** The storage child's bounded IPC protocol. No paths or process arguments are written to logs. */
|
|
2
|
+
import { type ChildProcess } from "node:child_process";
|
|
3
|
+
import type { SweepRecord } from "./storage-sweep.js";
|
|
4
|
+
export type SweepRequest = {
|
|
5
|
+
version: 1;
|
|
6
|
+
databaseFile: string;
|
|
7
|
+
poolRoot: string | null;
|
|
8
|
+
tempRoots: string[];
|
|
9
|
+
actor: string;
|
|
10
|
+
at: string;
|
|
11
|
+
};
|
|
12
|
+
export type SweepExecutor = (request: SweepRequest) => Promise<SweepRecord>;
|
|
13
|
+
export declare function sweepRequest(value: unknown): value is SweepRequest;
|
|
14
|
+
export declare function sweepResponse(value: unknown, request: SweepRequest): value is {
|
|
15
|
+
version: 1;
|
|
16
|
+
record: SweepRecord;
|
|
17
|
+
};
|
|
18
|
+
export type SweepCompletion = {
|
|
19
|
+
ok: true;
|
|
20
|
+
record: SweepRecord;
|
|
21
|
+
} | {
|
|
22
|
+
ok: false;
|
|
23
|
+
reason: "failed" | "timeout" | "stopped";
|
|
24
|
+
};
|
|
25
|
+
export type SweepSupervisor = {
|
|
26
|
+
readonly running: boolean;
|
|
27
|
+
start: (request: SweepRequest, complete: (result: SweepCompletion) => void, owner?: object) => boolean;
|
|
28
|
+
stop: (owner?: object) => void;
|
|
29
|
+
};
|
|
30
|
+
export declare const SWEEP_TIMEOUT_MS: number;
|
|
31
|
+
export declare const SWEEP_KILL_GRACE_MS = 5000;
|
|
32
|
+
/** The sweep runs behind everything else: a positive nice and, on macOS, the background CPU and I/O policy. */
|
|
33
|
+
export declare const SWEEP_NICE = 10;
|
|
34
|
+
/** How the child starts: taskpolicy execs Node in place, so the pid, its group and the IPC channel stay the same. */
|
|
35
|
+
export declare function sweepLaunch(platform?: NodeJS.Platform, exists?: (path: string) => boolean): {
|
|
36
|
+
execPath: string;
|
|
37
|
+
prefix: string[];
|
|
38
|
+
};
|
|
39
|
+
/** Owns one child without lending its lifetime to a reconcile pass or its database connection. */
|
|
40
|
+
export declare function createSweepSupervisor(options?: {
|
|
41
|
+
timeoutMs?: number;
|
|
42
|
+
killGraceMs?: number;
|
|
43
|
+
launch?: () => ChildProcess;
|
|
44
|
+
signal?: (child: ChildProcess, signal: NodeJS.Signals) => void;
|
|
45
|
+
}): SweepSupervisor;
|
|
46
|
+
export declare function storageSweepSupervisor(databaseFile: string): SweepSupervisor;
|
|
47
|
+
/** Awaitable only for explicit callers/tests; automatic maintenance always uses start(). */
|
|
48
|
+
export declare const runSweepChild: SweepExecutor;
|
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
/** The storage child's bounded IPC protocol. No paths or process arguments are written to logs. */
|
|
2
|
+
import { fork } from "node:child_process";
|
|
3
|
+
import { existsSync } from "node:fs";
|
|
4
|
+
import { setPriority } from "node:os";
|
|
5
|
+
import { resolve } from "node:path";
|
|
6
|
+
import { fileURLToPath } from "node:url";
|
|
7
|
+
const object = (value) => typeof value === "object" && value !== null && !Array.isArray(value);
|
|
8
|
+
const date = (value) => typeof value === "string" && Number.isFinite(Date.parse(value));
|
|
9
|
+
const strings = (value) => Array.isArray(value) && value.every(one => typeof one === "string");
|
|
10
|
+
export function sweepRequest(value) {
|
|
11
|
+
return object(value) && value.version === 1 && typeof value.databaseFile === "string" && value.databaseFile !== ":memory:" &&
|
|
12
|
+
(value.poolRoot === null || typeof value.poolRoot === "string") && strings(value.tempRoots) && typeof value.actor === "string" && date(value.at);
|
|
13
|
+
}
|
|
14
|
+
export function sweepResponse(value, request) {
|
|
15
|
+
if (!object(value) || value.version !== 1 || !object(value.record))
|
|
16
|
+
return false;
|
|
17
|
+
const record = value.record;
|
|
18
|
+
const kinds = ["test temp", "orphans", ...(request.poolRoot === null ? [] : ["checkouts"]), "runtimes", "dependencies"];
|
|
19
|
+
if (!date(record.at) || record.source !== "automatic" || record.actor !== request.actor || !Array.isArray(record.parts) || record.parts.length !== kinds.length)
|
|
20
|
+
return false;
|
|
21
|
+
return record.parts.every((part, index) => object(part) && part.kind === kinds[index] &&
|
|
22
|
+
[part.count, part.bytes, part.failed].every(n => typeof n === "number" && Number.isSafeInteger(n) && n >= 0) &&
|
|
23
|
+
strings(part.items) && part.items.length <= 20 && part.items.every(item => item.length <= 16_384) &&
|
|
24
|
+
(part.more === undefined || typeof part.more === "boolean") && (part.problem === undefined || (typeof part.problem === "string" && part.problem.length <= 65_536)));
|
|
25
|
+
}
|
|
26
|
+
export const SWEEP_TIMEOUT_MS = 20 * 60_000;
|
|
27
|
+
export const SWEEP_KILL_GRACE_MS = 5_000;
|
|
28
|
+
/** The sweep runs behind everything else: a positive nice and, on macOS, the background CPU and I/O policy. */
|
|
29
|
+
export const SWEEP_NICE = 10;
|
|
30
|
+
const TASKPOLICY = "/usr/sbin/taskpolicy";
|
|
31
|
+
/** How the child starts: taskpolicy execs Node in place, so the pid, its group and the IPC channel stay the same. */
|
|
32
|
+
export function sweepLaunch(platform = process.platform, exists = existsSync) {
|
|
33
|
+
return platform === "darwin" && exists(TASKPOLICY) ? { execPath: TASKPOLICY, prefix: ["-b", process.execPath] } : { execPath: process.execPath, prefix: [] };
|
|
34
|
+
}
|
|
35
|
+
/** Only source checkouts need tsx; installed builds use the sibling child and the same Node. */
|
|
36
|
+
function forkSweep() {
|
|
37
|
+
const source = import.meta.url.endsWith(".ts");
|
|
38
|
+
const launch = sweepLaunch();
|
|
39
|
+
const child = fork(fileURLToPath(new URL(source ? "./storage-sweep-child.ts" : "./storage-sweep-child.js", import.meta.url)), [], {
|
|
40
|
+
execPath: launch.execPath, execArgv: [...launch.prefix, ...(source ? ["--import", "tsx"] : [])], stdio: ["ignore", "ignore", "ignore", "ipc"], detached: true,
|
|
41
|
+
});
|
|
42
|
+
// Before the request is sent, so every rm/du the child starts inherits it.
|
|
43
|
+
if (child.pid !== undefined) {
|
|
44
|
+
try {
|
|
45
|
+
setPriority(child.pid, SWEEP_NICE);
|
|
46
|
+
}
|
|
47
|
+
catch { /* the default priority still works */ }
|
|
48
|
+
}
|
|
49
|
+
// The service never waits on the child to exit; stop() and the child's parent-loss guard end it.
|
|
50
|
+
child.unref();
|
|
51
|
+
child.channel?.unref();
|
|
52
|
+
return child;
|
|
53
|
+
}
|
|
54
|
+
/** The fork is a group leader: rm/du and any other descendants must receive the same signal. */
|
|
55
|
+
function signalGroup(child, signal) {
|
|
56
|
+
if (child.pid === undefined)
|
|
57
|
+
return;
|
|
58
|
+
if (process.platform !== "win32") {
|
|
59
|
+
try {
|
|
60
|
+
process.kill(-child.pid, signal);
|
|
61
|
+
return;
|
|
62
|
+
}
|
|
63
|
+
catch (error) {
|
|
64
|
+
if (error.code === "ESRCH")
|
|
65
|
+
return;
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
try {
|
|
69
|
+
child.kill(signal);
|
|
70
|
+
}
|
|
71
|
+
catch { /* already gone */ }
|
|
72
|
+
}
|
|
73
|
+
/** Owns one child without lending its lifetime to a reconcile pass or its database connection. */
|
|
74
|
+
export function createSweepSupervisor(options = {}) {
|
|
75
|
+
const timeoutMs = options.timeoutMs ?? SWEEP_TIMEOUT_MS;
|
|
76
|
+
const killGraceMs = options.killGraceMs ?? SWEEP_KILL_GRACE_MS;
|
|
77
|
+
for (const value of [timeoutMs, killGraceMs]) {
|
|
78
|
+
if (!Number.isSafeInteger(value) || value < 1 || value > 2_147_483_647)
|
|
79
|
+
throw new Error("Invalid sweep process time limit");
|
|
80
|
+
}
|
|
81
|
+
const signal = options.signal ?? signalGroup;
|
|
82
|
+
let active = null;
|
|
83
|
+
const stoppedOwners = new WeakSet();
|
|
84
|
+
return {
|
|
85
|
+
get running() { return active !== null; },
|
|
86
|
+
stop(owner) {
|
|
87
|
+
if (owner !== undefined)
|
|
88
|
+
stoppedOwners.add(owner);
|
|
89
|
+
if (owner === undefined || active?.owner === owner)
|
|
90
|
+
active?.stop();
|
|
91
|
+
},
|
|
92
|
+
start(request, complete, owner) {
|
|
93
|
+
if (active !== null || (owner !== undefined && stoppedOwners.has(owner)))
|
|
94
|
+
return false;
|
|
95
|
+
let child;
|
|
96
|
+
let settled = false, closed = false, killing = false, killed = false;
|
|
97
|
+
let record = null;
|
|
98
|
+
let deadline;
|
|
99
|
+
let grace;
|
|
100
|
+
const settle = (result) => {
|
|
101
|
+
if (settled)
|
|
102
|
+
return;
|
|
103
|
+
settled = true;
|
|
104
|
+
clearTimeout(deadline);
|
|
105
|
+
complete(result);
|
|
106
|
+
};
|
|
107
|
+
const release = () => {
|
|
108
|
+
if (!closed || grace !== undefined)
|
|
109
|
+
return;
|
|
110
|
+
clearTimeout(deadline);
|
|
111
|
+
child.removeListener("message", message);
|
|
112
|
+
child.removeListener("error", error);
|
|
113
|
+
child.removeListener("close", close);
|
|
114
|
+
// A late IPC write error must not become an unhandled EventEmitter error.
|
|
115
|
+
child.on("error", () => { });
|
|
116
|
+
active = null;
|
|
117
|
+
};
|
|
118
|
+
const kill = () => {
|
|
119
|
+
if (killed)
|
|
120
|
+
return;
|
|
121
|
+
killed = true;
|
|
122
|
+
killing = true;
|
|
123
|
+
clearTimeout(grace);
|
|
124
|
+
grace = undefined;
|
|
125
|
+
signal(child, "SIGKILL");
|
|
126
|
+
release();
|
|
127
|
+
};
|
|
128
|
+
const terminate = (reason) => {
|
|
129
|
+
if (settled)
|
|
130
|
+
return;
|
|
131
|
+
killing = true;
|
|
132
|
+
signal(child, "SIGTERM");
|
|
133
|
+
// Even if the leader exits on TERM, descendants may ignore it. Keep the group owned through KILL.
|
|
134
|
+
grace = setTimeout(kill, killGraceMs);
|
|
135
|
+
grace.unref?.();
|
|
136
|
+
settle({ ok: false, reason });
|
|
137
|
+
};
|
|
138
|
+
const message = (value) => {
|
|
139
|
+
if (settled)
|
|
140
|
+
return;
|
|
141
|
+
if (record !== null || !sweepResponse(value, request)) {
|
|
142
|
+
terminate("failed");
|
|
143
|
+
return;
|
|
144
|
+
}
|
|
145
|
+
record = value.record;
|
|
146
|
+
};
|
|
147
|
+
const error = () => terminate("failed");
|
|
148
|
+
const close = (code) => {
|
|
149
|
+
closed = true;
|
|
150
|
+
if (!settled) {
|
|
151
|
+
if (code === 0 && record !== null)
|
|
152
|
+
settle({ ok: true, record });
|
|
153
|
+
else {
|
|
154
|
+
signal(child, "SIGKILL");
|
|
155
|
+
settle({ ok: false, reason: "failed" });
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
if (!killing || grace === undefined)
|
|
159
|
+
release();
|
|
160
|
+
};
|
|
161
|
+
try {
|
|
162
|
+
child = (options.launch ?? forkSweep)();
|
|
163
|
+
}
|
|
164
|
+
catch {
|
|
165
|
+
complete({ ok: false, reason: "failed" });
|
|
166
|
+
return true;
|
|
167
|
+
}
|
|
168
|
+
active = { owner, stop: () => { kill(); settle({ ok: false, reason: "stopped" }); } };
|
|
169
|
+
child.on("message", message);
|
|
170
|
+
child.on("error", error);
|
|
171
|
+
child.once("close", close);
|
|
172
|
+
deadline = setTimeout(() => terminate("timeout"), timeoutMs);
|
|
173
|
+
deadline.unref?.();
|
|
174
|
+
try {
|
|
175
|
+
child.send(request, error => { if (error !== null)
|
|
176
|
+
terminate("failed"); });
|
|
177
|
+
}
|
|
178
|
+
catch {
|
|
179
|
+
terminate("failed");
|
|
180
|
+
}
|
|
181
|
+
return true;
|
|
182
|
+
},
|
|
183
|
+
};
|
|
184
|
+
}
|
|
185
|
+
// All watch loops in this process share a flight. Other services arbitrate the claim in SQLite.
|
|
186
|
+
const supervisors = new Map();
|
|
187
|
+
export function storageSweepSupervisor(databaseFile) {
|
|
188
|
+
const key = resolve(databaseFile);
|
|
189
|
+
let supervisor = supervisors.get(key);
|
|
190
|
+
if (supervisor === undefined) {
|
|
191
|
+
supervisor = createSweepSupervisor();
|
|
192
|
+
supervisors.set(key, supervisor);
|
|
193
|
+
}
|
|
194
|
+
return supervisor;
|
|
195
|
+
}
|
|
196
|
+
/** Awaitable only for explicit callers/tests; automatic maintenance always uses start(). */
|
|
197
|
+
export const runSweepChild = request => new Promise((resolve, reject) => {
|
|
198
|
+
createSweepSupervisor().start(request, result => {
|
|
199
|
+
if (result.ok)
|
|
200
|
+
resolve(result.record);
|
|
201
|
+
else
|
|
202
|
+
reject(new Error("Sweep child failed"));
|
|
203
|
+
});
|
|
204
|
+
});
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import { type SweepDeps, type SweepRecord } from "./storage-sweep.js";
|
|
2
|
+
import { type SweepSupervisor } from "./storage-sweep-process.js";
|
|
3
|
+
import { type Store } from "./store.js";
|
|
4
|
+
/** How often the service looks again at a due sweep. */
|
|
5
|
+
export declare const SWEEP_CHECK_MS: number;
|
|
6
|
+
/** After this long without an automatic sweep, it runs even while work is running or waiting. */
|
|
7
|
+
export declare const SWEEP_FORCE_MS: number;
|
|
8
|
+
export type SweepBusy = "run" | "claim" | "ready";
|
|
9
|
+
/** What keeps a due sweep waiting now, or null when nothing is running or waiting. */
|
|
10
|
+
export declare function sweepBusy(store: Store, now: Date): SweepBusy | null;
|
|
11
|
+
/** Whether a long-lived Toolroll service holds a current lease. Rows from earlier services don't count. */
|
|
12
|
+
export declare function sweepServiceRunning(store: Store, now: Date): boolean;
|
|
13
|
+
export type SweepState = "off" | "sweeping" | "no-service" | "recent" | "retry" | "busy" | "forced" | "due";
|
|
14
|
+
export type SweepSchedule = {
|
|
15
|
+
state: SweepState;
|
|
16
|
+
/** The latest automatic sweep's recorded time, or null when there has been none. */
|
|
17
|
+
lastAt: string | null;
|
|
18
|
+
/** When the next sweep is due, or null when it is off or due now. */
|
|
19
|
+
nextAt: string | null;
|
|
20
|
+
busy: SweepBusy | null;
|
|
21
|
+
};
|
|
22
|
+
/** The sweep's schedule from what is recorded now; reads only. */
|
|
23
|
+
export declare function sweepSchedule(store: Store, now: Date, env?: NodeJS.ProcessEnv): SweepSchedule;
|
|
24
|
+
/** Why the sweep is waiting or when it is next due, after "Automatic sweep: ". `when` formats a time for the reader. */
|
|
25
|
+
export declare function sweepScheduleWords(schedule: SweepSchedule, when: (at: string) => string): string;
|
|
26
|
+
/**
|
|
27
|
+
* The service's claim and launch. A one-shot caller has no `owner` and never claims. A hung child never holds
|
|
28
|
+
* reconciliation or the caller's store open; the result is saved by the flight's own connection.
|
|
29
|
+
*/
|
|
30
|
+
export declare function idleStorageSweep(store: Store, deps: SweepDeps, now: () => Date, supervisor?: SweepSupervisor): SweepRecord | null;
|
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* When the automatic storage sweep (storage-sweep.ts) starts, and what `toolroll storage` and Settings → Storage say
|
|
3
|
+
* about it. A sweep is due a day after the last one (an hour after one that failed somewhere). Once due, the
|
|
4
|
+
* long-lived Toolroll service looks at most every 15 minutes and starts it only while nothing is running or waiting:
|
|
5
|
+
* no open run, no live claim and no task a worker could claim now. Work held by an approval, a hold or a provider limit
|
|
6
|
+
* is not waiting to start, so it never defers the sweep. 48 hours after the last automatic sweep it runs anyway: its
|
|
7
|
+
* scans run in a low-priority child, and every removal asks again whether live work uses the folder.
|
|
8
|
+
*
|
|
9
|
+
* Only the service claims a sweep (it passes its owner): a one-shot `toolroll reconcile` exits before a child would
|
|
10
|
+
* finish, and a claim it made would be lost until the next day. The idle gates are asked again inside the transaction
|
|
11
|
+
* that claims the sweep, so work that started since the first look still defers it.
|
|
12
|
+
*/
|
|
13
|
+
import { diagnoseTaskDispatch, diagnosisIsDispatchable } from "./dispatch.js";
|
|
14
|
+
import { lastSweep, RETRY_MS, saveSweep, storageSweepOff, sweepFailure, SWEEP_CURSOR, SWEEP_EVERY_MS } from "./storage-sweep.js";
|
|
15
|
+
import { storageSweepSupervisor, SWEEP_KILL_GRACE_MS, SWEEP_TIMEOUT_MS } from "./storage-sweep-process.js";
|
|
16
|
+
import { openStoreNoMigrate } from "./store.js";
|
|
17
|
+
/** How often the service looks again at a due sweep. */
|
|
18
|
+
export const SWEEP_CHECK_MS = 15 * 60_000;
|
|
19
|
+
/** After this long without an automatic sweep, it runs even while work is running or waiting. */
|
|
20
|
+
export const SWEEP_FORCE_MS = 48 * 3_600_000;
|
|
21
|
+
/** What keeps a due sweep waiting now, or null when nothing is running or waiting. */
|
|
22
|
+
export function sweepBusy(store, now) {
|
|
23
|
+
const db = store.handle;
|
|
24
|
+
if (db.prepare("SELECT 1 FROM run WHERE finished_at IS NULL LIMIT 1").get() !== undefined)
|
|
25
|
+
return "run";
|
|
26
|
+
if (db.prepare("SELECT 1 FROM claim WHERE released_at IS NULL AND expires_at > ? LIMIT 1").get(now.toISOString()) !== undefined)
|
|
27
|
+
return "claim";
|
|
28
|
+
// Ready means a worker could claim it now: the same answer the task page gives, not merely queued.
|
|
29
|
+
if (store.listReady(now).some(ref => diagnosisIsDispatchable(diagnoseTaskDispatch(store, ref.externalId, now))))
|
|
30
|
+
return "ready";
|
|
31
|
+
return null;
|
|
32
|
+
}
|
|
33
|
+
/** Whether a long-lived Toolroll service holds a current lease. Rows from earlier services don't count. */
|
|
34
|
+
export function sweepServiceRunning(store, now) {
|
|
35
|
+
return store.handle.prepare("SELECT 1 FROM watch_lease WHERE owner GLOB 'toolroll-service:*' AND expires_at > ? LIMIT 1").get(now.toISOString()) !== undefined;
|
|
36
|
+
}
|
|
37
|
+
const dueAt = (store) => store.serviceCursor(SWEEP_CURSOR) + SWEEP_EVERY_MS;
|
|
38
|
+
/** Count from the last result, or the durable installation time before the first result. */
|
|
39
|
+
function forcedFrom(store, last) {
|
|
40
|
+
if (last !== null)
|
|
41
|
+
return Date.parse(last.at);
|
|
42
|
+
const installed = store.installationFact("installed-at");
|
|
43
|
+
return installed === null ? null : Date.parse(installed);
|
|
44
|
+
}
|
|
45
|
+
/** The sweep's schedule from what is recorded now; reads only. */
|
|
46
|
+
export function sweepSchedule(store, now, env = process.env) {
|
|
47
|
+
const last = lastSweep(store, "automatic");
|
|
48
|
+
const lastAt = last?.at ?? null;
|
|
49
|
+
if (storageSweepOff(env))
|
|
50
|
+
return { state: "off", lastAt, nextAt: null, busy: null };
|
|
51
|
+
const cursor = store.serviceCursor(SWEEP_CURSOR);
|
|
52
|
+
// Results record completion time, which may be later than the claim. An older result cannot finish a new claim.
|
|
53
|
+
if (cursor !== 0 && now.getTime() - cursor < SWEEP_TIMEOUT_MS + SWEEP_KILL_GRACE_MS && now.getTime() >= cursor && (lastAt === null || Date.parse(lastAt) < cursor)) {
|
|
54
|
+
return { state: "sweeping", lastAt, nextAt: null, busy: null };
|
|
55
|
+
}
|
|
56
|
+
const due = dueAt(store);
|
|
57
|
+
const nextAt = cursor === 0 || due <= now.getTime() ? null : new Date(due).toISOString();
|
|
58
|
+
if (!sweepServiceRunning(store, now))
|
|
59
|
+
return { state: "no-service", lastAt, nextAt, busy: null };
|
|
60
|
+
if (nextAt !== null) {
|
|
61
|
+
// Completion can follow the claim: elapsed sweep time alone does not mean a retry.
|
|
62
|
+
const retry = last !== null && Date.parse(last.at) >= cursor && last.parts.some(one => one.failed > 0 || one.more === true || one.problem !== undefined);
|
|
63
|
+
return { state: retry ? "retry" : "recent", lastAt, nextAt, busy: null };
|
|
64
|
+
}
|
|
65
|
+
const from = forcedFrom(store, last);
|
|
66
|
+
if (from !== null && now.getTime() - from >= SWEEP_FORCE_MS)
|
|
67
|
+
return { state: "forced", lastAt, nextAt: null, busy: null };
|
|
68
|
+
const busy = sweepBusy(store, now);
|
|
69
|
+
return { state: busy === null ? "due" : "busy", lastAt, nextAt: null, busy };
|
|
70
|
+
}
|
|
71
|
+
const BUSY_WORDS = { run: "work is running", claim: "a worker is starting a task", ready: "a task is ready to start" };
|
|
72
|
+
/** Why the sweep is waiting or when it is next due, after "Automatic sweep: ". `when` formats a time for the reader. */
|
|
73
|
+
export function sweepScheduleWords(schedule, when) {
|
|
74
|
+
switch (schedule.state) {
|
|
75
|
+
case "off": return "off.";
|
|
76
|
+
case "sweeping": return "sweeping now.";
|
|
77
|
+
case "no-service": return "waiting: no Toolroll service running.";
|
|
78
|
+
case "recent": return `swept recently, next due ${when(schedule.nextAt)}.`;
|
|
79
|
+
case "retry": return `retry due ${when(schedule.nextAt)}.`;
|
|
80
|
+
case "busy": return `due, waiting: ${BUSY_WORDS[schedule.busy]}.`;
|
|
81
|
+
case "forced": return "over 48 hours since the last sweep; it runs at the next check even if work is running.";
|
|
82
|
+
case "due": return "due; it starts within 15 minutes while nothing is running.";
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
// The last look per flight, so a due sweep is looked at no more than every 15 minutes.
|
|
86
|
+
const looked = new WeakMap();
|
|
87
|
+
/**
|
|
88
|
+
* The service's claim and launch. A one-shot caller has no `owner` and never claims. A hung child never holds
|
|
89
|
+
* reconciliation or the caller's store open; the result is saved by the flight's own connection.
|
|
90
|
+
*/
|
|
91
|
+
export function idleStorageSweep(store, deps, now, supervisor = storageSweepSupervisor(deps.databaseFile)) {
|
|
92
|
+
if (deps.owner === undefined || storageSweepOff() || supervisor.running)
|
|
93
|
+
return null;
|
|
94
|
+
const at = now();
|
|
95
|
+
// Older installations have no creation fact. Anchor once on their first service pass; never backdate a claim.
|
|
96
|
+
if (store.installationFact("installed-at") === null)
|
|
97
|
+
store.recordInstallationFact("installed-at", at.toISOString(), at);
|
|
98
|
+
if (at.getTime() < dueAt(store))
|
|
99
|
+
return null;
|
|
100
|
+
const seen = looked.get(supervisor);
|
|
101
|
+
if (seen !== undefined && at.getTime() - seen < SWEEP_CHECK_MS && at.getTime() >= seen)
|
|
102
|
+
return null;
|
|
103
|
+
looked.set(supervisor, at.getTime());
|
|
104
|
+
const ready = () => {
|
|
105
|
+
if (at.getTime() < dueAt(store))
|
|
106
|
+
return false;
|
|
107
|
+
const from = forcedFrom(store, lastSweep(store, "automatic"));
|
|
108
|
+
return (from !== null && at.getTime() - from >= SWEEP_FORCE_MS) || sweepBusy(store, at) === null;
|
|
109
|
+
};
|
|
110
|
+
if (!ready())
|
|
111
|
+
return null;
|
|
112
|
+
// This handle belongs to the flight, not to the pass that has already returned by the time it finishes.
|
|
113
|
+
const connection = openStoreNoMigrate(deps.databaseFile);
|
|
114
|
+
if (!connection.ok)
|
|
115
|
+
throw new Error("Sweep result store unavailable");
|
|
116
|
+
const results = connection.store;
|
|
117
|
+
let completed = null;
|
|
118
|
+
let owned = false;
|
|
119
|
+
try {
|
|
120
|
+
// Asked again where the cursor moves: work that started since the first look still defers the sweep.
|
|
121
|
+
const mine = store.transact(() => {
|
|
122
|
+
if (!ready())
|
|
123
|
+
return false;
|
|
124
|
+
store.setServiceCursor(SWEEP_CURSOR, at.getTime(), at);
|
|
125
|
+
return true;
|
|
126
|
+
});
|
|
127
|
+
if (!mine)
|
|
128
|
+
return null;
|
|
129
|
+
const request = { version: 1, databaseFile: deps.databaseFile, poolRoot: deps.pool?.root ?? null, tempRoots: [...deps.tempRoots], actor: deps.actor ?? "worker", at: at.toISOString() };
|
|
130
|
+
owned = supervisor.start(request, result => {
|
|
131
|
+
const record = result.ok ? result.record : sweepFailure(request.at, request.actor, deps.pool !== null, result.reason);
|
|
132
|
+
try {
|
|
133
|
+
results.transact(() => {
|
|
134
|
+
if (!results.schemaCurrent())
|
|
135
|
+
throw new Error("Sweep result store changed");
|
|
136
|
+
saveSweep(results, record);
|
|
137
|
+
// Only this claim can reschedule itself. A later claim must never be overwritten by a late callback.
|
|
138
|
+
if (record.parts.some(one => one.failed > 0 || one.more === true || one.problem !== undefined) && results.serviceCursor(SWEEP_CURSOR) === at.getTime()) {
|
|
139
|
+
results.setServiceCursor(SWEEP_CURSOR, Math.max(1, now().getTime() - SWEEP_EVERY_MS + RETRY_MS), now());
|
|
140
|
+
}
|
|
141
|
+
});
|
|
142
|
+
completed = record;
|
|
143
|
+
}
|
|
144
|
+
catch {
|
|
145
|
+
// No exception/path from a process or database is safe to copy into a log.
|
|
146
|
+
console.error("Could not save the storage sweep result.");
|
|
147
|
+
}
|
|
148
|
+
finally {
|
|
149
|
+
results.close();
|
|
150
|
+
}
|
|
151
|
+
}, deps.owner);
|
|
152
|
+
if (!owned) {
|
|
153
|
+
store.transact(() => {
|
|
154
|
+
if (store.serviceCursor(SWEEP_CURSOR) === at.getTime())
|
|
155
|
+
store.setServiceCursor(SWEEP_CURSOR, Math.max(1, at.getTime() - SWEEP_EVERY_MS + RETRY_MS), now());
|
|
156
|
+
});
|
|
157
|
+
}
|
|
158
|
+
return completed;
|
|
159
|
+
}
|
|
160
|
+
finally {
|
|
161
|
+
if (!owned)
|
|
162
|
+
results.close();
|
|
163
|
+
}
|
|
164
|
+
}
|
package/dist/storage-sweep.d.ts
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
import { type Located } from "./orphan-sweep.js";
|
|
2
2
|
import { type ProcessRow } from "./run-closeout.js";
|
|
3
3
|
import type { Store } from "./store.js";
|
|
4
|
-
import type
|
|
4
|
+
import { type WorktreePool } from "./worktree.js";
|
|
5
|
+
import { type SweepCompletion, type SweepExecutor } from "./storage-sweep-process.js";
|
|
6
|
+
export declare const SWEEP_CURSOR = "storage-sweep";
|
|
5
7
|
export declare const SWEEP_EVERY_MS = 86400000;
|
|
6
8
|
/** A sweep that failed somewhere, or left work for later, runs again this soon. */
|
|
7
9
|
export declare const RETRY_MS: number;
|
|
@@ -26,17 +28,40 @@ export type SweepDeps = {
|
|
|
26
28
|
pool: WorktreePool | null;
|
|
27
29
|
tempRoots: readonly string[];
|
|
28
30
|
actor?: string;
|
|
31
|
+
/** Parent lifecycle only; never sent to the child. */
|
|
32
|
+
owner?: object;
|
|
29
33
|
census?: () => ProcessRow[];
|
|
30
34
|
locate?: (pids: readonly number[]) => Map<number, Located>;
|
|
31
35
|
inUse?: () => string[];
|
|
36
|
+
scripts?: (pids: readonly number[]) => string[];
|
|
32
37
|
graceMs?: number;
|
|
33
38
|
};
|
|
34
39
|
export declare function storageSweepOff(env?: NodeJS.ProcessEnv): boolean;
|
|
35
|
-
export declare function
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
/**
|
|
39
|
-
|
|
40
|
+
export declare function sweepFailure(at: string, actor: string, withPool: boolean, reason: Exclude<SweepCompletion, {
|
|
41
|
+
ok: true;
|
|
42
|
+
}>["reason"]): SweepRecord;
|
|
43
|
+
/**
|
|
44
|
+
* What live work uses now. Live work: an open run, a current (unexpired, unreleased) lease, or a checkout leased, held
|
|
45
|
+
* by a process or whose task has either. It uses its checkouts, the shared install each links, and every folder its
|
|
46
|
+
* open runs' processes (with their groups and what they started) run from. `where` says where the given processes run
|
|
47
|
+
* from, or null when that can't be read: then `blind`, and everything stays. With nothing live, nothing is read from the OS.
|
|
48
|
+
*/
|
|
49
|
+
export type LiveWork = {
|
|
50
|
+
live: boolean;
|
|
51
|
+
paths: string[];
|
|
52
|
+
blind: boolean;
|
|
53
|
+
};
|
|
54
|
+
export declare function liveWork(store: Store, depsRoot: string, now: Date, where: (pids: readonly number[]) => string[] | null): LiveWork;
|
|
55
|
+
/**
|
|
56
|
+
* Whether live work uses `path`, asked again at every call, including process children. A cached process look
|
|
57
|
+
* cannot protect a child launched between the pre-rename and post-rename guards. All this work is in the child.
|
|
58
|
+
*/
|
|
59
|
+
export declare function liveUse(store: Store, deps: SweepDeps, now: () => Date): (path: string) => boolean;
|
|
60
|
+
/** Parent side: no filesystem/process scans here. The executor seam is for deterministic unit tests only. */
|
|
61
|
+
export declare function sweepStorage(store: Store, deps: SweepDeps, now: () => Date, execute?: SweepExecutor): Promise<SweepRecord>;
|
|
62
|
+
export declare function sweepKinds(withPool: boolean): SweepKind[];
|
|
63
|
+
/** Child side (also a focused unit-test seam): every scan and removal, without saving the parent's result. */
|
|
64
|
+
export declare function executeSweep(store: Store, deps: SweepDeps, now: () => Date): Promise<SweepRecord>;
|
|
40
65
|
/** Save what a sweep or a clean-up by hand did (the latest few are kept), with one ledger entry saying it. */
|
|
41
66
|
export declare function saveSweep(store: Store, record: SweepRecord): SweepRecord;
|
|
42
67
|
/** The latest sweep (of `source`, or either), or null when there has been none. */
|