toolroll 0.9.30 → 0.9.32

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. package/CHANGELOG.md +35 -0
  2. package/dist/assignment.d.ts +3 -0
  3. package/dist/assignment.js +8 -2
  4. package/dist/batch-checks.d.ts +15 -0
  5. package/dist/batch-checks.js +244 -0
  6. package/dist/batch-policy.d.ts +23 -0
  7. package/dist/batch-policy.js +25 -0
  8. package/dist/browser/workspace.js +13 -13
  9. package/dist/builder.js +14 -3
  10. package/dist/chat-actions.js +2 -2
  11. package/dist/check-levels-ui.d.ts +4 -0
  12. package/dist/check-levels-ui.js +21 -2
  13. package/dist/checkout-cleanup.d.ts +2 -1
  14. package/dist/checkout-cleanup.js +24 -8
  15. package/dist/desktop-update.d.ts +1 -1
  16. package/dist/exec.d.ts +2 -0
  17. package/dist/exec.js +11 -0
  18. package/dist/flow-gallery-ui.d.ts +14 -9
  19. package/dist/flow-gallery-ui.js +71 -29
  20. package/dist/flow-gallery.d.ts +26 -2
  21. package/dist/flow-gallery.js +182 -7
  22. package/dist/lead-capabilities.d.ts +0 -7
  23. package/dist/lead-capabilities.js +11 -1
  24. package/dist/mate-doors.js +2 -2
  25. package/dist/mate.d.ts +6 -2
  26. package/dist/mate.js +63 -38
  27. package/dist/mcp-connect.d.ts +3 -1
  28. package/dist/mcp-connect.js +11 -6
  29. package/dist/never-index.d.ts +3 -0
  30. package/dist/never-index.js +21 -0
  31. package/dist/operate.js +86 -2
  32. package/dist/orphan-sweep.d.ts +63 -0
  33. package/dist/orphan-sweep.js +235 -0
  34. package/dist/result-follow-ups.d.ts +22 -2
  35. package/dist/result-follow-ups.js +44 -10
  36. package/dist/serve.js +52 -19
  37. package/dist/shared-deps.d.ts +18 -2
  38. package/dist/shared-deps.js +129 -15
  39. package/dist/staged-runtimes.d.ts +42 -0
  40. package/dist/staged-runtimes.js +191 -0
  41. package/dist/storage-sweep.d.ts +50 -0
  42. package/dist/storage-sweep.js +203 -0
  43. package/dist/storage-ui.d.ts +5 -0
  44. package/dist/storage-ui.js +30 -4
  45. package/dist/storage.d.ts +2 -0
  46. package/dist/storage.js +15 -3
  47. package/dist/store.d.ts +17 -2
  48. package/dist/store.js +37 -5
  49. package/dist/subscription-chat.d.ts +3 -1
  50. package/dist/subscription-chat.js +1 -0
  51. package/dist/surface.js +1 -0
  52. package/dist/task-status.d.ts +13 -0
  53. package/dist/task-status.js +37 -3
  54. package/dist/team-leads.js +6 -2
  55. package/dist/team-runtime.js +2 -1
  56. package/dist/test-temp.d.ts +20 -2
  57. package/dist/test-temp.js +53 -13
  58. package/dist/toolroll-update.js +2 -0
  59. package/dist/workspace-ui.js +3 -1
  60. package/dist/worktree.js +9 -0
  61. package/package.json +1 -1
@@ -0,0 +1,42 @@
1
+ /** A deploy or update that never finished keeps its stage this long (it may be resumed). */
2
+ export declare const UNFINISHED_KEEP_MS: number;
3
+ export declare const STAGED_FOLDER = "staged-upgrades";
4
+ export type Stage = {
5
+ path: string;
6
+ name: string;
7
+ at: number;
8
+ finished: boolean;
9
+ prior: string | null;
10
+ keptAside: boolean;
11
+ };
12
+ export type StageKeep = "current" | "previous" | "newest" | "under way" | "kept-aside database";
13
+ export type StagePlan = {
14
+ keep: {
15
+ path: string;
16
+ why: StageKeep;
17
+ }[];
18
+ go: Stage[];
19
+ refused: string | null;
20
+ };
21
+ /** The `browser-*` and `release-*` stages: real folders directly in staged-upgrades, newest first. */
22
+ export declare function stagedRuntimes(stateDir: string): Stage[];
23
+ /**
24
+ * Paths a runtime may be running from: the launchd services' definitions, every `toolroll` (or older
25
+ * `standing-orders`) command on PATH or under nvm, this process, and `extra`.
26
+ */
27
+ export declare function runtimesInUse(extra?: readonly string[], home?: string, path?: string): string[];
28
+ /** What stays and what goes now, and why. */
29
+ export declare function stagePlan(stateDir: string, inUse: readonly string[], now: Date): StagePlan;
30
+ export type StagePrune = {
31
+ removed: string[];
32
+ failed: string[];
33
+ refused: string | null;
34
+ };
35
+ /**
36
+ * Remove the stages that go. Right before each goes, the plan is made again (what's in use now) and the stage must
37
+ * still be a real folder directly in staged-upgrades.
38
+ */
39
+ export declare function pruneStagedRuntimes(stateDir: string, now: Date, options?: {
40
+ inUse?: () => string[];
41
+ remove?: (path: string) => void;
42
+ }): StagePrune;
@@ -0,0 +1,191 @@
1
+ /**
2
+ * Staged runtimes (`staged-upgrades/`): each deploy (scripts/deploy-browser.mjs, `browser-*`) and each `toolroll
3
+ * update` (`release-*`) stages a whole runtime, a few hundred MB with its database backup. Oct 4: 9 of them, 2.2 GB.
4
+ *
5
+ * Kept: the current runtime (one the service, a `toolroll` command or this process runs from), the one before it
6
+ * (its journal's way back), the newest stage, a deploy or update still under way (no finished journal) for a week,
7
+ * and a stage holding a database a failed update kept aside. Everything else of those two kinds goes — after a deploy
8
+ * and in the plane's daily storage sweep. `rollback-*` records and anything else in the folder are never touched.
9
+ * With no current runtime found, nothing goes: which one is safe to remove is then unknown.
10
+ */
11
+ import { lstatSync, readFileSync, readdirSync, realpathSync, rmSync, statSync } from "node:fs";
12
+ import { homedir } from "node:os";
13
+ import { basename, dirname, join, resolve, sep } from "node:path";
14
+ const DAY_MS = 86_400_000;
15
+ /** A deploy or update that never finished keeps its stage this long (it may be resumed). */
16
+ export const UNFINISHED_KEEP_MS = 7 * DAY_MS;
17
+ export const STAGED_FOLDER = "staged-upgrades";
18
+ /** The canonical path, even of something that doesn't exist (yet): its nearest existing folder, resolved, plus the rest. */
19
+ function real(path) {
20
+ let at = resolve(path);
21
+ const rest = [];
22
+ for (let hops = 0; hops < 256; hops++) {
23
+ try {
24
+ return join(realpathSync(at), ...rest.reverse());
25
+ }
26
+ catch { /* not there */ }
27
+ const up = dirname(at);
28
+ if (up === at)
29
+ break;
30
+ rest.push(basename(at));
31
+ at = up;
32
+ }
33
+ return resolve(path);
34
+ }
35
+ const readJson = (file) => {
36
+ try {
37
+ const parsed = JSON.parse(readFileSync(file, "utf8"));
38
+ return parsed !== null && typeof parsed === "object" ? parsed : null;
39
+ }
40
+ catch {
41
+ return null;
42
+ }
43
+ };
44
+ /** The `browser-*` and `release-*` stages: real folders directly in staged-upgrades, newest first. */
45
+ export function stagedRuntimes(stateDir) {
46
+ const root = real(join(stateDir, STAGED_FOLDER));
47
+ let names = [];
48
+ try {
49
+ names = readdirSync(root);
50
+ }
51
+ catch {
52
+ return [];
53
+ }
54
+ const stages = [];
55
+ for (const name of names) {
56
+ if (!/^(browser|release)-/.test(name))
57
+ continue;
58
+ const path = join(root, name);
59
+ try {
60
+ if (!lstatSync(path).isDirectory())
61
+ continue;
62
+ }
63
+ catch {
64
+ continue;
65
+ }
66
+ const deploy = name.startsWith("browser-") ? readJson(join(path, "deployment.json")) : null;
67
+ const update = name.startsWith("release-") ? readJson(join(path, "update.json")) : null;
68
+ const journal = deploy ?? update;
69
+ const stamp = String(journal?.["deployedAt"] ?? journal?.["startedAt"] ?? journal?.["createdAt"] ?? "");
70
+ let at = Date.parse(stamp);
71
+ if (!Number.isFinite(at)) {
72
+ try {
73
+ at = statSync(path).mtimeMs;
74
+ }
75
+ catch {
76
+ at = 0;
77
+ }
78
+ }
79
+ const finished = deploy !== null ? deploy["phase"] === "deployed" : update !== null ? typeof update["finishedAt"] === "string" : false;
80
+ const from = update?.["from"];
81
+ const prior = typeof deploy?.["priorRuntime"] === "string" ? deploy["priorRuntime"] : typeof from?.dist === "string" ? from.dist : null;
82
+ let keptAside = false;
83
+ try {
84
+ keptAside = readdirSync(path).some(one => /^orders\.(?:kept|unreadable)\./.test(one));
85
+ }
86
+ catch {
87
+ keptAside = true;
88
+ }
89
+ stages.push({ path, name, at, finished, prior, keptAside });
90
+ }
91
+ return stages.sort((a, b) => b.at - a.at);
92
+ }
93
+ /**
94
+ * Paths a runtime may be running from: the launchd services' definitions, every `toolroll` (or older
95
+ * `standing-orders`) command on PATH or under nvm, this process, and `extra`.
96
+ */
97
+ export function runtimesInUse(extra = [], home = homedir(), path = process.env["PATH"] ?? "") {
98
+ const found = [...extra];
99
+ const agents = join(home, "Library", "LaunchAgents");
100
+ try {
101
+ for (const name of readdirSync(agents).filter(one => one.startsWith("com.toolroll.") || one.startsWith("com.standing-orders."))) {
102
+ for (const match of readFileSync(join(agents, name), "utf8").matchAll(/<string>([^<]*staged-upgrades[^<]*)<\/string>/g))
103
+ found.push(match[1]);
104
+ }
105
+ }
106
+ catch { /* no launch agents here */ }
107
+ const commands = path.split(":").filter(Boolean).flatMap(dir => ["toolroll", "standing-orders"].map(bin => join(dir, bin)));
108
+ try {
109
+ for (const version of readdirSync(join(home, ".nvm", "versions", "node")))
110
+ for (const bin of ["toolroll", "standing-orders"])
111
+ commands.push(join(home, ".nvm", "versions", "node", version, "bin", bin));
112
+ }
113
+ catch { /* no nvm */ }
114
+ for (const command of commands) {
115
+ try {
116
+ lstatSync(command);
117
+ found.push(command);
118
+ }
119
+ catch { /* not there */ }
120
+ }
121
+ if (process.argv[1] !== undefined)
122
+ found.push(process.argv[1]);
123
+ return found;
124
+ }
125
+ /** The stage `path` lies in, or null. */
126
+ function stageOf(stages, path) {
127
+ if (path === null || path === undefined || path === "")
128
+ return null;
129
+ for (const at of new Set([resolve(path), real(path)])) {
130
+ for (const stage of stages)
131
+ for (const base of new Set([stage.path, real(stage.path)])) {
132
+ if (at === base || at.startsWith(base + sep))
133
+ return stage;
134
+ }
135
+ }
136
+ return null;
137
+ }
138
+ /** What stays and what goes now, and why. */
139
+ export function stagePlan(stateDir, inUse, now) {
140
+ const stages = stagedRuntimes(stateDir);
141
+ const keep = new Map();
142
+ const current = inUse.map(path => stageOf(stages, path)).filter((one) => one !== null);
143
+ if (stages.length > 0 && current.length === 0)
144
+ return { keep: [], go: [], refused: "no staged runtime is in use, so which one runs now is unknown" };
145
+ for (const one of current)
146
+ keep.set(one.path, "current");
147
+ for (const one of current) {
148
+ const prior = stageOf(stages, one.prior) ?? stages.find(other => other.finished && other.path !== one.path && other.at < one.at) ?? null;
149
+ if (prior !== null && !keep.has(prior.path))
150
+ keep.set(prior.path, "previous");
151
+ }
152
+ if (stages[0] !== undefined && !keep.has(stages[0].path))
153
+ keep.set(stages[0].path, "newest");
154
+ for (const one of stages) {
155
+ if (keep.has(one.path))
156
+ continue;
157
+ if (one.keptAside)
158
+ keep.set(one.path, "kept-aside database");
159
+ else if (!one.finished && now.getTime() - one.at < UNFINISHED_KEEP_MS)
160
+ keep.set(one.path, "under way");
161
+ }
162
+ return { keep: [...keep].map(([path, why]) => ({ path, why })), go: stages.filter(one => !keep.has(one.path)), refused: null };
163
+ }
164
+ /**
165
+ * Remove the stages that go. Right before each goes, the plan is made again (what's in use now) and the stage must
166
+ * still be a real folder directly in staged-upgrades.
167
+ */
168
+ export function pruneStagedRuntimes(stateDir, now, options = {}) {
169
+ const inUse = options.inUse ?? (() => runtimesInUse());
170
+ const first = stagePlan(stateDir, inUse(), now);
171
+ if (first.refused !== null)
172
+ return { removed: [], failed: [], refused: first.refused };
173
+ const root = real(join(stateDir, STAGED_FOLDER));
174
+ const removed = [];
175
+ const failed = [];
176
+ for (const one of first.go) {
177
+ const again = stagePlan(stateDir, inUse(), now);
178
+ if (again.refused !== null || !again.go.some(other => other.path === one.path))
179
+ continue;
180
+ try {
181
+ if (lstatSync(one.path).isSymbolicLink() || dirname(one.path) !== root)
182
+ continue;
183
+ (options.remove ?? (path => rmSync(path, { recursive: true, force: true, maxRetries: 3 })))(one.path);
184
+ removed.push(one.path);
185
+ }
186
+ catch {
187
+ failed.push(one.path);
188
+ }
189
+ }
190
+ return { removed, failed, refused: null };
191
+ }
@@ -0,0 +1,50 @@
1
+ import { type Located } from "./orphan-sweep.js";
2
+ import { type ProcessRow } from "./run-closeout.js";
3
+ import type { Store } from "./store.js";
4
+ import type { WorktreePool } from "./worktree.js";
5
+ export declare const SWEEP_EVERY_MS = 86400000;
6
+ /** A sweep that failed somewhere, or left work for later, runs again this soon. */
7
+ export declare const RETRY_MS: number;
8
+ export type SweepKind = "test temp" | "orphans" | "checkouts" | "runtimes" | "dependencies";
9
+ export type SweepPart = {
10
+ kind: SweepKind;
11
+ count: number;
12
+ bytes: number;
13
+ failed: number;
14
+ more?: boolean;
15
+ problem?: string;
16
+ items: string[];
17
+ };
18
+ export type SweepRecord = {
19
+ at: string;
20
+ source: "automatic" | "manual";
21
+ actor: string;
22
+ parts: SweepPart[];
23
+ };
24
+ export type SweepDeps = {
25
+ databaseFile: string;
26
+ pool: WorktreePool | null;
27
+ tempRoots: readonly string[];
28
+ actor?: string;
29
+ census?: () => ProcessRow[];
30
+ locate?: (pids: readonly number[]) => Map<number, Located>;
31
+ inUse?: () => string[];
32
+ graceMs?: number;
33
+ };
34
+ export declare function storageSweepOff(env?: NodeJS.ProcessEnv): boolean;
35
+ export declare function storageSweepDue(store: Store, now: Date): boolean;
36
+ /** The daily sweep, when it's due and on: one worker takes it. Null when it wasn't this pass's to run. */
37
+ export declare function dailyStorageSweep(store: Store, deps: SweepDeps, now: () => Date): Promise<SweepRecord | null>;
38
+ /** One sweep, now, every part; saved and in the ledger. Each part's failure is its own: the others still run. */
39
+ export declare function sweepStorage(store: Store, deps: SweepDeps, now: () => Date): Promise<SweepRecord>;
40
+ /** Save what a sweep or a clean-up by hand did (the latest few are kept), with one ledger entry saying it. */
41
+ export declare function saveSweep(store: Store, record: SweepRecord): SweepRecord;
42
+ /** The latest sweep (of `source`, or either), or null when there has been none. */
43
+ export declare function lastSweep(store: Store, source?: SweepRecord["source"]): SweepRecord | null;
44
+ /** When the next automatic sweep is due, or null when it is off. */
45
+ export declare function nextSweepAt(store: Store): string | null;
46
+ export declare function partWords(one: Pick<SweepPart, "kind" | "count">): string;
47
+ /** "Removed 120 test temp folders and 2 finished checkouts, about 1.2 GB. Stopped 1 leftover process. 3 couldn't be removed." */
48
+ export declare function sweepWords(parts: readonly SweepPart[]): string;
49
+ /** The details a disclosure shows: each part's paths, problems and what's left for later, one line each. */
50
+ export declare function sweepDetails(parts: readonly SweepPart[]): string[];
@@ -0,0 +1,203 @@
1
+ /**
2
+ * The daily storage sweep: the plane cleans up after itself without anyone running a command. Once a day (one worker
3
+ * takes the day: the cursor moves in the same transaction) it removes test temp folders nothing touched for a day
4
+ * (test-temp.ts), stops orphans Toolroll left running (orphan-sweep.ts), removes finished checkouts the cleanup
5
+ * setting lets go (checkout-cleanup.ts), keeps the current, previous and newest staged runtimes (staged-runtimes.ts)
6
+ * and removes shared dependency installs no checkout has used for a week (shared-deps.ts).
7
+ *
8
+ * The everyday work stays where it happens: suites remove their own temp roots and processes, a run's processes stop
9
+ * when it ends and a finished checkout goes on the worker's next pass. This is the net for what slipped through.
10
+ *
11
+ * What each sweep did is saved (the latest few) and in the ledger: `toolroll storage` and Settings → Storage show it.
12
+ * A clean-up by hand (`storage clean --yes`) is saved the same way. A part that failed, or had more than one pass
13
+ * takes, brings the next sweep forward to an hour later. `TOOLROLL_STORAGE_SWEEP=off` turns the automatic sweep off
14
+ * (the unit tests do, so a test never sweeps the machine it runs on).
15
+ */
16
+ import { dirname } from "node:path";
17
+ import { cleanCheckouts } from "./checkout-cleanup.js";
18
+ import { durationWords, sweepOrphans, ownerRunning } from "./orphan-sweep.js";
19
+ import { processCensus } from "./run-closeout.js";
20
+ import { pruneUnusedShared, sharedDepsRoot, sharedUse, SHARED_UNUSED_MS } from "./shared-deps.js";
21
+ import { pruneStagedRuntimes, runtimesInUse, stagePlan } from "./staged-runtimes.js";
22
+ import { bytesWords, diskBytes } from "./storage.js";
23
+ import { removeStaleTestTemp } from "./test-temp.js";
24
+ const CURSOR = "storage-sweep";
25
+ export const SWEEP_EVERY_MS = 86_400_000;
26
+ /** A sweep that failed somewhere, or left work for later, runs again this soon. */
27
+ export const RETRY_MS = 60 * 60_000;
28
+ /** Sweeps kept. */
29
+ const KEEP = 30;
30
+ /** Test temp folders one sweep removes at most (Oct 4 found 107,418). */
31
+ const TEMP_MAX = 20_000;
32
+ /** Paths saved per part, for the disclosure. */
33
+ const ITEMS_MAX = 20;
34
+ export function storageSweepOff(env = process.env) {
35
+ return (env["TOOLROLL_STORAGE_SWEEP"] ?? "").trim().toLowerCase() === "off";
36
+ }
37
+ export function storageSweepDue(store, now) {
38
+ return now.getTime() - store.serviceCursor(CURSOR) >= SWEEP_EVERY_MS;
39
+ }
40
+ /** The daily sweep, when it's due and on: one worker takes it. Null when it wasn't this pass's to run. */
41
+ export async function dailyStorageSweep(store, deps, now) {
42
+ if (storageSweepOff())
43
+ return null;
44
+ const at = now();
45
+ const mine = store.transact(() => {
46
+ if (!storageSweepDue(store, at))
47
+ return false;
48
+ store.setServiceCursor(CURSOR, at.getTime(), at);
49
+ return true;
50
+ });
51
+ if (!mine)
52
+ return null;
53
+ const record = await sweepStorage(store, deps, now);
54
+ // Something failed or was left for later: the next sweep comes an hour from now, not tomorrow.
55
+ if (record.parts.some(one => one.failed > 0 || one.more === true || one.problem !== undefined)) {
56
+ store.setServiceCursor(CURSOR, Math.max(0, at.getTime() - SWEEP_EVERY_MS + RETRY_MS), now());
57
+ }
58
+ return record;
59
+ }
60
+ const capped = (items) => items.slice(0, ITEMS_MAX);
61
+ const failure = (error) => error instanceof Error ? error.message : String(error);
62
+ /** One sweep, now, every part; saved and in the ledger. Each part's failure is its own: the others still run. */
63
+ export async function sweepStorage(store, deps, now) {
64
+ const actor = deps.actor ?? "worker";
65
+ const read = deps.census ?? (() => processCensus());
66
+ const parts = [];
67
+ const part = async (kind, work) => {
68
+ try {
69
+ parts.push({ kind, ...await work() });
70
+ }
71
+ catch (error) {
72
+ parts.push({ kind, count: 0, bytes: 0, failed: 0, problem: failure(error), items: [] });
73
+ }
74
+ };
75
+ await part("test temp", async () => {
76
+ // A root whose owner still runs stays, however old it looks; with no process list, every owned root stays.
77
+ let alive = () => true;
78
+ try {
79
+ alive = ownerRunning(read());
80
+ }
81
+ catch { /* owned roots stay */ }
82
+ const done = removeStaleTestTemp(deps.tempRoots, now(), { ownerAlive: alive, max: TEMP_MAX });
83
+ return { count: done.removed.length, bytes: 0, failed: done.failed.length, ...(done.more ? { more: true } : {}), items: capped(done.removed) };
84
+ });
85
+ await part("orphans", async () => {
86
+ const done = await sweepOrphans(store, now, {
87
+ actor, pool: deps.pool, tempRoots: deps.tempRoots, census: read,
88
+ ...(deps.locate === undefined ? {} : { locate: deps.locate }), ...(deps.graceMs === undefined ? {} : { graceMs: deps.graceMs }),
89
+ });
90
+ return {
91
+ count: done.stopped.length, bytes: 0, failed: 0, ...(done.failed.length === 0 ? {} : { problem: done.failed.join("; ") }),
92
+ items: capped(done.stopped.map(one => `${one.name} (${one.pid}), ${durationWords(one.ranMs)}, in ${one.where}`)),
93
+ };
94
+ });
95
+ if (deps.pool !== null) {
96
+ const pool = deps.pool;
97
+ await part("checkouts", async () => {
98
+ const done = await cleanCheckouts(store, pool, now, { manual: false, actor });
99
+ return { count: done.removed.length, bytes: done.freed, failed: done.kept.filter(one => one.why === "git refused").length, items: capped(done.removed.map(one => one.path)) };
100
+ });
101
+ }
102
+ const stateDir = dirname(deps.databaseFile);
103
+ await part("runtimes", async () => {
104
+ const inUse = deps.inUse ?? (() => runtimesInUse());
105
+ const plan = stagePlan(stateDir, inUse(), now());
106
+ const sizes = diskBytes(plan.go.map(one => one.path));
107
+ const done = pruneStagedRuntimes(stateDir, now(), { inUse });
108
+ return {
109
+ count: done.removed.length, bytes: done.removed.reduce((sum, path) => sum + (sizes.get(path) ?? 0), 0), failed: done.failed.length,
110
+ ...(done.refused === null ? {} : { problem: `kept every staged runtime: ${done.refused}` }), items: capped(done.removed),
111
+ };
112
+ });
113
+ await part("dependencies", async () => {
114
+ const root = sharedDepsRoot(deps.databaseFile);
115
+ const checkouts = () => store.listWorktrees().map(row => ({ path: row.path, repo: row.repo }));
116
+ const candidates = sharedUse(root, checkouts()).filter(one => one.checkouts.length === 0 && now().getTime() - Date.parse(one.usedAt) > SHARED_UNUSED_MS);
117
+ const sizes = diskBytes(candidates.map(one => one.dir));
118
+ const done = pruneUnusedShared(root, checkouts, now());
119
+ return { count: done.removed.length, bytes: done.removed.reduce((sum, one) => sum + (sizes.get(one.dir) ?? 0), 0), failed: done.failed.length, items: capped(done.removed.map(one => one.dir)) };
120
+ });
121
+ return saveSweep(store, { at: now().toISOString(), source: "automatic", actor, parts });
122
+ }
123
+ /** Save what a sweep or a clean-up by hand did (the latest few are kept), with one ledger entry saying it. */
124
+ export function saveSweep(store, record) {
125
+ store.transact(() => {
126
+ store.handle.prepare("INSERT INTO storage_sweep (at, source, actor, result) VALUES (?, ?, ?, ?)").run(record.at, record.source, record.actor, JSON.stringify(record.parts));
127
+ store.handle.prepare("DELETE FROM storage_sweep WHERE id NOT IN (SELECT id FROM storage_sweep ORDER BY id DESC LIMIT ?)").run(KEEP);
128
+ });
129
+ const did = record.parts.some(one => one.count > 0);
130
+ const failed = record.parts.some(one => one.failed > 0 || one.problem !== undefined);
131
+ store.recordAction({ at: record.at, actor: record.actor, repo: null, taskId: null, runId: null, action: "storage sweep",
132
+ outcome: failed ? "partial" : did ? "removed" : "nothing due", source: record.source === "automatic" ? "policy" : "request", detail: sweepWords(record.parts) });
133
+ return record;
134
+ }
135
+ function parseParts(text) {
136
+ try {
137
+ const parsed = JSON.parse(text);
138
+ return Array.isArray(parsed) ? parsed.filter((one) => typeof one === "object" && one !== null && typeof one.kind === "string") : [];
139
+ }
140
+ catch {
141
+ return [];
142
+ }
143
+ }
144
+ /** The latest sweep (of `source`, or either), or null when there has been none. */
145
+ export function lastSweep(store, source) {
146
+ const row = store.handle.prepare(`SELECT at, source, actor, result FROM storage_sweep${source === undefined ? "" : " WHERE source = ?"} ORDER BY id DESC LIMIT 1`)
147
+ .get(...(source === undefined ? [] : [source]));
148
+ if (row === undefined)
149
+ return null;
150
+ return { at: String(row["at"]), source: row["source"] === "manual" ? "manual" : "automatic", actor: String(row["actor"]), parts: parseParts(String(row["result"])) };
151
+ }
152
+ /** When the next automatic sweep is due, or null when it is off. */
153
+ export function nextSweepAt(store) {
154
+ if (storageSweepOff())
155
+ return null;
156
+ const last = store.serviceCursor(CURSOR);
157
+ return last === 0 ? null : new Date(last + SWEEP_EVERY_MS).toISOString();
158
+ }
159
+ const NOUNS = {
160
+ "test temp": ["test temp folder", "test temp folders"],
161
+ orphans: ["leftover process", "leftover processes"],
162
+ checkouts: ["finished checkout", "finished checkouts"],
163
+ runtimes: ["staged runtime", "staged runtimes"],
164
+ dependencies: ["unused dependency install", "unused dependency installs"],
165
+ };
166
+ /** What a part looks at, for "Couldn't check …". */
167
+ const CHECKED = { "test temp": "test temp folders", orphans: "processes", checkouts: "checkouts", runtimes: "staged runtimes", dependencies: "dependency installs" };
168
+ export function partWords(one) {
169
+ return `${one.count.toLocaleString("en-US")} ${NOUNS[one.kind][one.count === 1 ? 0 : 1]}`;
170
+ }
171
+ function list(items) {
172
+ return items.length <= 1 ? items.join("") : `${items.slice(0, -1).join(", ")} and ${items.at(-1)}`;
173
+ }
174
+ /** "Removed 120 test temp folders and 2 finished checkouts, about 1.2 GB. Stopped 1 leftover process. 3 couldn't be removed." */
175
+ export function sweepWords(parts) {
176
+ const removed = parts.filter(one => one.kind !== "orphans" && one.count > 0);
177
+ const stopped = parts.find(one => one.kind === "orphans" && one.count > 0);
178
+ const bytes = removed.reduce((sum, one) => sum + one.bytes, 0);
179
+ const failed = parts.reduce((sum, one) => sum + one.failed, 0);
180
+ const problems = parts.filter(one => one.problem !== undefined).map(one => one.kind);
181
+ const said = [
182
+ ...(removed.length === 0 ? [] : [`Removed ${list(removed.map(partWords))}${bytes > 0 ? `, about ${bytesWords(bytes)}` : ""}.`]),
183
+ ...(stopped === undefined ? [] : [`Stopped ${partWords(stopped)}.`]),
184
+ ...(failed === 0 ? [] : [`${failed} couldn't be removed.`]),
185
+ ...(problems.length === 0 ? [] : [`Couldn't check ${list(problems.map(kind => CHECKED[kind]))}.`]),
186
+ ];
187
+ return said.length === 0 ? "Nothing was due." : said.join(" ");
188
+ }
189
+ /** The details a disclosure shows: each part's paths, problems and what's left for later, one line each. */
190
+ export function sweepDetails(parts) {
191
+ const lines = [];
192
+ for (const one of parts) {
193
+ if (one.problem !== undefined)
194
+ lines.push(`${NOUNS[one.kind][1][0].toUpperCase()}${NOUNS[one.kind][1].slice(1)}: ${one.problem}`);
195
+ if (one.more === true)
196
+ lines.push(`More ${NOUNS[one.kind][1]} are left for the next sweep.`);
197
+ for (const item of one.items)
198
+ lines.push(item);
199
+ if (one.items.length > 0 && one.count > one.items.length)
200
+ lines.push(`… and ${(one.count - one.items.length).toLocaleString("en-US")} more ${NOUNS[one.kind][1]}`);
201
+ }
202
+ return lines;
203
+ }
@@ -1,8 +1,13 @@
1
1
  import { type CheckoutPlan } from "./checkout-cleanup.js";
2
+ import { type SweepRecord } from "./storage-sweep.js";
2
3
  export declare const STORAGE_CSS: string;
3
4
  export type StorageView = {
4
5
  plan: CheckoutPlan;
5
6
  csrf: string;
7
+ sweep?: {
8
+ last: SweepRecord | null;
9
+ off: boolean;
10
+ };
6
11
  };
7
12
  export declare function storageHtml(view: StorageView, notice: {
8
13
  said?: string | null;
@@ -1,12 +1,14 @@
1
1
  /**
2
2
  * Settings → Storage: how much space task checkouts use, what a clean-up would free now, a Clean up button whose
3
- * preview (what goes, what stays and why, with paths) sits behind the password, and when a finished task's clean
4
- * checkout goes by itself. An instance operator's page.
3
+ * preview (what goes, what stays and why, with paths) sits behind the password, what the last storage sweep removed
4
+ * and when (its paths in a disclosure), and when a finished task's clean checkout goes by itself. An instance
5
+ * operator's page.
5
6
  */
6
7
  import { homedir } from "node:os";
7
8
  import { basename } from "node:path";
8
9
  import { previewDigest, whyWords } from "./checkout-cleanup.js";
9
10
  import { CLEANUP_CHOICES, bytesWords } from "./storage.js";
11
+ import { sweepDetails, sweepWords } from "./storage-sweep.js";
10
12
  const e = (value) => String(value ?? "").replace(/[&<>"']/g, c => ({ "&": "&amp;", "<": "&lt;", ">": "&gt;", '"': "&quot;", "'": "&#39;" })[c]);
11
13
  const plural = (count, one, many = `${one}s`) => `${count} ${count === 1 ? one : many}`;
12
14
  export const STORAGE_CSS = `.storage{max-width:720px;min-width:0;overflow-wrap:anywhere}.storage h2{font-size:1.0625rem;margin:24px 0 4px}` +
@@ -25,7 +27,31 @@ export const STORAGE_CSS = `.storage{max-width:720px;min-width:0;overflow-wrap:a
25
27
  `.storage form{display:grid;gap:10px;margin:0}.storage label{display:grid;gap:4px;font-size:.8125rem;font-weight:500;max-width:20rem}` +
26
28
  `.storage select,.storage input[type=password]{width:100%;box-sizing:border-box;min-height:2.125rem}.storage form button{justify-self:start;white-space:nowrap}` +
27
29
  `.storage form.setting button[type=submit],.storage .preview form.remove button[type=submit]{background:var(--primary);color:var(--primary-foreground);border-color:var(--primary);font-weight:600}` +
28
- `@media (max-width:900px){.storage details.clean-up>summary,.storage form button{min-height:44px}.storage label{max-width:none}.storage select,.storage input[type=password]{min-height:44px;font-size:16px}.storage .preview form.remove button{width:100%}}`;
30
+ `.storage .sweep{display:grid;gap:4px}.storage .sweep p{margin:0}.storage .sweep .problem{color:var(--destructive)}` +
31
+ `.storage details.sweep-details>summary{cursor:pointer;min-height:2.25rem;display:inline-flex;align-items:center;font-size:.8125rem;color:var(--muted-foreground)}` +
32
+ `.storage details.sweep-details ul{margin:4px 0 0;padding-left:18px;font-family:var(--font-mono, ui-monospace, monospace);font-size:.6875rem;color:var(--muted-foreground);overflow-wrap:anywhere}` +
33
+ `@media (max-width:900px){.storage details.sweep-details>summary{min-height:44px}.storage details.clean-up>summary,.storage form button{min-height:44px}.storage label{max-width:none}.storage select,.storage input[type=password]{min-height:44px;font-size:16px}.storage .preview form.remove button{width:100%}}`;
34
+ /** Paths under the home folder as ~/…, as the checkout list shows them. */
35
+ function homeAsTilde(line) {
36
+ const home = homedir();
37
+ return home === "" || home === "/" ? line : line.split(`${home}/`).join("~/");
38
+ }
39
+ /** "Oct 4, 3:12 AM" in this computer's time. */
40
+ function whenWords(at) {
41
+ const date = new Date(at);
42
+ return Number.isNaN(date.getTime()) ? at : date.toLocaleString("en-US", { month: "short", day: "numeric", hour: "numeric", minute: "2-digit" });
43
+ }
44
+ /** One line: when the last sweep ran and what it did; its paths and problems behind Details. */
45
+ function sweepHtml(sweep) {
46
+ const last = sweep.last;
47
+ if (last === null)
48
+ return `<p class="meta" data-last-sweep="">${sweep.off ? "Automatic sweep is off." : "Toolroll sweeps leftovers once a day. It hasn't run yet."}</p>`;
49
+ const who = last.source === "manual" ? `Cleaned up by ${e(last.actor)}` : "Swept";
50
+ const details = sweepDetails(last.parts);
51
+ const problem = last.parts.some(one => one.failed > 0 || one.problem !== undefined);
52
+ return `<div class="sweep" data-last-sweep="${e(last.at)}"><p${problem ? ' class="problem"' : ""}>${who} <time datetime="${e(last.at)}">${e(whenWords(last.at))}</time>. ${e(sweepWords(last.parts))}</p>` +
53
+ (details.length === 0 ? "" : `<details class="sweep-details"><summary>Details</summary><ul>${details.map(line => `<li>${e(homeAsTilde(line))}</li>`).join("")}</ul></details>`) + `</div>`;
54
+ }
29
55
  function item(one, csrf) {
30
56
  const discard = one.why === "has changes"
31
57
  ? `<details class="discard"><summary>Discard</summary><form method="post" action="/settings/storage/discard"><input type="hidden" name="csrf" value="${e(csrf)}"><input type="hidden" name="path" value="${e(one.path)}">` +
@@ -58,7 +84,7 @@ export function storageHtml(view, notice) {
58
84
  const options = CLEANUP_CHOICES.map(one => `<option value="${one.value}"${one.value === plan.cleanup ? " selected" : ""}>${e(one.label)}</option>`).join("");
59
85
  return `<section class="storage">${note}` +
60
86
  `<div class="card storage-state"><h2 data-checkout-bytes="${plan.totalBytes}">Checkouts use ${e(bytesWords(plan.totalBytes))}</h2><p class="meta">${e(facts)}</p>` +
61
- `<p data-clean-bytes="${plan.freeBytes}">${e(frees)}</p>${preview}</div>` +
87
+ `<p data-clean-bytes="${plan.freeBytes}">${e(frees)}</p>${view.sweep === undefined ? "" : sweepHtml(view.sweep)}${preview}</div>` +
62
88
  `<h2>Automatic cleanup</h2>` +
63
89
  `<form method="post" action="/settings/storage" class="setting">${hidden}` +
64
90
  `<label>Remove a finished task's clean checkout<select name="cleanup">${options}</select></label>` +
package/dist/storage.d.ts CHANGED
@@ -1,6 +1,8 @@
1
1
  import type { Store } from "./store.js";
2
2
  /** Settings → Storage: when a finished task's clean checkout is removed. One row, or none (the default). */
3
3
  export declare const CHECKOUT_CLEANUP_SCHEMA = "\nCREATE TABLE IF NOT EXISTS checkout_cleanup_setting (\n id INTEGER PRIMARY KEY CHECK (id = 1),\n cleanup TEXT NOT NULL CHECK (cleanup IN ('finished', '2d', '7d', 'never')),\n updated_by TEXT NOT NULL,\n updated_at TEXT NOT NULL\n);\n";
4
+ /** What each storage sweep or clean-up by hand did (storage-sweep.ts): the latest few. */
5
+ export declare const STORAGE_SWEEP_SCHEMA = "\nCREATE TABLE IF NOT EXISTS storage_sweep (\n id INTEGER PRIMARY KEY,\n at TEXT NOT NULL,\n source TEXT NOT NULL CHECK (source IN ('automatic', 'manual')),\n actor TEXT NOT NULL,\n result TEXT NOT NULL\n);\n";
4
6
  export type CheckoutCleanup = "finished" | "2d" | "7d" | "never";
5
7
  export declare const DEFAULT_CLEANUP: CheckoutCleanup;
6
8
  export declare const CLEANUP_CHOICES: readonly {
package/dist/storage.js CHANGED
@@ -3,7 +3,9 @@
3
3
  * beside the database, by kind, and what storage retention takes care of.
4
4
  * Build checkouts and staged releases are the big ones; the worker removes a
5
5
  * finished task's clean checkout as the checkout cleanup setting says
6
- * (checkout-cleanup.ts), and a deploy keeps its last few releases.
6
+ * (checkout-cleanup.ts), and the daily storage sweep (storage-sweep.ts) takes
7
+ * what slips through: stale test temp folders, leftover processes, extra
8
+ * staged runtimes and dependency installs unused for a week.
7
9
  */
8
10
  import { spawnSync } from "node:child_process";
9
11
  import { lstatSync, readdirSync } from "node:fs";
@@ -18,6 +20,16 @@ CREATE TABLE IF NOT EXISTS checkout_cleanup_setting (
18
20
  updated_at TEXT NOT NULL
19
21
  );
20
22
  `;
23
+ /** What each storage sweep or clean-up by hand did (storage-sweep.ts): the latest few. */
24
+ export const STORAGE_SWEEP_SCHEMA = `
25
+ CREATE TABLE IF NOT EXISTS storage_sweep (
26
+ id INTEGER PRIMARY KEY,
27
+ at TEXT NOT NULL,
28
+ source TEXT NOT NULL CHECK (source IN ('automatic', 'manual')),
29
+ actor TEXT NOT NULL,
30
+ result TEXT NOT NULL
31
+ );
32
+ `;
21
33
  export const DEFAULT_CLEANUP = "finished";
22
34
  export const CLEANUP_CHOICES = [
23
35
  { value: "finished", label: "When its task is complete or cancelled", days: 0 },
@@ -124,8 +136,8 @@ export function storageReport(store, databaseFile, checkoutNote) {
124
136
  const lines = [
125
137
  { what: "Database", bytes: take(name => name.startsWith(base)) },
126
138
  { what: "Build checkouts", bytes: take(name => name === "worktrees"), count: checkouts.length, ...(checkoutNote === undefined ? {} : { note: checkoutNote }) },
127
- { what: "Shared dependencies", bytes: take(name => name === DEPS_FOLDER), count: sharedCopies(join(folder, DEPS_FOLDER)).length, note: "one install per lockfile, linked into checkouts" },
128
- { what: "Staged releases", bytes: take(name => name === "staged-upgrades"), count: staged, note: "a deploy keeps the release it installed, the one before and the newest few" },
139
+ { what: "Shared dependencies", bytes: take(name => name === DEPS_FOLDER), count: sharedCopies(join(folder, DEPS_FOLDER)).length, note: "one install per lockfile, linked into checkouts; one unused for a week goes" },
140
+ { what: "Staged releases", bytes: take(name => name === "staged-upgrades"), count: staged, note: "kept: the one running, the one before it and the newest" },
129
141
  { what: "Evidence", bytes: take(name => name === "evidence"), note: "the audit record of every run; kept" },
130
142
  { what: "Backups", bytes: take(name => name === "backups") },
131
143
  ];