toolroll 0.9.30 → 0.9.32

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. package/CHANGELOG.md +35 -0
  2. package/dist/assignment.d.ts +3 -0
  3. package/dist/assignment.js +8 -2
  4. package/dist/batch-checks.d.ts +15 -0
  5. package/dist/batch-checks.js +244 -0
  6. package/dist/batch-policy.d.ts +23 -0
  7. package/dist/batch-policy.js +25 -0
  8. package/dist/browser/workspace.js +13 -13
  9. package/dist/builder.js +14 -3
  10. package/dist/chat-actions.js +2 -2
  11. package/dist/check-levels-ui.d.ts +4 -0
  12. package/dist/check-levels-ui.js +21 -2
  13. package/dist/checkout-cleanup.d.ts +2 -1
  14. package/dist/checkout-cleanup.js +24 -8
  15. package/dist/desktop-update.d.ts +1 -1
  16. package/dist/exec.d.ts +2 -0
  17. package/dist/exec.js +11 -0
  18. package/dist/flow-gallery-ui.d.ts +14 -9
  19. package/dist/flow-gallery-ui.js +71 -29
  20. package/dist/flow-gallery.d.ts +26 -2
  21. package/dist/flow-gallery.js +182 -7
  22. package/dist/lead-capabilities.d.ts +0 -7
  23. package/dist/lead-capabilities.js +11 -1
  24. package/dist/mate-doors.js +2 -2
  25. package/dist/mate.d.ts +6 -2
  26. package/dist/mate.js +63 -38
  27. package/dist/mcp-connect.d.ts +3 -1
  28. package/dist/mcp-connect.js +11 -6
  29. package/dist/never-index.d.ts +3 -0
  30. package/dist/never-index.js +21 -0
  31. package/dist/operate.js +86 -2
  32. package/dist/orphan-sweep.d.ts +63 -0
  33. package/dist/orphan-sweep.js +235 -0
  34. package/dist/result-follow-ups.d.ts +22 -2
  35. package/dist/result-follow-ups.js +44 -10
  36. package/dist/serve.js +52 -19
  37. package/dist/shared-deps.d.ts +18 -2
  38. package/dist/shared-deps.js +129 -15
  39. package/dist/staged-runtimes.d.ts +42 -0
  40. package/dist/staged-runtimes.js +191 -0
  41. package/dist/storage-sweep.d.ts +50 -0
  42. package/dist/storage-sweep.js +203 -0
  43. package/dist/storage-ui.d.ts +5 -0
  44. package/dist/storage-ui.js +30 -4
  45. package/dist/storage.d.ts +2 -0
  46. package/dist/storage.js +15 -3
  47. package/dist/store.d.ts +17 -2
  48. package/dist/store.js +37 -5
  49. package/dist/subscription-chat.d.ts +3 -1
  50. package/dist/subscription-chat.js +1 -0
  51. package/dist/surface.js +1 -0
  52. package/dist/task-status.d.ts +13 -0
  53. package/dist/task-status.js +37 -3
  54. package/dist/team-leads.js +6 -2
  55. package/dist/team-runtime.js +2 -1
  56. package/dist/test-temp.d.ts +20 -2
  57. package/dist/test-temp.js +53 -13
  58. package/dist/toolroll-update.js +2 -0
  59. package/dist/workspace-ui.js +3 -1
  60. package/dist/worktree.js +9 -0
  61. package/package.json +1 -1
@@ -30,10 +30,13 @@ import { envValue } from "./names.js";
30
30
  export const ONE_CLICK = [
31
31
  { id: "stripe", label: "Stripe", url: "https://mcp.stripe.com/", about: "Payments, customers, refunds and invoices." },
32
32
  { id: "notion", label: "Notion", url: "https://mcp.notion.com/mcp", about: "Pages and databases: search, read and write." },
33
- { id: "linear", label: "Linear", url: "https://mcp.linear.app/mcp", about: "Issues, projects and comments." },
34
- { id: "sentry", label: "Sentry", url: "https://mcp.sentry.dev/mcp", about: "Errors and issues, with their stack traces." },
33
+ { id: "linear", label: "Linear", url: "https://mcp.linear.app/mcp", about: "Issues, projects and comments.",
34
+ reads: ["issue", "project", "team", "comment", "label", "cycle", "document", "status", "statuses", "user"] },
35
+ { id: "sentry", label: "Sentry", url: "https://mcp.sentry.dev/mcp", about: "Errors and issues, with their stack traces.",
36
+ reads: ["organization", "project", "issue", "detail", "details", "event", "trace", "release", "tag", "attachment"] },
35
37
  { id: "atlassian", label: "Jira and Confluence", url: "https://mcp.atlassian.com/v1/mcp", about: "Jira issues and Confluence pages." },
36
- { id: "intercom", label: "Intercom", url: "https://mcp.intercom.com/mcp", about: "Customer conversations and contacts." },
38
+ { id: "intercom", label: "Intercom", url: "https://mcp.intercom.com/mcp", about: "Customer conversations and contacts.",
39
+ reads: ["conversation", "contact"] },
37
40
  { id: "attio", label: "Attio", url: "https://mcp.attio.com/mcp", about: "Your CRM: people, companies, deals and notes." },
38
41
  { id: "zapier", label: "Zapier", url: "https://mcp.zapier.com/api/mcp/mcp", about: "Thousands of apps, through the actions you choose in Zapier." },
39
42
  { id: "square", label: "Square", url: "https://mcp.squareup.com/mcp", about: "Payments, orders, customers and catalog." },
@@ -42,8 +45,10 @@ export const ONE_CLICK = [
42
45
  { id: "webflow", label: "Webflow", url: "https://mcp.webflow.com/mcp", about: "Sites, pages and CMS items." },
43
46
  { id: "wix", label: "Wix", url: "https://mcp.wix.com/mcp", about: "Your Wix site, store and bookings." },
44
47
  { id: "canva", label: "Canva", url: "https://mcp.canva.com/mcp", about: "Designs: search, create and export." },
45
- { id: "vercel", label: "Vercel", url: "https://mcp.vercel.com/", about: "Projects, deployments and logs." },
46
- { id: "cloudflare", label: "Cloudflare", url: "https://mcp.cloudflare.com/mcp", about: "Workers, DNS and your Cloudflare account." },
48
+ { id: "vercel", label: "Vercel", url: "https://mcp.vercel.com/", about: "Projects, deployments and logs.",
49
+ reads: ["team", "project", "deployment", "build", "runtime", "log", "logs", "vercel", "documentation"] },
50
+ { id: "cloudflare", label: "Cloudflare", url: "https://mcp.cloudflare.com/mcp", about: "Workers, DNS and your Cloudflare account.",
51
+ reads: ["account", "worker", "workers", "observability", "analytics", "log", "logs", "page", "pages", "project", "deployment"] },
47
52
  // Mobbin's sign-in is its Supabase auth server, on another origin: its protected-resource metadata names it.
48
53
  { id: "mobbin", label: "Mobbin", url: "https://api.mobbin.com/mcp", about: "Real app screens and flows to learn from.",
49
54
  reads: ["screen", "flow", "app", "site", "element", "pattern", "ui", "ios", "android", "web"] },
@@ -237,7 +242,7 @@ export async function startConnect(input, fetcher = fetch, now = Date.now()) {
237
242
  go.searchParams.set("resource", resource);
238
243
  if (server.scopes.length > 0)
239
244
  go.searchParams.set("scope", server.scopes.join(" "));
240
- return { ok: true, go: go.toString(), state, visit: { service: service.id, repo: input.repo, by: input.by, kit: input.kit ?? null, verifier, clientId, clientSecret, token: server.token, resource, redirect, expires: now + VISIT_MS } };
245
+ return { ok: true, go: go.toString(), state, visit: { service: service.id, repo: input.repo, by: input.by, kit: input.kit ?? null, template: input.template ?? null, verifier, clientId, clientSecret, token: server.token, resource, redirect, expires: now + VISIT_MS } };
241
246
  }
242
247
  /** The spec a connected service joins the project with: its MCP address, signed in with the token its sign-in gave. */
243
248
  export function connectedSpec(serviceId) {
@@ -0,0 +1,3 @@
1
+ export declare const NEVER_INDEX = ".metadata_never_index";
2
+ /** Put the marker in `dir` (an existing folder). False when it couldn't be written. */
3
+ export declare function markNeverIndex(dir: string): boolean;
@@ -0,0 +1,21 @@
1
+ /**
2
+ * Spotlight leaves Toolroll's churn alone: a `.metadata_never_index` file in a folder tells macOS not to index it or
3
+ * what is inside. Toolroll puts one in the folders it fills and empties all day — the checkouts folder and each
4
+ * project's folder in it (never inside a checkout, where a build's `git add -A` would commit it), shared dependency
5
+ * installs, staged runtimes and test temp roots. Measured Oct 4: fseventsd at 15 GB with 107k test temp folders.
6
+ *
7
+ * A marker that can't be written is only noted: it never decides whether anything is removed.
8
+ */
9
+ import { writeFileSync } from "node:fs";
10
+ import { join } from "node:path";
11
+ export const NEVER_INDEX = ".metadata_never_index";
12
+ /** Put the marker in `dir` (an existing folder). False when it couldn't be written. */
13
+ export function markNeverIndex(dir) {
14
+ try {
15
+ writeFileSync(join(dir, NEVER_INDEX), "", { flag: "a" });
16
+ return true;
17
+ }
18
+ catch {
19
+ return false;
20
+ }
21
+ }
package/dist/operate.js CHANGED
@@ -3,6 +3,8 @@ import { leadNameOf } from "./lead-identity.js";
3
3
  import { maybeTriggerRepair } from "./dispose.js";
4
4
  import { CHECK_LEVEL_HINTS, CHECK_LEVEL_WORDS, isCheckLevel, liveQuickCommand, projectCheckLevel, quickVerifyKey, setProjectCheckLevel, setTaskCheckLevel, suggestQuickCommand } from "./check-levels.js";
5
5
  import { fileAddTestsTask, followUpChecksOf, requestFollowUpChecks, runFollowUpCheck, runWaitingChecks } from "./result-follow-ups.js";
6
+ import { runBatchChecks } from "./batch-checks.js";
7
+ import { BATCH_HINT, BATCH_WINDOW_MS, projectBatchChecks, setProjectBatchChecks } from "./batch-policy.js";
6
8
  import { buildReviewPass } from "./build-review.js";
7
9
  import { buildReviewLines, buildReviewOf } from "./review-switch.js";
8
10
  import { parseProtectedPaths } from "./approval-policy.js";
@@ -11,6 +13,7 @@ import { evidencePack, exportDay, ledgerExportChunks, standaloneEvidenceHtml } f
11
13
  import { matchesOutsideCheckpoint } from "./ledger-chain.js";
12
14
  import { CLEANUP_CHOICES, bytesWords, cleanupWords, diskBytes, parseCleanup, storageReport } from "./storage.js";
13
15
  import { removeStaleTestTemp, tempRoots, testTempFolders } from "./test-temp.js";
16
+ import { dailyStorageSweep, lastSweep, nextSweepAt, saveSweep, storageSweepOff as storageSweepOffNow, sweepDetails, sweepWords as storageSweepWords } from "./storage-sweep.js";
14
17
  import { removeShared, sharedDepsRoot, sharedUse, unusedShared } from "./shared-deps.js";
15
18
  import { checkoutPlan, cleanCheckouts, discardCheckout, finishedCheckouts, slimKeptCheckouts, whyWords } from "./checkout-cleanup.js";
16
19
  import { closeOutCheckouts, closeOutRuns, stoppedWords } from "./run-closeout.js";
@@ -996,6 +999,8 @@ async function dispatch(command, positional, flags, context) {
996
999
  return projectDeleteCommand(positional, flags, context);
997
1000
  if (positional[0] === "concurrency")
998
1001
  return projectConcurrencyCommand(positional, flags, context);
1002
+ if (positional[0] === "checks")
1003
+ return projectChecksCommand(positional, flags, context);
999
1004
  if (positional[0] === "demo")
1000
1005
  return projectDemoCommand(positional, flags, context);
1001
1006
  return runProjectCommand(positional, flags, context);
@@ -4541,7 +4546,18 @@ async function reconcileCommand(flags, context) {
4541
4546
  retention = null; /* the next pass tries again */
4542
4547
  }
4543
4548
  const retained = retention === null ? [] : retention.counts.filter(one => one.count > 0);
4549
+ // The daily storage sweep (storage-sweep.ts): stale test temp folders, orphans Toolroll left running, finished
4550
+ // checkouts, extra staged runtimes and dependency installs unused for a week; saved for Settings → Storage.
4551
+ let storageSwept = null;
4552
+ try {
4553
+ storageSwept = await dailyStorageSweep(store, { databaseFile: context.databaseFile, pool: worktrees, tempRoots: context.tempRoots ?? tempRoots() }, clock);
4554
+ }
4555
+ catch {
4556
+ storageSwept = null; /* the next pass tries again */
4557
+ }
4558
+ const swept = storageSwept === null ? [] : storageSwept.parts.filter(one => one.count > 0);
4544
4559
  const nothing = retained.length === 0 &&
4560
+ swept.length === 0 &&
4545
4561
  recovered.length === 0 &&
4546
4562
  reaped.length === 0 &&
4547
4563
  adoption.adopted.length === 0 &&
@@ -4562,6 +4578,7 @@ async function reconcileCommand(flags, context) {
4562
4578
  checkoutsSlimmed: slimmed.slimmed.map(one => one.path),
4563
4579
  liveViewsSwept: liveSwept.removed.length,
4564
4580
  retention: retention === null ? null : { counts: retention.counts, freedBytes: retention.freed },
4581
+ storageSweep: storageSwept,
4565
4582
  }, () => nothing
4566
4583
  ? ["Nothing to reconcile. Everything is where it should be."]
4567
4584
  : [
@@ -4575,6 +4592,7 @@ async function reconcileCommand(flags, context) {
4575
4592
  ...(slimmed.slimmed.length === 0 ? [] : [`Dropped dependencies and build output from ${slimmed.slimmed.length} kept checkout(s), about ${bytesWords(slimmed.slimmed.reduce((sum, one) => sum + one.bytes, 0))}.`]),
4576
4593
  ...(liveSwept.removed.length === 0 ? [] : [`Cleared ${liveSwept.removed.length} finished live view(s).`]),
4577
4594
  ...(retention === null || retained.length === 0 ? [] : [`Retention removed ${sweepWords(retention.counts, retention.freed)}.`]),
4595
+ ...(storageSwept === null || swept.length === 0 ? [] : [`Storage sweep: ${storageSweepWords(storageSwept.parts)}`]),
4578
4596
  ]);
4579
4597
  }
4580
4598
  // ---- gaps -----------------------------------------------------------------
@@ -7658,11 +7676,15 @@ async function runWatchLoop(args) {
7658
7676
  run: async () => {
7659
7677
  if (store.isDemo())
7660
7678
  return;
7679
+ // Batch checks first: a cohort that is ready runs one check for all its results.
7680
+ const batched = await runBatchChecks(store, context.evidenceRoot, { now: context.clock, shouldStop: stopping });
7681
+ if (batched > 0)
7682
+ progress(`watch: settled ${batched} batch-checked result(s)`);
7661
7683
  const ran = await runWaitingChecks(store, context.evidenceRoot, { now: context.clock, shouldStop: stopping });
7662
7684
  if (ran > 0)
7663
7685
  progress(`watch: ran ${ran} follow-up check(s)`);
7664
7686
  // A finished check may send a build's card down its failure path: now, not on the next beat.
7665
- if (ran > 0)
7687
+ if (ran + batched > 0)
7666
7688
  moveCards(store, repo, context.clock(), context.evidenceRoot);
7667
7689
  },
7668
7690
  });
@@ -12231,6 +12253,45 @@ async function projectConcurrencyCommand(positional, flags, context) {
12231
12253
  ...(changed.before > changed.after ? ["Builds already running carry on; the new number applies to the next one."] : []),
12232
12254
  ]);
12233
12255
  }
12256
+ /** `project checks [--batch on|off] --repo <p>`: whether the project's results share one full check when they
12257
+ * finish close together (batch-checks.ts). Off by default; changing it is an approver's act, in the ledger. */
12258
+ async function projectChecksCommand(positional, flags, context) {
12259
+ const command = "project checks";
12260
+ const allowed = new Set(["repo", "batch", "as", "token", "token-file", "token-env", "db", "json"]);
12261
+ for (const name of flags.keys())
12262
+ if (!allowed.has(name))
12263
+ return fail(context.write, context.json, command, "usage", `--${name} is not a project checks option.`, EXIT.usage);
12264
+ const repoFlag = text(flags, "repo");
12265
+ const wanted = flags.get("batch");
12266
+ if (positional.length > 1 || repoFlag === undefined || (wanted !== undefined && wanted !== "on" && wanted !== "off")) {
12267
+ return fail(context.write, context.json, command, "usage", "Use project checks [--batch on|off] --repo <project path>.", EXIT.usage);
12268
+ }
12269
+ const registered = await loadRepos(registryPathOf(context)).catch(() => ({ error: "unreadable" }));
12270
+ const known = [...new Set([...context.store.knownRepos(), ...context.store.listProjects().map(one => one.path), ...("error" in registered ? [] : registered.repos)])];
12271
+ const repo = known.find(one => one === repoFlag || one === resolve(repoFlag) || one === canonicalProject(repoFlag));
12272
+ if (repo === undefined)
12273
+ return fail(context.write, context.json, command, "not-found", "That isn't a project Toolroll knows.", EXIT.refused);
12274
+ const level = projectCheckLevel(context.store, repo).level;
12275
+ const words = (on) => [
12276
+ `${repo}: batch checks are ${on ? "on" : "off"}. ${on ? BATCH_HINT : "Each result runs its own full check."}`,
12277
+ ...(on && level !== "full" ? [`Checks are ${CHECK_LEVEL_WORDS[level]} for this project, so nothing is batched until they are Full.`] : []),
12278
+ ];
12279
+ if (wanted === undefined) {
12280
+ const current = projectBatchChecks(context.store, repo);
12281
+ return succeed(context.write, context.json, command, { repo, batch: current.on, windowMinutes: current.windowMs / 60_000, level, setBy: current.setBy }, () => words(current.on));
12282
+ }
12283
+ const acting = await askCredentials(flags, context);
12284
+ const verified = acting === null ? null : authenticateApprover(context.store, acting.name, acting.token, repo);
12285
+ if (acting === null || verified === null || !verified.ok || !context.store.accountCanAccess(acting.name, repo)) {
12286
+ return fail(context.write, context.json, command, "refused", "An approver for this project changes batch checks: pass --as and --token (or use the remembered login).", EXIT.refused);
12287
+ }
12288
+ const on = wanted === "on";
12289
+ const changed = setProjectBatchChecks(context.store, repo, on, acting.name, context.clock());
12290
+ return succeed(context.write, context.json, command, { repo, batch: on, before: changed.before, changed: changed.changed, windowMinutes: BATCH_WINDOW_MS / 60_000, level }, () => [
12291
+ ...words(on),
12292
+ ...(changed.changed && !on ? ["Results already waiting are checked now."] : []),
12293
+ ]);
12294
+ }
12234
12295
  /** `project demo [<url>|off] --repo <p>` (review 827): the project's own demo or dev server, which its scouts may
12235
12296
  * screenshot. Changing it is an approver's act. */
12236
12297
  async function projectDemoCommand(positional, flags, context) {
@@ -12625,17 +12686,33 @@ async function storageCommand(positional, flags, context) {
12625
12686
  return copies.map(one => ({ key: one.key, repo: one.repo, path: one.dir, bytes: sizes.get(one.dir) ?? 0, checkouts: one.checkouts.length, usedBy: one.checkouts, retired: one.retired, node: one.node, createdAt: one.createdAt, usedAt: one.usedAt }));
12626
12687
  };
12627
12688
  const sharedLine = (one) => ` ${bytesWords(one.bytes).padStart(8)} ${projectName(one.repo)} ${one.key.slice(0, 8)}: ${one.checkouts === 1 ? "1 checkout uses it" : `${one.checkouts} checkouts use it`}${one.retired ? " (changed after install; never linked again)" : ""}`;
12689
+ // What the latest sweep (automatic, or a clean-up by hand) did and when (storage-sweep.ts).
12690
+ const sweepFacts = () => {
12691
+ const last = lastSweep(store);
12692
+ const automatic = last?.source === "automatic" ? last : lastSweep(store, "automatic");
12693
+ return { last, lastAutomatic: automatic, next: nextSweepAt(store) };
12694
+ };
12695
+ const sweepLines = (facts) => {
12696
+ const when = (at) => new Date(at).toLocaleString("en-US", { month: "short", day: "numeric", hour: "numeric", minute: "2-digit" });
12697
+ const one = facts.last;
12698
+ const head = one === null ? (facts.next === null && storageSweepOffNow() ? "Automatic sweep: off." : "Automatic sweep: not run yet; it runs once a day.")
12699
+ : `Last swept ${when(one.at)}${one.source === "manual" ? ` by ${one.actor}` : " automatically"}: ${storageSweepWords(one.parts)}`;
12700
+ const auto = one !== null && one.source === "manual" && facts.lastAutomatic !== null ? [`Last automatic sweep ${when(facts.lastAutomatic.at)}: ${storageSweepWords(facts.lastAutomatic.parts)}`] : [];
12701
+ return [head, ...auto, ...(one === null ? [] : sweepDetails(one.parts).slice(0, 8).map(line => ` ${line}`))];
12702
+ };
12628
12703
  if (action === undefined) {
12629
12704
  const plan = await checkoutPlan(store, pool, context.clock(), { manual: true });
12630
12705
  const report = storageReport(store, context.databaseFile, plan.go.length === 0 ? "nothing to clean up" : `about ${bytesWords(plan.freeBytes)} to clean up`);
12631
12706
  const { staleBytes: _, ...temp } = leftovers(false);
12632
12707
  const shared = sharedFacts(sharedUse(depsRoot, checkoutRows()));
12633
- return succeed(context.write, context.json, "storage", { ...report, checkouts: checkoutSummary(plan), testTemp: temp, shared }, () => [
12708
+ const sweep = sweepFacts();
12709
+ return succeed(context.write, context.json, "storage", { ...report, checkouts: checkoutSummary(plan), testTemp: temp, shared, sweep }, () => [
12634
12710
  `${report.folder}: ${bytesWords(report.total)}`,
12635
12711
  ...report.lines.map(one => ` ${one.what.padEnd(19)} ${bytesWords(one.bytes).padStart(8)}${one.count === undefined ? "" : ` (${one.count})`}${one.note === undefined ? "" : ` ${one.note}`}`),
12636
12712
  ...(shared.length === 0 ? [] : ["Shared dependencies:", ...shared.map(sharedLine)]),
12637
12713
  ...summary(plan),
12638
12714
  leftoverLine({ ...temp, staleBytes: 0 }),
12715
+ ...sweepLines(sweep),
12639
12716
  ]);
12640
12717
  }
12641
12718
  if (action === "cleanup") {
@@ -12692,6 +12769,13 @@ async function storageCommand(positional, flags, context) {
12692
12769
  for (const debris of unused.debris)
12693
12770
  removeShared(debris);
12694
12771
  const sharedFreed = sharedRemoved.reduce((sum, one) => sum + one.bytes, 0);
12772
+ // Saved like an automatic sweep, so Storage shows what this clean-up did and when.
12773
+ const parts = [
12774
+ { kind: "checkouts", count: done.removed.length, bytes: done.freed, failed: done.kept.filter(one => one.why === "git refused").length, items: done.removed.slice(0, 20).map(one => one.path) },
12775
+ { kind: "test temp", count: swept.removed.length, bytes: temp.staleBytes, failed: swept.failed.length, items: swept.removed.slice(0, 20) },
12776
+ { kind: "dependencies", count: sharedRemoved.length, bytes: sharedFreed, failed: unused.copies.length - sharedRemoved.length, items: sharedRemoved.slice(0, 20).map(one => one.path) },
12777
+ ];
12778
+ saveSweep(store, { at: context.clock().toISOString(), source: "manual", actor, parts });
12695
12779
  return succeed(context.write, context.json, command, { removed: done.removed, kept: done.kept, freedBytes: done.freed, testTempRemoved: swept.removed.length, testTempFailed: swept.failed, sharedRemoved, sharedFreedBytes: sharedFreed }, () => [
12696
12780
  done.removed.length === 0 ? (swept.removed.length === 0 && sharedRemoved.length === 0 ? "Nothing to clean up." : "No checkouts to clean up.") : `Removed ${done.removed.length} checkout${done.removed.length === 1 ? "" : "s"}, about ${bytesWords(done.freed)}; their branches stay.`,
12697
12781
  ...done.removed.map(itemLine),
@@ -0,0 +1,63 @@
1
+ import { type ProcessRow, type StopOptions } from "./run-closeout.js";
2
+ import type { Store } from "./store.js";
3
+ import { type TempOwner } from "./test-temp.js";
4
+ import type { WorktreePool } from "./worktree.js";
5
+ /** Where a process runs from: its working folder and executable, when the OS says. */
6
+ export type Located = {
7
+ cwd?: string;
8
+ exe?: string;
9
+ };
10
+ /** Working folder and executable of each pid: /proc on Linux, lsof on macOS. Throws when the OS can't be asked. */
11
+ export declare function locateProcesses(pids: readonly number[]): Map<number, Located>;
12
+ /** A folder an orphan may be stopped from: a let-go checkout (only what was born before it was let go) or a temp root. */
13
+ export type SweepRoot = {
14
+ path: string;
15
+ kind: "checkout" | "test temp";
16
+ bornBefore?: number;
17
+ repo?: string | null;
18
+ taskId?: string | null;
19
+ };
20
+ /** The root `located` runs from inside of, if any (its working folder first, then its executable). */
21
+ export declare function rootOf(located: Located | undefined, roots: readonly SweepRoot[]): SweepRoot | null;
22
+ export type OrphanTarget = {
23
+ row: ProcessRow;
24
+ root: SweepRoot;
25
+ orphan: number;
26
+ };
27
+ /**
28
+ * Which processes go: each orphan (parent 1, no terminal, not protected) running from inside a root — for a checkout,
29
+ * born before it was let go — and every process descending from one that goes (none at a terminal or protected).
30
+ */
31
+ export declare function orphanTargets(census: readonly ProcessRow[], located: ReadonlyMap<number, Located>, roots: readonly SweepRoot[], guard: {
32
+ pids: ReadonlySet<number>;
33
+ groups: ReadonlySet<number>;
34
+ }): OrphanTarget[];
35
+ /** Whether a temp root's owner (pid and start) is still the running process. */
36
+ export declare function ownerRunning(census: readonly ProcessRow[]): (owner: TempOwner) => boolean;
37
+ /**
38
+ * The roots orphans may be stopped from now: every checkout let go with no live run on its task and not held, and
39
+ * every test temp root whose owner is gone or that nothing touched for a day.
40
+ */
41
+ export declare function sweepRoots(store: Store, pool: WorktreePool | null, census: readonly ProcessRow[], tempRoots: readonly string[], now: Date): SweepRoot[];
42
+ export type OrphanStop = {
43
+ pid: number;
44
+ name: string;
45
+ where: string;
46
+ kind: SweepRoot["kind"];
47
+ ranMs: number;
48
+ killed: boolean;
49
+ };
50
+ export type OrphanSweep = {
51
+ stopped: OrphanStop[];
52
+ failed: string[];
53
+ };
54
+ export type OrphanOptions = StopOptions & {
55
+ actor?: string;
56
+ pool?: WorktreePool | null;
57
+ tempRoots?: readonly string[];
58
+ locate?: (pids: readonly number[]) => Map<number, Located>;
59
+ self?: number;
60
+ };
61
+ /** Stop the orphans Toolroll left (see the top of this file), each in the ledger. */
62
+ export declare function sweepOrphans(store: Store, now: () => Date, options?: OrphanOptions): Promise<OrphanSweep>;
63
+ export declare function durationWords(ms: number): string;
@@ -0,0 +1,235 @@
1
+ /**
2
+ * Orphans Toolroll left running (the daily storage sweep, storage-sweep.ts): a process whose parent is gone (parent
3
+ * pid 1) that runs from inside a task checkout whose run has ended, or from inside a test temp root whose suite is
4
+ * gone — a preview server a run started and never stopped (Oct 4: `astro preview --port 4399` from a toolroll-site
5
+ * checkout, 4 days on), a browser a killed journey left. It is stopped with whatever it started (SIGTERM, then
6
+ * SIGKILL after the grace), and each one is in the ledger.
7
+ *
8
+ * "Runs from inside" is its working folder or its executable, read from the OS (lsof on macOS, /proc on Linux) —
9
+ * never its arguments, which can carry private provider inputs. An observation that can't be read stops nothing.
10
+ *
11
+ * Never touched: anything whose parent is alive (a person's shell, the lead's deploy), anything at a terminal, the
12
+ * live service and its ancestry, anything in a checkout leased or with a live run, a process born after its
13
+ * checkout was let go (somebody started it since), anything in a temp root whose owner still runs or that something
14
+ * touched within the day, and anything outside Toolroll's checkouts and test temp roots. Each target is looked at
15
+ * again right before the signal: the same process (pid and birth), still an orphan, still inside.
16
+ */
17
+ import { execFileSync } from "node:child_process";
18
+ import { existsSync, readlinkSync, realpathSync } from "node:fs";
19
+ import { basename, sep } from "node:path";
20
+ import { processCensus, protectedProcesses, stopProcesses } from "./run-closeout.js";
21
+ import { tempOwner, testTempFolders } from "./test-temp.js";
22
+ const DAY_MS = 86_400_000;
23
+ /** `ps` ages are whole seconds. */
24
+ const BIRTH_SLACK_MS = 2_000;
25
+ /** Working folder and executable of each pid: /proc on Linux, lsof on macOS. Throws when the OS can't be asked. */
26
+ export function locateProcesses(pids) {
27
+ const found = new Map();
28
+ if (pids.length === 0 || process.platform === "win32")
29
+ return found;
30
+ if (process.platform === "linux") {
31
+ for (const pid of pids) {
32
+ const one = {};
33
+ try {
34
+ one.cwd = readlinkSync(`/proc/${pid}/cwd`);
35
+ }
36
+ catch { /* gone, or not ours */ }
37
+ try {
38
+ one.exe = readlinkSync(`/proc/${pid}/exe`);
39
+ }
40
+ catch { /* gone, or not ours */ }
41
+ if (one.cwd !== undefined || one.exe !== undefined)
42
+ found.set(pid, one);
43
+ }
44
+ return found;
45
+ }
46
+ for (let at = 0; at < pids.length; at += 400) {
47
+ let text;
48
+ try {
49
+ text = execFileSync("lsof", ["-nP", "-a", "-p", pids.slice(at, at + 400).join(","), "-d", "cwd,txt", "-Fpfn"], { encoding: "utf8", timeout: 30_000, maxBuffer: 64 * 1024 * 1024, stdio: ["ignore", "pipe", "ignore"] });
50
+ }
51
+ catch (error) {
52
+ // lsof exits 1 when some pid has gone or can't be read; what it printed still stands. No output at all: unreadable.
53
+ const out = error.stdout;
54
+ if (typeof out !== "string" || error.status !== 1)
55
+ throw new Error("lsof could not list working folders");
56
+ text = out;
57
+ }
58
+ let pid = 0;
59
+ let fd = "";
60
+ for (const line of text.split("\n")) {
61
+ if (line.startsWith("p")) {
62
+ pid = Number(line.slice(1));
63
+ fd = "";
64
+ }
65
+ else if (line.startsWith("f"))
66
+ fd = line.slice(1);
67
+ else if (line.startsWith("n") && pid > 0) {
68
+ const one = found.get(pid) ?? {};
69
+ // The first txt is the executable; the rest are libraries it loaded.
70
+ if (fd === "cwd")
71
+ one.cwd = line.slice(1);
72
+ else if (fd === "txt" && one.exe === undefined)
73
+ one.exe = line.slice(1);
74
+ found.set(pid, one);
75
+ }
76
+ }
77
+ }
78
+ return found;
79
+ }
80
+ function canonical(path) {
81
+ let real = path;
82
+ try {
83
+ real = realpathSync(path);
84
+ }
85
+ catch { /* gone */ }
86
+ return [...new Set([path, real])];
87
+ }
88
+ /** The root `located` runs from inside of, if any (its working folder first, then its executable). */
89
+ export function rootOf(located, roots) {
90
+ if (located === undefined)
91
+ return null;
92
+ for (const at of [located.cwd, located.exe]) {
93
+ if (at === undefined || at === "")
94
+ continue;
95
+ for (const root of roots) {
96
+ if (canonical(root.path).some(base => base !== "/" && base !== "" && (at === base || at.startsWith(base + sep))))
97
+ return root;
98
+ }
99
+ }
100
+ return null;
101
+ }
102
+ /**
103
+ * Which processes go: each orphan (parent 1, no terminal, not protected) running from inside a root — for a checkout,
104
+ * born before it was let go — and every process descending from one that goes (none at a terminal or protected).
105
+ */
106
+ export function orphanTargets(census, located, roots, guard) {
107
+ const safe = (row) => row.pid > 1 && !row.terminal && !guard.pids.has(row.pid) && !guard.groups.has(row.pgid);
108
+ const out = new Map();
109
+ for (const row of census) {
110
+ if (row.ppid !== 1 || !safe(row))
111
+ continue;
112
+ const root = rootOf(located.get(row.pid), roots);
113
+ if (root === null)
114
+ continue;
115
+ if (root.bornBefore !== undefined && row.bornAt > root.bornBefore + BIRTH_SLACK_MS)
116
+ continue;
117
+ out.set(row.pid, { row, root, orphan: row.pid });
118
+ }
119
+ // Whatever an orphan that goes started goes with it.
120
+ const children = new Map();
121
+ for (const row of census)
122
+ children.set(row.ppid, [...(children.get(row.ppid) ?? []), row]);
123
+ for (const top of [...out.values()]) {
124
+ const pending = [...(children.get(top.row.pid) ?? [])];
125
+ while (pending.length > 0) {
126
+ const row = pending.pop();
127
+ if (out.has(row.pid) || !safe(row))
128
+ continue;
129
+ out.set(row.pid, { row, root: top.root, orphan: top.orphan });
130
+ pending.push(...(children.get(row.pid) ?? []));
131
+ }
132
+ }
133
+ return [...out.values()];
134
+ }
135
+ /** Whether a temp root's owner (pid and start) is still the running process. */
136
+ export function ownerRunning(census) {
137
+ const byPid = new Map(census.map(row => [row.pid, row]));
138
+ return owner => { const now = byPid.get(owner.pid); return now !== undefined && Math.abs(now.bornAt - owner.startedAt) <= BIRTH_SLACK_MS; };
139
+ }
140
+ /**
141
+ * The roots orphans may be stopped from now: every checkout let go with no live run on its task and not held, and
142
+ * every test temp root whose owner is gone or that nothing touched for a day.
143
+ */
144
+ export function sweepRoots(store, pool, census, tempRoots, now) {
145
+ const roots = [];
146
+ const live = store.handle.prepare("SELECT 1 FROM run WHERE task_ref = ? AND finished_at IS NULL LIMIT 1");
147
+ for (const row of store.listWorktrees()) {
148
+ if (row.releasedAt === null || row.runner !== null || !existsSync(row.path))
149
+ continue;
150
+ if (pool !== null && pool.inUse(row.path).held)
151
+ continue;
152
+ if (row.taskRef !== null && live.get(row.taskRef) !== undefined)
153
+ continue;
154
+ const releasedAt = Date.parse(row.releasedAt);
155
+ if (!Number.isFinite(releasedAt))
156
+ continue;
157
+ roots.push({ path: row.path, kind: "checkout", bornBefore: releasedAt, repo: row.repo, taskId: row.taskRef === null ? null : store.externalIdFor(row.taskRef) });
158
+ }
159
+ const running = ownerRunning(census);
160
+ const found = testTempFolders(tempRoots, now);
161
+ for (const one of found.all) {
162
+ const owner = tempOwner(one.path);
163
+ const gone = owner !== null ? !running(owner) : now.getTime() - one.touchedAt > DAY_MS;
164
+ if (gone)
165
+ roots.push({ path: one.path, kind: "test temp" });
166
+ }
167
+ return roots;
168
+ }
169
+ /** Stop the orphans Toolroll left (see the top of this file), each in the ledger. */
170
+ export async function sweepOrphans(store, now, options = {}) {
171
+ if (process.platform === "win32")
172
+ return { stopped: [], failed: [] };
173
+ const read = options.census ?? (() => processCensus());
174
+ const locate = options.locate ?? locateProcesses;
175
+ const failed = [];
176
+ let census;
177
+ try {
178
+ census = read();
179
+ }
180
+ catch {
181
+ return { stopped: [], failed: ["the process list couldn't be read"] };
182
+ }
183
+ const roots = sweepRoots(store, options.pool ?? null, census, options.tempRoots ?? [], now());
184
+ if (roots.length === 0)
185
+ return { stopped: [], failed };
186
+ const guard = protectedProcesses(census, options.self);
187
+ const orphans = census.filter(row => row.ppid === 1 && row.pid > 1 && !row.terminal && !guard.pids.has(row.pid) && !guard.groups.has(row.pgid)).map(row => row.pid);
188
+ let located;
189
+ try {
190
+ located = locate(orphans);
191
+ }
192
+ catch (error) {
193
+ return { stopped: [], failed: [`working folders couldn't be read: ${error.message}`] };
194
+ }
195
+ const first = orphanTargets(census, located, roots, guard);
196
+ if (first.length === 0)
197
+ return { stopped: [], failed };
198
+ // Right before the signal, everything again: the same process, still an orphan (or under one), still inside.
199
+ let again;
200
+ try {
201
+ const fresh = read();
202
+ const freshGuard = protectedProcesses(fresh, options.self);
203
+ const freshLocated = locate(fresh.filter(row => row.ppid === 1).map(row => row.pid));
204
+ const before = new Map(first.map(one => [one.row.pid, one]));
205
+ again = orphanTargets(fresh, freshLocated, roots, freshGuard).filter(one => {
206
+ const was = before.get(one.row.pid);
207
+ return was !== undefined && Math.abs(was.row.bornAt - one.row.bornAt) <= BIRTH_SLACK_MS && was.root.path === one.root.path;
208
+ });
209
+ }
210
+ catch (error) {
211
+ return { stopped: [], failed: [`a second look failed: ${error.message}`] };
212
+ }
213
+ if (again.length === 0)
214
+ return { stopped: [], failed };
215
+ const done = await stopProcesses(again.map(one => one.row), options);
216
+ const killed = new Set(done.killed.map(row => row.pid));
217
+ const at = now();
218
+ const stopped = [];
219
+ for (const one of again) {
220
+ const ranMs = Math.max(0, at.getTime() - one.row.bornAt);
221
+ const where = basename(one.root.path);
222
+ stopped.push({ pid: one.row.pid, name: one.row.name, where: one.root.path, kind: one.root.kind, ranMs, killed: killed.has(one.row.pid) });
223
+ store.recordAction({ at: at.toISOString(), actor: options.actor ?? "worker", repo: one.root.repo ?? null, taskId: one.root.taskId ?? null, runId: null,
224
+ action: "orphan process stopped", outcome: "stopped", source: "work",
225
+ detail: `${one.row.name} (${one.row.pid}) running ${durationWords(ranMs)} from ${one.root.kind === "checkout" ? "checkout" : "test temp folder"} ${where}${one.orphan === one.row.pid ? "; its parent had gone" : `; started by orphan ${one.orphan}`}${killed.has(one.row.pid) ? "; needed SIGKILL" : ""}` });
226
+ }
227
+ return { stopped, failed };
228
+ }
229
+ export function durationWords(ms) {
230
+ const minutes = Math.round(ms / 60_000);
231
+ if (minutes < 60)
232
+ return `${minutes} min`;
233
+ const hours = Math.round(minutes / 60);
234
+ return hours < 48 ? `${hours} h` : `${Math.round(hours / 24)} days`;
235
+ }
@@ -8,6 +8,19 @@ export declare const FOLLOW_UP_LOG = "follow-up check log v1";
8
8
  export declare const FOLLOW_UP_RECEIPT = "follow-up check receipt v1";
9
9
  export declare const ADD_TESTS_ACTION = "tests task filed";
10
10
  export type RunLevel = Exclude<CheckLevel, "off">;
11
+ /** How a batch check (batch-checks.ts) reached this result's outcome. `together`: one check on a temporary
12
+ * merge of `members`; `split`: its batch failed, so it was checked on its own; `conflict`: it couldn't be merged
13
+ * with the others, so it was checked on its own; `alone`: nothing joined it within the window. */
14
+ export type BatchFacts = {
15
+ mode: "together" | "split" | "conflict" | "alone";
16
+ tested: string;
17
+ base: string | null;
18
+ members: {
19
+ task: string;
20
+ run: number;
21
+ head: string;
22
+ }[];
23
+ };
11
24
  export type FollowUpCheck = {
12
25
  request: number;
13
26
  runId: number;
@@ -15,13 +28,19 @@ export type FollowUpCheck = {
15
28
  head: string;
16
29
  actor: string;
17
30
  at: string;
18
- why: "person" | "pull-request";
31
+ why: "person" | "pull-request" | "batch";
19
32
  /** The approved command's digest when checks were asked for: only that exact command runs. */
20
33
  digest: string;
21
34
  state: "waiting" | "running" | "passed" | "failed" | "not-run";
22
35
  exitCode: number | null;
23
36
  logArtifactId: number | null;
24
37
  note: string | null;
38
+ /** A batch request: the project base it is merged onto. */
39
+ base?: string | null;
40
+ /** The commit the check actually ran on: the result's own, or a temporary batch commit. Null until it ran. */
41
+ tested?: string | null;
42
+ /** From the sealed receipt of a batch check. */
43
+ batch?: BatchFacts | null;
25
44
  };
26
45
  type Refusal = {
27
46
  ok: false;
@@ -43,8 +62,9 @@ export declare function requestFollowUpChecks(store: Store, input: {
43
62
  level: RunLevel;
44
63
  existing: boolean;
45
64
  } | Refusal;
46
- /** Requests nobody has started yet, oldest first. */
65
+ /** Requests nobody has started yet, oldest first. Batch requests wait for the batch pass (batch-checks.ts). */
47
66
  export declare function waitingCheckRequests(store: Store, now: Date): FollowUpCheck[];
67
+ export declare function waitingRequestsOfAnyKind(store: Store, now: Date): FollowUpCheck[];
48
68
  export type CheckRunDeps = {
49
69
  runner?: Runner;
50
70
  git?: Runner;