werkmeister-core 0.95.2 → 0.95.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -1,5 +1,20 @@
1
1
  # sandbox-core
2
2
 
3
+ ## 0.95.3
4
+
5
+ ### Patch Changes
6
+
7
+ - f6bfd16: When werkmeister stops a workspace on its own (preempted, or idle), it records why in `.werkmeister/last-stop.json`, and the "Capacity freed" nudge that brings the session back says who stopped it and which commands that killed (SW-702).
8
+ - 27d4ef6: A feature with a `docker exec` running in its containers (a session's test run, not only the gate's) is busy: preemption never picks it and the idle reaper never stops it. `budget status` shows it as `run in flight`. When werkmeister does stop a workspace on its own (preempted, or idle), its session's nudge names each command the stop killed and says to rerun it (SW-856).
9
+
10
+ `with-fixture-workspace --running` takes an optional 6th column, the running `docker exec`s, so an exercise can show a busy holder.
11
+
12
+ - 0270f6f: An allocate inside a workspace container no longer prunes the other workspaces' leases: it sees only its own folder, so every other lease looked gone. Leases are pruned from the host only (SW-860).
13
+ - 8977cdb: A start that fails in compose now says `werkmeister feature logs <feature>` shows why, not "see the compose output above": when the grant starts a waiter, that output goes to another process, and the session's nudge carried the dead pointer (SW-861).
14
+ - 740bdaa: `feature up` leaves the waiting queue and clears `waiting.json` once its containers start. An admit that returned early (Docker unreadable, or a zero-cost start) left both, so the hook kept saying "queued" over a running feature (SW-863).
15
+ - ddc7d5f: The services `pr gate` and `pr open` start now share one 30-minute wait for room in the budget, keeping their queue place, instead of 10 minutes each that ended by dropping the place (SW-869).
16
+ - 7c27198: The grant pass no longer stops a parked `--service` waiter's own running containers: with no wait it could not park again, so the workspace was left with nothing running and no queue place. Any other way a parked waiter leaves the queue now tells its session why (SW-873).
17
+
3
18
  ## 0.95.2
4
19
 
5
20
  ### Patch Changes
@@ -122,6 +122,15 @@ export declare function readWaiting(root: string): WaitingFile | null;
122
122
  export declare function liveWaiting(root: string): WaitingFile | null;
123
123
  export declare function writeWaiting(root: string, w: WaitingFile): void;
124
124
  export declare function clearWaiting(root: string): void;
125
+ /** Why werkmeister last stopped this workspace on its own (SW-702): preemption or the idle reaper. */
126
+ export interface LastStop {
127
+ at: string;
128
+ why: string;
129
+ killed: string[];
130
+ }
131
+ export declare function writeLastStop(root: string, stop: LastStop): void;
132
+ /** Read and remove the last stop, so one grant reports it once. */
133
+ export declare function claimLastStop(root: string): LastStop | null;
125
134
  /** The pending "continue" for a live session, left by Resume for the pane to deliver. */
126
135
  export declare function readNudge(root: string): string | null;
127
136
  export declare function writeNudge(root: string, message: string, now?: Date): void;
@@ -449,6 +449,27 @@ export function writeWaiting(root, w) {
449
449
  export function clearWaiting(root) {
450
450
  fs.rmSync(waitingPath(root), { force: true });
451
451
  }
452
+ const lastStopPath = (root) => path.join(dotDir(root), "last-stop.json");
453
+ export function writeLastStop(root, stop) {
454
+ writeAtomic(lastStopPath(root), stop);
455
+ }
456
+ /** Read and remove the last stop, so one grant reports it once. */
457
+ export function claimLastStop(root) {
458
+ try {
459
+ const o = JSON.parse(fs.readFileSync(lastStopPath(root), "utf8"));
460
+ fs.rmSync(lastStopPath(root), { force: true });
461
+ if (typeof o?.why !== "string")
462
+ return null;
463
+ return {
464
+ at: String(o.at),
465
+ why: o.why,
466
+ killed: Array.isArray(o.killed) ? o.killed.map(String) : [],
467
+ };
468
+ }
469
+ catch {
470
+ return null;
471
+ }
472
+ }
452
473
  /** The pending "continue" for a live session, left by Resume for the pane to deliver. */
453
474
  export function readNudge(root) {
454
475
  try {
package/dist/docker.d.ts CHANGED
@@ -180,6 +180,11 @@ export declare function containerLogs(project: string, service: string | null, {
180
180
  * single entry is renamed to the destination's basename.
181
181
  */
182
182
  export declare function copyFileToContainer(container: string, hostPath: string, containerPath: string): Promise<void>;
183
+ /**
184
+ * The command line of each `docker exec` running in *names* now, for telling a session what a
185
+ * stop killed (SW-856). Empty when none run or the daemon can't be read: a stop never waits on it.
186
+ */
187
+ export declare function runningExecCommands(names: string[]): Promise<string[]>;
183
188
  export interface ExecResult {
184
189
  exitCode: number;
185
190
  stdout: string;
package/dist/docker.js CHANGED
@@ -531,6 +531,27 @@ export async function copyFileToContainer(container, hostPath, containerPath) {
531
531
  .getContainer(container)
532
532
  .putArchive(pack, { path: path.dirname(containerPath) }));
533
533
  }
534
+ /**
535
+ * The command line of each `docker exec` running in *names* now, for telling a session what a
536
+ * stop killed (SW-856). Empty when none run or the daemon can't be read: a stop never waits on it.
537
+ */
538
+ export async function runningExecCommands(names) {
539
+ const out = [];
540
+ for (const name of names) {
541
+ try {
542
+ const { ExecIDs } = await docker().getContainer(name).inspect();
543
+ for (const id of ExecIDs ?? []) {
544
+ const e = await docker().getExec(id).inspect();
545
+ if (e.Running)
546
+ out.push([e.ProcessConfig.entrypoint, ...e.ProcessConfig.arguments].join(" "));
547
+ }
548
+ }
549
+ catch {
550
+ // Gone already, or no daemon: nothing to name.
551
+ }
552
+ }
553
+ return out;
554
+ }
534
555
  /**
535
556
  * Run a command inside a running container and capture what it wrote.
536
557
  *
@@ -26,7 +26,7 @@ export interface FakeWorld {
26
26
  rules?: unknown[];
27
27
  /** Containers the fake docker shows as running, to the gate's `ps --format {{.Names}}…` only. */
28
28
  running?: string[];
29
- /** `docker inspect` lines (`feature\tworkdir\tstartedAt\tmemory\tnanoCpus`) the scheduler reads as running features (SW-685). */
29
+ /** `docker inspect` lines (`feature\tworkdir\tstartedAt\tmemory\tnanoCpus\texecs`) the scheduler reads as running features (SW-685). */
30
30
  containers?: string[];
31
31
  /** Set: `docker exec` runs its last argument (the gate row's command) with `sh -c` here and exits with its code. Absent: exit 0. */
32
32
  execCwd?: string;
@@ -71,6 +71,8 @@ export interface FixtureContainer {
71
71
  /** Bytes reserved. */
72
72
  memory: number;
73
73
  nanoCpus: number;
74
+ /** `docker exec`s running in it (SW-856): the scheduler treats any as work in flight. */
75
+ execs?: number;
74
76
  }
75
77
  export interface FixturePresets {
76
78
  /** An approved PR for a one-file change in packages/lib, with `demo-lib` running, for `pr gate`. */
@@ -88,9 +88,9 @@ export function fixturePreset({ gate, bare, change, ...opts }) {
88
88
  ? c.workspace
89
89
  : Number.isNaN(Date.parse(c.startedAt))
90
90
  ? c.startedAt
91
- : [c.memory, c.nanoCpus].find((n) => !Number.isSafeInteger(n) || n < 0);
91
+ : [c.memory, c.nanoCpus, c.execs ?? 0].find((n) => !Number.isSafeInteger(n) || n < 0);
92
92
  if (bad !== undefined)
93
- throw new Error(`--running: bad value ${JSON.stringify(bad)}: feature and workspace are lowercase names, startedAt a date, memory and nanoCpus whole numbers`);
93
+ throw new Error(`--running: bad value ${JSON.stringify(bad)}: feature and workspace are lowercase names, startedAt a date, memory, nanoCpus and execs whole numbers`);
94
94
  }
95
95
  const badEntry = opts.queue?.find((q) => !QUEUE_ENTRY.test(q));
96
96
  if (badEntry !== undefined)
@@ -267,7 +267,7 @@ ${githubProject(repos.map((r) => r.name))}${opts.config ? YAML.stringify(opts.co
267
267
  const containers = (opts.containers ?? []).map((c) => {
268
268
  const ws = path.join(root, "ws", c.workspace);
269
269
  fs.mkdirSync(ws, { recursive: true });
270
- return [c.feature, ws, c.startedAt, c.memory, c.nanoCpus].join("\t");
270
+ return [c.feature, ws, c.startedAt, c.memory, c.nanoCpus, c.execs ?? 0].join("\t");
271
271
  });
272
272
  fs.mkdirSync(home, { recursive: true });
273
273
  const fakes = writeFakes(path.join(root, "bin"), {
@@ -110,6 +110,8 @@ export declare function startService(feature: string, workspace: string, service
110
110
  grant?: boolean;
111
111
  waitMs?: number;
112
112
  }): Promise<void>;
113
+ /** One deadline for all of a `pr gate` / `pr open` run's service starts (SW-869): giving up drops the queue place, so wait longer in place rather than ask for a rerun. */
114
+ export declare const GATE_CAPACITY_WAIT_MS: number;
113
115
  /**
114
116
  * Admit starting *service* (and what compose starts with it) against the budget,
115
117
  * waiting in the queue up to 10 min (SW-540): since SW-342 these starts are the
@@ -128,7 +130,7 @@ export declare function admitServiceStart(feature: string, workspace: string, se
128
130
  grant?: boolean;
129
131
  }): Promise<void>;
130
132
  export declare const RANK_WORD: Record<Rank, string>;
131
- /** A running lease for a person to read: `name (3 CPUs, 0.8 GiB, idle, gate running)`. */
133
+ /** A running lease for a person to read: `name (3 CPUs, 0.8 GiB, idle, run in flight)`. */
132
134
  export declare function describeLease(l: Lease): string;
133
135
  /**
134
136
  * Who holds the budget, the head *feature* waits behind, and what happens next: the same lines for a
@@ -200,6 +202,8 @@ export interface RunningFeature {
200
202
  memory: number;
201
203
  /** Σ `HostConfig.NanoCpus`, cores. */
202
204
  cpus: number;
205
+ /** `docker exec`s running in its containers now (`.ExecIDs`: Docker drops an exec when it exits). */
206
+ execs: number;
203
207
  }
204
208
  export interface IdleInput {
205
209
  minutes: number;
@@ -226,7 +230,9 @@ export interface StopIdleOptions {
226
230
  only?: string;
227
231
  now?: number;
228
232
  run?: (cmd: string[]) => RunResult;
229
- stop?: (feature: string) => Promise<unknown>;
233
+ stop?: (feature: string, opts: {
234
+ why: string;
235
+ }) => Promise<unknown>;
230
236
  /** Override for whether anyone waits for capacity; default: the queue file. */
231
237
  queued?: boolean;
232
238
  }
@@ -273,6 +279,11 @@ export declare function admitOrQueue(feature: string, workspace: string, byPerso
273
279
  export declare function grantWaiters(): Promise<string[]>;
274
280
  /** The daemon's side of a client's release (SW-185 D3): start waiters without holding the caller. */
275
281
  export declare function grantInBackground(): number;
282
+ /**
283
+ * What a session hears when werkmeister stopped its containers under these commands (SW-856):
284
+ * they died with them (exit 137), so the session reruns them rather than reading a test failure.
285
+ */
286
+ export declare function killedNudge(why: string, killed: string[]): string;
276
287
  /**
277
288
  * Stop *feature*'s containers, leaving them and its slot in place so
278
289
  * `restart`/`up` can resume it.
@@ -280,8 +291,9 @@ export declare function grantInBackground(): number;
280
291
  * Throws `NOT_RUNNING` when the feature holds no slot, and
281
292
  * `COMPOSE_STOP_FAILED` when a container fails to stop.
282
293
  */
283
- export declare function stop(feature: string, { grant }?: {
294
+ export declare function stop(feature: string, { grant, why }?: {
284
295
  grant?: boolean;
296
+ why?: string;
285
297
  }): Promise<StopResult>;
286
298
  /**
287
299
  * Restart *feature*'s containers in place.
package/dist/lifecycle.js CHANGED
@@ -26,7 +26,7 @@
26
26
  import * as fs from "node:fs";
27
27
  import * as path from "node:path";
28
28
  import { setTimeout as delay } from "node:timers/promises";
29
- import { agentOf, clearPaused, clearWaiting, nudgePath, readPaused, readSessions, readWaiting, STALE_SESSION_MS, writeNudge, writePaused, writeWaiting, } from "./agent_state.js";
29
+ import { agentOf, claimLastStop, clearPaused, clearWaiting, nudgePath, readPaused, readSessions, readWaiting, STALE_SESSION_MS, writeLastStop, writeNudge, writePaused, writeWaiting, } from "./agent_state.js";
30
30
  import { ensureSharedNetwork, isCaddyRunning, reloadCaddy, startCaddy } from "./caddy.js";
31
31
  import { Colors } from "./colors.js";
32
32
  import {
@@ -35,7 +35,7 @@ import {
35
35
  composeDown, composeDownAsync, composeRestart, composeStop, composeUp, composeUpAsync, composeUpServiceAsync, dependencyOrder, getContainerStatus, isOneShotService, liveProject, workspaceForFeature, } from "./compose.js";
36
36
  import { COMPOSE_FILE, migrateWorkspaceCompose } from "./compose_file.js";
37
37
  import { getVolumeBaseNames, loadPorts, loadServices, needsCluster, sharedProxySkipReason, } from "./config.js";
38
- import { docker, listVolumeNames, projectContainers, removeVolume } from "./docker.js";
38
+ import { docker, listVolumeNames, projectContainers, removeVolume, runningExecCommands, } from "./docker.js";
39
39
  import { dotDir } from "./dot_dir.js";
40
40
  import { SandboxError } from "./errors.js";
41
41
  import { ingressHostnames, loadFeatureRouters, warnUnresolvedHosts, } from "./feature_routing.js";
@@ -187,9 +187,17 @@ export async function up(feature, workspace, { signal, byPerson } = {}) {
187
187
  requireSharedNetwork();
188
188
  info("Starting containers...");
189
189
  if (!(await withStartRecord(workspace, "up", () => composeUpAsync(feature, signal)))) {
190
- throw new SandboxError("Failed to start containers. Check docker compose output above.", "COMPOSE_UP_FAILED");
190
+ throw new SandboxError(
191
+ // Not "above": a grant runs this in another process, and its nudge carries this text (SW-861).
192
+ `Failed to start containers: \`werkmeister feature logs ${feature}\` shows why.`, "COMPOSE_UP_FAILED");
191
193
  }
192
194
  containersStarted = true;
195
+ // Up now: two admits return before the one that dequeues (Docker unreadable, a zero-cost start), and
196
+ // a leftover entry keeps the hook saying "queued" over running containers (SW-863), as in restart().
197
+ await dequeue(feature);
198
+ clearWaiting(workspace);
199
+ // Its stop is answered by this start; a later grant must not report it (SW-702). The grant claims it first.
200
+ claimLastStop(workspace);
193
201
  }
194
202
  else {
195
203
  info("No local services to start -- routing entirely to k8s");
@@ -326,11 +334,13 @@ export async function startService(feature, workspace, service, composeName = CO
326
334
  if (!started) {
327
335
  // A timed-out or cancelled up leaves the container Created, never started (SW-86).
328
336
  const state = (await getContainerStatus(feature)).find((c) => c.service === service)?.status;
329
- throw new SandboxError(`Failed to start ${service}${state ? ` (left ${state})` : ""} — see the compose output above`, "COMPOSE_UP_FAILED");
337
+ throw new SandboxError(`Failed to start ${service}${state ? ` (left ${state})` : ""}: \`werkmeister feature logs ${feature} ${service}\` shows why`, "COMPOSE_UP_FAILED");
330
338
  }
331
339
  }
332
340
  const SERVICE_POLL_MS = 15_000;
333
341
  const SERVICE_WAIT_MS = 10 * 60_000;
342
+ /** One deadline for all of a `pr gate` / `pr open` run's service starts (SW-869): giving up drops the queue place, so wait longer in place rather than ask for a rerun. */
343
+ export const GATE_CAPACITY_WAIT_MS = 30 * 60_000;
334
344
  /**
335
345
  * Admit starting *service* (and what compose starts with it) against the budget,
336
346
  * waiting in the queue up to 10 min (SW-540): since SW-342 these starts are the
@@ -360,7 +370,12 @@ export async function admitServiceStart(feature, workspace, service, { sleep = d
360
370
  // Never byPerson: an agent's gate in a person's workspace must not stop another run's containers.
361
371
  let position = await admitOrQueue(feature, workspace, false, starting);
362
372
  // Not in a person's workspace: their running app must not go down because they asked for one more service.
363
- if (position !== null && ownRunning.length && !released.length && readAutonomous(workspace)) {
373
+ // Nor in the grant pass (SW-873): with no wait it would stop its own set and drop it, never parked.
374
+ if (position !== null &&
375
+ grant &&
376
+ ownRunning.length &&
377
+ !released.length &&
378
+ readAutonomous(workspace)) {
364
379
  // Release instead of hold (SW-675): two gates each holding core while waiting for one more
365
380
  // service deadlock a full budget. Stopped, not removed: one request then brings back the set.
366
381
  await stop(feature, { grant: false });
@@ -390,7 +405,7 @@ export async function admitServiceStart(feature, workspace, service, { sleep = d
390
405
  writeWaiting(workspace, { ...waiting, parked: { service } });
391
406
  parked = true;
392
407
  }
393
- throw new SandboxError(`No capacity to start ${services.join(", ")} for '${feature}' after ${waitMs / 60_000} min ` +
408
+ throw new SandboxError(`No capacity to start ${services.join(", ")} for '${feature}' after ${Math.round(waitMs / 60_000)} min ` +
394
409
  `(queue position ${position})${parked ? "; it stays queued" : ""}.` +
395
410
  (released.length
396
411
  ? ` ${[...up].join(", ")} stayed stopped; \`werkmeister feature up\` starts the feature again.`
@@ -400,7 +415,7 @@ export async function admitServiceStart(feature, workspace, service, { sleep = d
400
415
  if (!waited) {
401
416
  warn(
402
417
  // SW-734: say how a wait the person doesn't want ends.
403
- `Waiting for capacity to start ${services.join(", ")}: position ${position}, up to ${waitMs / 60_000} min. ` +
418
+ `Waiting for capacity to start ${services.join(", ")}: position ${position}, up to ${Math.round(waitMs / 60_000)} min. ` +
404
419
  "Stopping the command ends the wait; `pr gate` and `pr open` start no service the PR file's `skip_tests` names.");
405
420
  }
406
421
  await sleep(SERVICE_POLL_MS);
@@ -429,9 +444,9 @@ export const RANK_WORD = {
429
444
  working: "working",
430
445
  person: "a person's workspace",
431
446
  };
432
- /** A running lease for a person to read: `name (3 CPUs, 0.8 GiB, idle, gate running)`. */
447
+ /** A running lease for a person to read: `name (3 CPUs, 0.8 GiB, idle, run in flight)`. */
433
448
  export function describeLease(l) {
434
- return `${l.feature} (${cpus(l.cost.cpus)}, ${gib(l.cost.memory)}, ${RANK_WORD[l.rank]}${l.busy ? ", gate running" : ""})`;
449
+ return `${l.feature} (${cpus(l.cost.cpus)}, ${gib(l.cost.memory)}, ${RANK_WORD[l.rank]}${l.busy ? ", run in flight" : ""})`;
435
450
  }
436
451
  /**
437
452
  * Who holds the budget, the head *feature* waits behind, and what happens next: the same lines for a
@@ -587,6 +602,9 @@ export function otherLiveSessions(workspaceRoot, sessionId) {
587
602
  }
588
603
  /** Why a running feature is not idle, or null when it is. */
589
604
  function busyReason(f, input, cutoff) {
605
+ // A test run or probe in its container is work in flight, whatever the session record says (SW-856).
606
+ if (f.execs)
607
+ return "docker exec running";
590
608
  // An open session that has sat idle past the cutoff does not count (SW-39 D3), nor, while others
591
609
  // queue, one that has waited on a person that long: its status stays busy through the ask (SW-675).
592
610
  const recent = (s) => (s.statusUpdatedAt ?? Infinity) > cutoff;
@@ -641,19 +659,27 @@ export function runningFeatures(runCmd = (cmd) => run(cmd, { capture: true, time
641
659
  if (!list.size)
642
660
  return [];
643
661
  const feature = `or (index .Config.Labels "${LABEL_FEATURE}") (index .Config.Labels "${LEGACY_LABEL_FEATURE}")`;
644
- const fmt = `{{${feature}}}\t{{index .Config.Labels "${LABEL_WORKDIR}"}}\t{{.State.StartedAt}}\t{{.HostConfig.MemoryReservation}}\t{{.HostConfig.NanoCpus}}`;
662
+ const fmt = `{{${feature}}}\t{{index .Config.Labels "${LABEL_WORKDIR}"}}\t{{.State.StartedAt}}\t{{.HostConfig.MemoryReservation}}\t{{.HostConfig.NanoCpus}}\t{{len .ExecIDs}}`;
645
663
  const inspect = runCmd(["docker", "inspect", "--format", fmt, ...list]);
646
664
  if (inspect.status !== 0)
647
665
  return null;
648
666
  const byFeature = new Map();
649
667
  for (const line of inspect.stdout.split("\n").filter(Boolean)) {
650
- const [feature, workspace, started, memory, nanoCpus] = line.split("\t");
668
+ const [feature, workspace, started, memory, nanoCpus, execs] = line.split("\t");
651
669
  if (!feature || !workspace)
652
670
  continue;
653
- const f = byFeature.get(feature) ?? { feature, workspace, startedAt: [], memory: 0, cpus: 0 };
671
+ const f = byFeature.get(feature) ?? {
672
+ feature,
673
+ workspace,
674
+ startedAt: [],
675
+ memory: 0,
676
+ cpus: 0,
677
+ execs: 0,
678
+ };
654
679
  f.startedAt.push(Date.parse(started));
655
680
  f.memory += Number(memory) || 0;
656
681
  f.cpus += (Number(nanoCpus) || 0) / 1e9;
682
+ f.execs += Number(execs) || 0;
657
683
  byFeature.set(feature, f);
658
684
  }
659
685
  return [...byFeature.values()];
@@ -699,7 +725,7 @@ export async function stopIdle(minutes, opts = {}) {
699
725
  }
700
726
  const asking = input.sessions(f.workspace).some((s) => s.asking);
701
727
  try {
702
- await (opts.stop ?? stop)(f.feature);
728
+ await (opts.stop ?? stop)(f.feature, { why: `idle ≥ ${minutes} min` });
703
729
  lines.push(asking
704
730
  ? `${f.feature}: stopped (waiting on a person ≥ ${minutes} min, others queued)`
705
731
  : `${f.feature}: stopped (idle ≥ ${minutes} min)`);
@@ -733,11 +759,19 @@ const GRANTED_NUDGE = "Capacity freed: this workspace's containers are up. Conti
733
759
  * The granted session's nudge. Preemption stops on-demand containers too, and the grant's `up` starts
734
760
  * only the default set, so name each one left stopped and how to start it (SW-840).
735
761
  */
736
- async function grantedNudge(feature) {
762
+ async function grantedNudge(feature, last) {
737
763
  const stopped = (await getContainerStatus(feature)).filter((c) => c.status !== "running" && !isOneShotService(c.service, c.role));
764
+ // Say why it was down at all (SW-702): nothing else tells the session who stopped it.
765
+ const why = last
766
+ ? `Werkmeister had stopped it (${last.why})` +
767
+ (last.killed.length
768
+ ? `, killing ${last.killed.map((c) => `\`${c}\``).join(", ")}: rerun them`
769
+ : "") +
770
+ ". "
771
+ : "";
738
772
  if (!stopped.length)
739
- return GRANTED_NUDGE;
740
- return (`Capacity freed: this workspace started, but these containers are still stopped: ${stopped.map((c) => c.name).join(", ")}. ` +
773
+ return why + GRANTED_NUDGE;
774
+ return (`${why}Capacity freed: this workspace started, but these containers are still stopped: ${stopped.map((c) => c.name).join(", ")}. ` +
741
775
  `Start them with ${stopped.map((c) => `\`werkmeister feature up --service ${c.service}\``).join(", ")}, ` +
742
776
  "then continue where you left off.");
743
777
  }
@@ -768,7 +802,9 @@ export function runningLeases(runCmd) {
768
802
  : state?.agent === "working"
769
803
  ? "working"
770
804
  : "idle";
771
- const busy = readPaused(f.workspace) !== null ||
805
+ // A running `docker exec` (a session's test run, not only the gate's) is never stopped (SW-856).
806
+ const busy = f.execs > 0 ||
807
+ readPaused(f.workspace) !== null ||
772
808
  (mtimeMs(lastExecMarker(f.workspace)) ?? 0) > gateCutoff;
773
809
  const lease = {
774
810
  feature: f.feature,
@@ -817,7 +853,7 @@ async function admitOrQueueHere(feature, workspace, byPerson, starting) {
817
853
  const victimWs = readLeases()[victim]?.workspace || workspaceForFeature(victim);
818
854
  try {
819
855
  // stop, not pause: a paused container keeps its memory.
820
- await stop(victim, { grant: false });
856
+ await stop(victim, { grant: false, why: `preempted to make room for '${feature}'` });
821
857
  }
822
858
  catch (e) {
823
859
  // A failed stop can still have killed it (SW-738): queue it to come back, and wait rather than
@@ -867,46 +903,57 @@ async function grantWaitersNow() {
867
903
  // A lease pruned by a process that can't see the folder must not cost the waiter its start (SW-848).
868
904
  const workspace = readLeases()[feature]?.workspace || workspaceForFeature(feature);
869
905
  if (!workspace) {
906
+ // No folder to nudge in: say it here, at least (SW-873).
907
+ warn(`'${feature}' left the waiting queue: its workspace can't be found`);
870
908
  await dequeue(feature);
871
909
  continue;
872
910
  }
873
- const waiting = readWaiting(workspace);
874
- if (waiting?.parked) {
875
- if (await startParked(feature, workspace, waiting.parked.service)) {
876
- started.push(feature);
877
- if (liveSessions(workspace).length)
878
- writeNudge(workspace, await grantedNudge(feature));
879
- }
880
- continue;
881
- }
882
- // A service start's own poll admits it; `up` would start the wrong set and dequeue it.
883
- if (waiting?.services)
884
- continue;
885
- let r;
911
+ // Before up(), which drops the record on any start, so only this grant's nudge reports it (SW-702).
912
+ const last = claimLastStop(workspace);
886
913
  try {
887
- r = await up(feature, workspace, { byPerson: false });
888
- }
889
- catch (e) {
890
- // One waiter's failed re-up (HOST_LOAD, NO_COMPOSE, ...) must not end the pass for the rest (SW-737).
891
- const message = `Could not start '${feature}' from the waiting queue: ${e.message}`;
892
- warn(message);
893
- // Admission already dropped it (never fits, or a start that failed after admit): only its session can act (SW-682, SW-848).
894
- if (!readQueue().includes(feature)) {
895
- clearWaiting(workspace);
896
- // A never-fits rerun would fail the same way; its message already says what to change.
897
- const neverFits = e instanceof SandboxError && e.code === "NEVER_FITS";
898
- writeNudge(workspace, neverFits
899
- ? message
900
- : `${message}\nIt left the queue: run \`werkmeister feature up\` to queue it again.`);
914
+ const waiting = readWaiting(workspace);
915
+ if (waiting?.parked) {
916
+ if (await startParked(feature, workspace, waiting.parked.service)) {
917
+ started.push(feature);
918
+ if (liveSessions(workspace).length)
919
+ writeNudge(workspace, await grantedNudge(feature, last));
920
+ }
921
+ continue;
901
922
  }
902
- continue;
923
+ // A service start's own poll admits it; `up` would start the wrong set and dequeue it.
924
+ if (waiting?.services)
925
+ continue;
926
+ let r;
927
+ try {
928
+ r = await up(feature, workspace, { byPerson: false });
929
+ }
930
+ catch (e) {
931
+ // One waiter's failed re-up (HOST_LOAD, NO_COMPOSE, ...) must not end the pass for the rest (SW-737).
932
+ const message = `Could not start '${feature}' from the waiting queue: ${e.message}`;
933
+ warn(message);
934
+ // Admission already dropped it (never fits, or a start that failed after admit): only its session can act (SW-682, SW-848).
935
+ if (!readQueue().includes(feature)) {
936
+ clearWaiting(workspace);
937
+ // A never-fits rerun would fail the same way; its message already says what to change.
938
+ const neverFits = e instanceof SandboxError && e.code === "NEVER_FITS";
939
+ writeNudge(workspace, neverFits
940
+ ? message
941
+ : `${message}\nIt left the queue: run \`werkmeister feature up\` to queue it again.`);
942
+ }
943
+ continue;
944
+ }
945
+ // Backfill (SW-675): a later waiter may fit; admission itself holds the room for a starved head.
946
+ if (r.waiting !== undefined)
947
+ continue;
948
+ started.push(feature);
949
+ if (liveSessions(workspace).length)
950
+ writeNudge(workspace, await grantedNudge(feature, last));
951
+ }
952
+ finally {
953
+ // Still waiting: the grant that starts it reports the stop.
954
+ if (last && !started.includes(feature) && readQueue().includes(feature))
955
+ writeLastStop(workspace, last);
903
956
  }
904
- // Backfill (SW-675): a later waiter may fit; admission itself holds the room for a starved head.
905
- if (r.waiting !== undefined)
906
- continue;
907
- started.push(feature);
908
- if (liveSessions(workspace).length)
909
- writeNudge(workspace, await grantedNudge(feature));
910
957
  }
911
958
  return started;
912
959
  }
@@ -923,7 +970,9 @@ async function startParked(feature, workspace, service) {
923
970
  });
924
971
  }
925
972
  catch (e) {
926
- if (!(e instanceof SandboxError && e.code === "NO_CAPACITY")) {
973
+ // No room keeps it parked; any other way out of the queue its session must hear (SW-873).
974
+ if (!(e instanceof SandboxError && e.code === "NO_CAPACITY") ||
975
+ !readQueue().includes(feature)) {
927
976
  const message = `Could not start '${feature}' ${service} from the waiting queue: ${e.message}`;
928
977
  warn(message);
929
978
  // Its session was told it continues once started; it must hear that it won't (SW-701 review).
@@ -955,6 +1004,15 @@ async function grantWaitersQuietly() {
955
1004
  warn(`Could not start the next waiting feature: ${e instanceof Error ? e.message : String(e)}`);
956
1005
  }
957
1006
  }
1007
+ /**
1008
+ * What a session hears when werkmeister stopped its containers under these commands (SW-856):
1009
+ * they died with them (exit 137), so the session reruns them rather than reading a test failure.
1010
+ */
1011
+ export function killedNudge(why, killed) {
1012
+ return (`Werkmeister stopped this workspace's containers (${why}) while these ran in them: ` +
1013
+ `${killed.map((c) => `\`${c}\``).join(", ")}. They were killed (exit 137), not failed: start the ` +
1014
+ "containers with `werkmeister feature up` (`--service <name>` for an on-demand one), then rerun them.");
1015
+ }
958
1016
  /**
959
1017
  * Stop *feature*'s containers, leaving them and its slot in place so
960
1018
  * `restart`/`up` can resume it.
@@ -962,9 +1020,26 @@ async function grantWaitersQuietly() {
962
1020
  * Throws `NOT_RUNNING` when the feature holds no slot, and
963
1021
  * `COMPOSE_STOP_FAILED` when a container fails to stop.
964
1022
  */
965
- export async function stop(feature, { grant = true } = {}) {
1023
+ export async function stop(feature, { grant = true, why } = {}) {
966
1024
  refuseInsideOwnContainer(feature, "stop");
967
1025
  const slot = await requireSlot(feature);
1026
+ // *why* marks a stop nobody in the workspace asked for: name what it kills to the session (SW-856).
1027
+ const workspace = why ? workspaceForFeature(feature) : null;
1028
+ if (why && workspace) {
1029
+ const names = (await getContainerStatus(feature))
1030
+ .filter((c) => c.status === "running")
1031
+ .map((c) => c.name);
1032
+ const killed = await runningExecCommands(names);
1033
+ // Best effort: a folder gone, or unseen from a container (SW-860), must not block the stop.
1034
+ try {
1035
+ writeLastStop(workspace, { at: new Date().toISOString(), why, killed });
1036
+ if (killed.length)
1037
+ writeNudge(workspace, killedNudge(why, killed));
1038
+ }
1039
+ catch (e) {
1040
+ warn(`Could not tell '${feature}' why it stopped: ${e.message}`);
1041
+ }
1042
+ }
968
1043
  info(`Stopping containers for '${feature}' (slot ${slot})...`);
969
1044
  if (!(await composeStop(feature))) {
970
1045
  throw new SandboxError(`Failed to stop containers for '${feature}'`, "COMPOSE_STOP_FAILED");
@@ -19,7 +19,7 @@ import { GATE_COMPOSE_FILE, renderGateCompose } from "./generate.js";
19
19
  import { updateNoteStatus } from "./graduate.js";
20
20
  import { eachIssue, issueStatuses, moveIssue } from "./issue_status.js";
21
21
  import { groupOf, issueIdOf, issueRefsOf, issueTracker, projectOf } from "./issue_tracker.js";
22
- import { grantWaiters, idleStopMinutes, lastExecMarker, liveSessions, readServicesFromWorkspace, runningFeatures, startService, stop as stopFeature, stopIdle, } from "./lifecycle.js";
22
+ import { GATE_CAPACITY_WAIT_MS, grantWaiters, idleStopMinutes, lastExecMarker, liveSessions, readServicesFromWorkspace, runningFeatures, startService, stop as stopFeature, stopIdle, } from "./lifecycle.js";
23
23
  import { originSlug, prWatchScheduleState, } from "./machine_setup.js";
24
24
  import { openDecisions } from "./notes.js";
25
25
  import { prRulesPath } from "./pr_guard.js";
@@ -58,6 +58,8 @@ async function httpStatus(url) {
58
58
  }
59
59
  }
60
60
  export function defaultDeps() {
61
+ // Built once per command, so the wait is a total across its starts, not per start (SW-869).
62
+ const capacityDeadline = Date.now() + GATE_CAPACITY_WAIT_MS;
61
63
  return {
62
64
  run: execRun,
63
65
  httpStatus,
@@ -100,7 +102,10 @@ export function defaultDeps() {
100
102
  const granted = await grantWaiters().catch((e) => [`waiting queue: ${e.message}`]);
101
103
  return { lines: [...lines, ...granted.map((f) => `${f}: started from the waiting queue`)] };
102
104
  },
103
- startService: (ws, feature, service, composeName) => startService(feature, ws, service, composeName, { skipCurrentSetups: true }),
105
+ startService: (ws, feature, service, composeName) => startService(feature, ws, service, composeName, {
106
+ skipCurrentSetups: true,
107
+ waitMs: Math.max(0, capacityDeadline - Date.now()),
108
+ }),
104
109
  portHolders: (ws, feature, services) => portHolders(feature, ws, services),
105
110
  gateCompose: (ws, feature, services) => renderGateCompose(feature, Number(readEnv(ws, "WERKMEISTER_SLOT")), readServicesFromWorkspace(ws), services, ws),
106
111
  };
@@ -19,7 +19,7 @@ export interface Lease {
19
19
  feature: string;
20
20
  cost: Cost;
21
21
  rank: Rank;
22
- /** Mid-gate or paused: stopping it would break a run in flight. */
22
+ /** A `docker exec` running in it, mid-gate or paused: stopping it would break a run in flight. */
23
23
  busy: boolean;
24
24
  }
25
25
  export interface Request {
package/dist/slots.d.ts CHANGED
@@ -53,6 +53,8 @@ export declare function getAllSlotsFromDocker(): Promise<Record<string, SlotEntr
53
53
  /** Reads Docker labels; injectable so tests need no daemon. */
54
54
  export interface SlotDeps {
55
55
  labels: () => Promise<Record<string, SlotEntry> | null>;
56
+ /** Docker's marker file, for tests; default `/.dockerenv`. */
57
+ dockerenv?: string;
56
58
  }
57
59
  /**
58
60
  * Allocate *feature*'s slot and lease it before the lock is released.
package/dist/slots.js CHANGED
@@ -25,6 +25,7 @@ import { workspaceForFeature } from "./compose.js";
25
25
  import { baselineUpstream, loadPorts, loadServices, sandboxHome, slotPortRange } from "./config.js";
26
26
  import { docker } from "./docker.js";
27
27
  import { SandboxError } from "./errors.js";
28
+ import { insideContainer } from "./global_config.js";
28
29
  import { sandboxLock } from "./locks.js";
29
30
  import { error, info, success, warn } from "./reporter.js";
30
31
  import { pickSlot } from "./scheduler.js";
@@ -175,11 +176,15 @@ const defaultSlotDeps = { labels: getAllSlotsFromDocker };
175
176
  * label and a lease disagree on a slot's owner the label wins: those
176
177
  * containers are what bind its ports.
177
178
  */
178
- function heldSlots(dockerSlots) {
179
+ function heldSlots(dockerSlots, dockerenv) {
179
180
  const leases = readLeases();
180
- for (const [f, l] of Object.entries(leases)) {
181
- if (l.workspace && !fs.existsSync(l.workspace))
182
- delete leases[f];
181
+ // A container mounts the real ~/.werkmeister but only its own workspace: every other folder
182
+ // reads gone from there, so only the host prunes (SW-860).
183
+ if (!insideContainer(dockerenv)) {
184
+ for (const [f, l] of Object.entries(leases)) {
185
+ if (l.workspace && !fs.existsSync(l.workspace))
186
+ delete leases[f];
187
+ }
183
188
  }
184
189
  for (const [f, e] of Object.entries(dockerSlots ?? {})) {
185
190
  leases[f] ??= { slot: e.slot, workspace: workspaceForFeature(f) ?? "" };
@@ -210,7 +215,7 @@ export function allocateSlot(feature, preferredSlot, workspace, deps = defaultSl
210
215
  function allocateSlotHere(feature, preferredSlot, workspace, deps) {
211
216
  const count = slotCount();
212
217
  return sandboxLock("allocate", async () => {
213
- const { leases, held } = heldSlots(await deps.labels());
218
+ const { leases, held } = heldSlots(await deps.labels(), deps.dockerenv);
214
219
  const slot = pickSlot(feature, held, count, preferredSlot ?? null);
215
220
  if (slot === null) {
216
221
  error(`All ${count} feature slots are occupied!`);
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "werkmeister-core",
3
- "version": "0.95.2",
3
+ "version": "0.95.3",
4
4
  "license": "Apache-2.0",
5
5
  "type": "module",
6
6
  "description": "werkmeister's shared library. Its public API is werkmeister-core/sdk; every other entry point is internal.",