werkmeister-core 0.95.2 → 0.95.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/dist/agent_state.d.ts +9 -0
- package/dist/agent_state.js +21 -0
- package/dist/docker.d.ts +5 -0
- package/dist/docker.js +21 -0
- package/dist/fixture_fakes.d.ts +1 -1
- package/dist/fixture_workspace.d.ts +2 -0
- package/dist/fixture_workspace.js +3 -3
- package/dist/lifecycle.d.ts +15 -3
- package/dist/lifecycle.js +129 -54
- package/dist/pr_lifecycle.js +7 -2
- package/dist/scheduler.d.ts +1 -1
- package/dist/slots.d.ts +2 -0
- package/dist/slots.js +10 -5
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,20 @@
|
|
|
1
1
|
# sandbox-core
|
|
2
2
|
|
|
3
|
+
## 0.95.3
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- f6bfd16: When werkmeister stops a workspace on its own (preempted, or idle), it records why in `.werkmeister/last-stop.json`, and the "Capacity freed" nudge that brings the session back says who stopped it and which commands that killed (SW-702).
|
|
8
|
+
- 27d4ef6: A feature with a `docker exec` running in its containers (a session's test run, not only the gate's) is busy: preemption never picks it and the idle reaper never stops it. `budget status` shows it as `run in flight`. When werkmeister does stop a workspace on its own (preempted, or idle), its session's nudge names each command the stop killed and says to rerun it (SW-856).
|
|
9
|
+
|
|
10
|
+
`with-fixture-workspace --running` takes an optional 6th column, the running `docker exec`s, so an exercise can show a busy holder.
|
|
11
|
+
|
|
12
|
+
- 0270f6f: An allocate inside a workspace container no longer prunes the other workspaces' leases: it sees only its own folder, so every other lease looked gone. Leases are pruned from the host only (SW-860).
|
|
13
|
+
- 8977cdb: A start that fails in compose now says `werkmeister feature logs <feature>` shows why, not "see the compose output above": when the grant starts a waiter, that output goes to another process, and the session's nudge carried the dead pointer (SW-861).
|
|
14
|
+
- 740bdaa: `feature up` leaves the waiting queue and clears `waiting.json` once its containers start. An admit that returned early (Docker unreadable, or a zero-cost start) left both, so the hook kept saying "queued" over a running feature (SW-863).
|
|
15
|
+
- ddc7d5f: The services `pr gate` and `pr open` start now share one 30-minute wait for room in the budget, keeping their queue place, instead of 10 minutes each that ended by dropping the place (SW-869).
|
|
16
|
+
- 7c27198: The grant pass no longer stops a parked `--service` waiter's own running containers: with no wait it could not park again, so the workspace was left with nothing running and no queue place. Any other way a parked waiter leaves the queue now tells its session why (SW-873).
|
|
17
|
+
|
|
3
18
|
## 0.95.2
|
|
4
19
|
|
|
5
20
|
### Patch Changes
|
package/dist/agent_state.d.ts
CHANGED
|
@@ -122,6 +122,15 @@ export declare function readWaiting(root: string): WaitingFile | null;
|
|
|
122
122
|
export declare function liveWaiting(root: string): WaitingFile | null;
|
|
123
123
|
export declare function writeWaiting(root: string, w: WaitingFile): void;
|
|
124
124
|
export declare function clearWaiting(root: string): void;
|
|
125
|
+
/** Why werkmeister last stopped this workspace on its own (SW-702): preemption or the idle reaper. */
|
|
126
|
+
export interface LastStop {
|
|
127
|
+
at: string;
|
|
128
|
+
why: string;
|
|
129
|
+
killed: string[];
|
|
130
|
+
}
|
|
131
|
+
export declare function writeLastStop(root: string, stop: LastStop): void;
|
|
132
|
+
/** Read and remove the last stop, so one grant reports it once. */
|
|
133
|
+
export declare function claimLastStop(root: string): LastStop | null;
|
|
125
134
|
/** The pending "continue" for a live session, left by Resume for the pane to deliver. */
|
|
126
135
|
export declare function readNudge(root: string): string | null;
|
|
127
136
|
export declare function writeNudge(root: string, message: string, now?: Date): void;
|
package/dist/agent_state.js
CHANGED
|
@@ -449,6 +449,27 @@ export function writeWaiting(root, w) {
|
|
|
449
449
|
export function clearWaiting(root) {
|
|
450
450
|
fs.rmSync(waitingPath(root), { force: true });
|
|
451
451
|
}
|
|
452
|
+
const lastStopPath = (root) => path.join(dotDir(root), "last-stop.json");
|
|
453
|
+
export function writeLastStop(root, stop) {
|
|
454
|
+
writeAtomic(lastStopPath(root), stop);
|
|
455
|
+
}
|
|
456
|
+
/** Read and remove the last stop, so one grant reports it once. */
|
|
457
|
+
export function claimLastStop(root) {
|
|
458
|
+
try {
|
|
459
|
+
const o = JSON.parse(fs.readFileSync(lastStopPath(root), "utf8"));
|
|
460
|
+
fs.rmSync(lastStopPath(root), { force: true });
|
|
461
|
+
if (typeof o?.why !== "string")
|
|
462
|
+
return null;
|
|
463
|
+
return {
|
|
464
|
+
at: String(o.at),
|
|
465
|
+
why: o.why,
|
|
466
|
+
killed: Array.isArray(o.killed) ? o.killed.map(String) : [],
|
|
467
|
+
};
|
|
468
|
+
}
|
|
469
|
+
catch {
|
|
470
|
+
return null;
|
|
471
|
+
}
|
|
472
|
+
}
|
|
452
473
|
/** The pending "continue" for a live session, left by Resume for the pane to deliver. */
|
|
453
474
|
export function readNudge(root) {
|
|
454
475
|
try {
|
package/dist/docker.d.ts
CHANGED
|
@@ -180,6 +180,11 @@ export declare function containerLogs(project: string, service: string | null, {
|
|
|
180
180
|
* single entry is renamed to the destination's basename.
|
|
181
181
|
*/
|
|
182
182
|
export declare function copyFileToContainer(container: string, hostPath: string, containerPath: string): Promise<void>;
|
|
183
|
+
/**
|
|
184
|
+
* The command line of each `docker exec` running in *names* now, for telling a session what a
|
|
185
|
+
* stop killed (SW-856). Empty when none run or the daemon can't be read: a stop never waits on it.
|
|
186
|
+
*/
|
|
187
|
+
export declare function runningExecCommands(names: string[]): Promise<string[]>;
|
|
183
188
|
export interface ExecResult {
|
|
184
189
|
exitCode: number;
|
|
185
190
|
stdout: string;
|
package/dist/docker.js
CHANGED
|
@@ -531,6 +531,27 @@ export async function copyFileToContainer(container, hostPath, containerPath) {
|
|
|
531
531
|
.getContainer(container)
|
|
532
532
|
.putArchive(pack, { path: path.dirname(containerPath) }));
|
|
533
533
|
}
|
|
534
|
+
/**
|
|
535
|
+
* The command line of each `docker exec` running in *names* now, for telling a session what a
|
|
536
|
+
* stop killed (SW-856). Empty when none run or the daemon can't be read: a stop never waits on it.
|
|
537
|
+
*/
|
|
538
|
+
export async function runningExecCommands(names) {
|
|
539
|
+
const out = [];
|
|
540
|
+
for (const name of names) {
|
|
541
|
+
try {
|
|
542
|
+
const { ExecIDs } = await docker().getContainer(name).inspect();
|
|
543
|
+
for (const id of ExecIDs ?? []) {
|
|
544
|
+
const e = await docker().getExec(id).inspect();
|
|
545
|
+
if (e.Running)
|
|
546
|
+
out.push([e.ProcessConfig.entrypoint, ...e.ProcessConfig.arguments].join(" "));
|
|
547
|
+
}
|
|
548
|
+
}
|
|
549
|
+
catch {
|
|
550
|
+
// Gone already, or no daemon: nothing to name.
|
|
551
|
+
}
|
|
552
|
+
}
|
|
553
|
+
return out;
|
|
554
|
+
}
|
|
534
555
|
/**
|
|
535
556
|
* Run a command inside a running container and capture what it wrote.
|
|
536
557
|
*
|
package/dist/fixture_fakes.d.ts
CHANGED
|
@@ -26,7 +26,7 @@ export interface FakeWorld {
|
|
|
26
26
|
rules?: unknown[];
|
|
27
27
|
/** Containers the fake docker shows as running, to the gate's `ps --format {{.Names}}…` only. */
|
|
28
28
|
running?: string[];
|
|
29
|
-
/** `docker inspect` lines (`feature\tworkdir\tstartedAt\tmemory\tnanoCpus`) the scheduler reads as running features (SW-685). */
|
|
29
|
+
/** `docker inspect` lines (`feature\tworkdir\tstartedAt\tmemory\tnanoCpus\texecs`) the scheduler reads as running features (SW-685). */
|
|
30
30
|
containers?: string[];
|
|
31
31
|
/** Set: `docker exec` runs its last argument (the gate row's command) with `sh -c` here and exits with its code. Absent: exit 0. */
|
|
32
32
|
execCwd?: string;
|
|
@@ -71,6 +71,8 @@ export interface FixtureContainer {
|
|
|
71
71
|
/** Bytes reserved. */
|
|
72
72
|
memory: number;
|
|
73
73
|
nanoCpus: number;
|
|
74
|
+
/** `docker exec`s running in it (SW-856): the scheduler treats any as work in flight. */
|
|
75
|
+
execs?: number;
|
|
74
76
|
}
|
|
75
77
|
export interface FixturePresets {
|
|
76
78
|
/** An approved PR for a one-file change in packages/lib, with `demo-lib` running, for `pr gate`. */
|
|
@@ -88,9 +88,9 @@ export function fixturePreset({ gate, bare, change, ...opts }) {
|
|
|
88
88
|
? c.workspace
|
|
89
89
|
: Number.isNaN(Date.parse(c.startedAt))
|
|
90
90
|
? c.startedAt
|
|
91
|
-
: [c.memory, c.nanoCpus].find((n) => !Number.isSafeInteger(n) || n < 0);
|
|
91
|
+
: [c.memory, c.nanoCpus, c.execs ?? 0].find((n) => !Number.isSafeInteger(n) || n < 0);
|
|
92
92
|
if (bad !== undefined)
|
|
93
|
-
throw new Error(`--running: bad value ${JSON.stringify(bad)}: feature and workspace are lowercase names, startedAt a date, memory and
|
|
93
|
+
throw new Error(`--running: bad value ${JSON.stringify(bad)}: feature and workspace are lowercase names, startedAt a date, memory, nanoCpus and execs whole numbers`);
|
|
94
94
|
}
|
|
95
95
|
const badEntry = opts.queue?.find((q) => !QUEUE_ENTRY.test(q));
|
|
96
96
|
if (badEntry !== undefined)
|
|
@@ -267,7 +267,7 @@ ${githubProject(repos.map((r) => r.name))}${opts.config ? YAML.stringify(opts.co
|
|
|
267
267
|
const containers = (opts.containers ?? []).map((c) => {
|
|
268
268
|
const ws = path.join(root, "ws", c.workspace);
|
|
269
269
|
fs.mkdirSync(ws, { recursive: true });
|
|
270
|
-
return [c.feature, ws, c.startedAt, c.memory, c.nanoCpus].join("\t");
|
|
270
|
+
return [c.feature, ws, c.startedAt, c.memory, c.nanoCpus, c.execs ?? 0].join("\t");
|
|
271
271
|
});
|
|
272
272
|
fs.mkdirSync(home, { recursive: true });
|
|
273
273
|
const fakes = writeFakes(path.join(root, "bin"), {
|
package/dist/lifecycle.d.ts
CHANGED
|
@@ -110,6 +110,8 @@ export declare function startService(feature: string, workspace: string, service
|
|
|
110
110
|
grant?: boolean;
|
|
111
111
|
waitMs?: number;
|
|
112
112
|
}): Promise<void>;
|
|
113
|
+
/** One deadline for all of a `pr gate` / `pr open` run's service starts (SW-869): giving up drops the queue place, so wait longer in place rather than ask for a rerun. */
|
|
114
|
+
export declare const GATE_CAPACITY_WAIT_MS: number;
|
|
113
115
|
/**
|
|
114
116
|
* Admit starting *service* (and what compose starts with it) against the budget,
|
|
115
117
|
* waiting in the queue up to 10 min (SW-540): since SW-342 these starts are the
|
|
@@ -128,7 +130,7 @@ export declare function admitServiceStart(feature: string, workspace: string, se
|
|
|
128
130
|
grant?: boolean;
|
|
129
131
|
}): Promise<void>;
|
|
130
132
|
export declare const RANK_WORD: Record<Rank, string>;
|
|
131
|
-
/** A running lease for a person to read: `name (3 CPUs, 0.8 GiB, idle,
|
|
133
|
+
/** A running lease for a person to read: `name (3 CPUs, 0.8 GiB, idle, run in flight)`. */
|
|
132
134
|
export declare function describeLease(l: Lease): string;
|
|
133
135
|
/**
|
|
134
136
|
* Who holds the budget, the head *feature* waits behind, and what happens next: the same lines for a
|
|
@@ -200,6 +202,8 @@ export interface RunningFeature {
|
|
|
200
202
|
memory: number;
|
|
201
203
|
/** Σ `HostConfig.NanoCpus`, cores. */
|
|
202
204
|
cpus: number;
|
|
205
|
+
/** `docker exec`s running in its containers now (`.ExecIDs`: Docker drops an exec when it exits). */
|
|
206
|
+
execs: number;
|
|
203
207
|
}
|
|
204
208
|
export interface IdleInput {
|
|
205
209
|
minutes: number;
|
|
@@ -226,7 +230,9 @@ export interface StopIdleOptions {
|
|
|
226
230
|
only?: string;
|
|
227
231
|
now?: number;
|
|
228
232
|
run?: (cmd: string[]) => RunResult;
|
|
229
|
-
stop?: (feature: string
|
|
233
|
+
stop?: (feature: string, opts: {
|
|
234
|
+
why: string;
|
|
235
|
+
}) => Promise<unknown>;
|
|
230
236
|
/** Override for whether anyone waits for capacity; default: the queue file. */
|
|
231
237
|
queued?: boolean;
|
|
232
238
|
}
|
|
@@ -273,6 +279,11 @@ export declare function admitOrQueue(feature: string, workspace: string, byPerso
|
|
|
273
279
|
export declare function grantWaiters(): Promise<string[]>;
|
|
274
280
|
/** The daemon's side of a client's release (SW-185 D3): start waiters without holding the caller. */
|
|
275
281
|
export declare function grantInBackground(): number;
|
|
282
|
+
/**
|
|
283
|
+
* What a session hears when werkmeister stopped its containers under these commands (SW-856):
|
|
284
|
+
* they died with them (exit 137), so the session reruns them rather than reading a test failure.
|
|
285
|
+
*/
|
|
286
|
+
export declare function killedNudge(why: string, killed: string[]): string;
|
|
276
287
|
/**
|
|
277
288
|
* Stop *feature*'s containers, leaving them and its slot in place so
|
|
278
289
|
* `restart`/`up` can resume it.
|
|
@@ -280,8 +291,9 @@ export declare function grantInBackground(): number;
|
|
|
280
291
|
* Throws `NOT_RUNNING` when the feature holds no slot, and
|
|
281
292
|
* `COMPOSE_STOP_FAILED` when a container fails to stop.
|
|
282
293
|
*/
|
|
283
|
-
export declare function stop(feature: string, { grant }?: {
|
|
294
|
+
export declare function stop(feature: string, { grant, why }?: {
|
|
284
295
|
grant?: boolean;
|
|
296
|
+
why?: string;
|
|
285
297
|
}): Promise<StopResult>;
|
|
286
298
|
/**
|
|
287
299
|
* Restart *feature*'s containers in place.
|
package/dist/lifecycle.js
CHANGED
|
@@ -26,7 +26,7 @@
|
|
|
26
26
|
import * as fs from "node:fs";
|
|
27
27
|
import * as path from "node:path";
|
|
28
28
|
import { setTimeout as delay } from "node:timers/promises";
|
|
29
|
-
import { agentOf, clearPaused, clearWaiting, nudgePath, readPaused, readSessions, readWaiting, STALE_SESSION_MS, writeNudge, writePaused, writeWaiting, } from "./agent_state.js";
|
|
29
|
+
import { agentOf, claimLastStop, clearPaused, clearWaiting, nudgePath, readPaused, readSessions, readWaiting, STALE_SESSION_MS, writeLastStop, writeNudge, writePaused, writeWaiting, } from "./agent_state.js";
|
|
30
30
|
import { ensureSharedNetwork, isCaddyRunning, reloadCaddy, startCaddy } from "./caddy.js";
|
|
31
31
|
import { Colors } from "./colors.js";
|
|
32
32
|
import {
|
|
@@ -35,7 +35,7 @@ import {
|
|
|
35
35
|
composeDown, composeDownAsync, composeRestart, composeStop, composeUp, composeUpAsync, composeUpServiceAsync, dependencyOrder, getContainerStatus, isOneShotService, liveProject, workspaceForFeature, } from "./compose.js";
|
|
36
36
|
import { COMPOSE_FILE, migrateWorkspaceCompose } from "./compose_file.js";
|
|
37
37
|
import { getVolumeBaseNames, loadPorts, loadServices, needsCluster, sharedProxySkipReason, } from "./config.js";
|
|
38
|
-
import { docker, listVolumeNames, projectContainers, removeVolume } from "./docker.js";
|
|
38
|
+
import { docker, listVolumeNames, projectContainers, removeVolume, runningExecCommands, } from "./docker.js";
|
|
39
39
|
import { dotDir } from "./dot_dir.js";
|
|
40
40
|
import { SandboxError } from "./errors.js";
|
|
41
41
|
import { ingressHostnames, loadFeatureRouters, warnUnresolvedHosts, } from "./feature_routing.js";
|
|
@@ -187,9 +187,17 @@ export async function up(feature, workspace, { signal, byPerson } = {}) {
|
|
|
187
187
|
requireSharedNetwork();
|
|
188
188
|
info("Starting containers...");
|
|
189
189
|
if (!(await withStartRecord(workspace, "up", () => composeUpAsync(feature, signal)))) {
|
|
190
|
-
throw new SandboxError(
|
|
190
|
+
throw new SandboxError(
|
|
191
|
+
// Not "above": a grant runs this in another process, and its nudge carries this text (SW-861).
|
|
192
|
+
`Failed to start containers: \`werkmeister feature logs ${feature}\` shows why.`, "COMPOSE_UP_FAILED");
|
|
191
193
|
}
|
|
192
194
|
containersStarted = true;
|
|
195
|
+
// Up now: two admits return before the one that dequeues (Docker unreadable, a zero-cost start), and
|
|
196
|
+
// a leftover entry keeps the hook saying "queued" over running containers (SW-863), as in restart().
|
|
197
|
+
await dequeue(feature);
|
|
198
|
+
clearWaiting(workspace);
|
|
199
|
+
// Its stop is answered by this start; a later grant must not report it (SW-702). The grant claims it first.
|
|
200
|
+
claimLastStop(workspace);
|
|
193
201
|
}
|
|
194
202
|
else {
|
|
195
203
|
info("No local services to start -- routing entirely to k8s");
|
|
@@ -326,11 +334,13 @@ export async function startService(feature, workspace, service, composeName = CO
|
|
|
326
334
|
if (!started) {
|
|
327
335
|
// A timed-out or cancelled up leaves the container Created, never started (SW-86).
|
|
328
336
|
const state = (await getContainerStatus(feature)).find((c) => c.service === service)?.status;
|
|
329
|
-
throw new SandboxError(`Failed to start ${service}${state ? ` (left ${state})` : ""}
|
|
337
|
+
throw new SandboxError(`Failed to start ${service}${state ? ` (left ${state})` : ""}: \`werkmeister feature logs ${feature} ${service}\` shows why`, "COMPOSE_UP_FAILED");
|
|
330
338
|
}
|
|
331
339
|
}
|
|
332
340
|
const SERVICE_POLL_MS = 15_000;
|
|
333
341
|
const SERVICE_WAIT_MS = 10 * 60_000;
|
|
342
|
+
/** One deadline for all of a `pr gate` / `pr open` run's service starts (SW-869): giving up drops the queue place, so wait longer in place rather than ask for a rerun. */
|
|
343
|
+
export const GATE_CAPACITY_WAIT_MS = 30 * 60_000;
|
|
334
344
|
/**
|
|
335
345
|
* Admit starting *service* (and what compose starts with it) against the budget,
|
|
336
346
|
* waiting in the queue up to 10 min (SW-540): since SW-342 these starts are the
|
|
@@ -360,7 +370,12 @@ export async function admitServiceStart(feature, workspace, service, { sleep = d
|
|
|
360
370
|
// Never byPerson: an agent's gate in a person's workspace must not stop another run's containers.
|
|
361
371
|
let position = await admitOrQueue(feature, workspace, false, starting);
|
|
362
372
|
// Not in a person's workspace: their running app must not go down because they asked for one more service.
|
|
363
|
-
|
|
373
|
+
// Nor in the grant pass (SW-873): with no wait it would stop its own set and drop it, never parked.
|
|
374
|
+
if (position !== null &&
|
|
375
|
+
grant &&
|
|
376
|
+
ownRunning.length &&
|
|
377
|
+
!released.length &&
|
|
378
|
+
readAutonomous(workspace)) {
|
|
364
379
|
// Release instead of hold (SW-675): two gates each holding core while waiting for one more
|
|
365
380
|
// service deadlock a full budget. Stopped, not removed: one request then brings back the set.
|
|
366
381
|
await stop(feature, { grant: false });
|
|
@@ -390,7 +405,7 @@ export async function admitServiceStart(feature, workspace, service, { sleep = d
|
|
|
390
405
|
writeWaiting(workspace, { ...waiting, parked: { service } });
|
|
391
406
|
parked = true;
|
|
392
407
|
}
|
|
393
|
-
throw new SandboxError(`No capacity to start ${services.join(", ")} for '${feature}' after ${waitMs / 60_000} min ` +
|
|
408
|
+
throw new SandboxError(`No capacity to start ${services.join(", ")} for '${feature}' after ${Math.round(waitMs / 60_000)} min ` +
|
|
394
409
|
`(queue position ${position})${parked ? "; it stays queued" : ""}.` +
|
|
395
410
|
(released.length
|
|
396
411
|
? ` ${[...up].join(", ")} stayed stopped; \`werkmeister feature up\` starts the feature again.`
|
|
@@ -400,7 +415,7 @@ export async function admitServiceStart(feature, workspace, service, { sleep = d
|
|
|
400
415
|
if (!waited) {
|
|
401
416
|
warn(
|
|
402
417
|
// SW-734: say how a wait the person doesn't want ends.
|
|
403
|
-
`Waiting for capacity to start ${services.join(", ")}: position ${position}, up to ${waitMs / 60_000} min. ` +
|
|
418
|
+
`Waiting for capacity to start ${services.join(", ")}: position ${position}, up to ${Math.round(waitMs / 60_000)} min. ` +
|
|
404
419
|
"Stopping the command ends the wait; `pr gate` and `pr open` start no service the PR file's `skip_tests` names.");
|
|
405
420
|
}
|
|
406
421
|
await sleep(SERVICE_POLL_MS);
|
|
@@ -429,9 +444,9 @@ export const RANK_WORD = {
|
|
|
429
444
|
working: "working",
|
|
430
445
|
person: "a person's workspace",
|
|
431
446
|
};
|
|
432
|
-
/** A running lease for a person to read: `name (3 CPUs, 0.8 GiB, idle,
|
|
447
|
+
/** A running lease for a person to read: `name (3 CPUs, 0.8 GiB, idle, run in flight)`. */
|
|
433
448
|
export function describeLease(l) {
|
|
434
|
-
return `${l.feature} (${cpus(l.cost.cpus)}, ${gib(l.cost.memory)}, ${RANK_WORD[l.rank]}${l.busy ? ",
|
|
449
|
+
return `${l.feature} (${cpus(l.cost.cpus)}, ${gib(l.cost.memory)}, ${RANK_WORD[l.rank]}${l.busy ? ", run in flight" : ""})`;
|
|
435
450
|
}
|
|
436
451
|
/**
|
|
437
452
|
* Who holds the budget, the head *feature* waits behind, and what happens next: the same lines for a
|
|
@@ -587,6 +602,9 @@ export function otherLiveSessions(workspaceRoot, sessionId) {
|
|
|
587
602
|
}
|
|
588
603
|
/** Why a running feature is not idle, or null when it is. */
|
|
589
604
|
function busyReason(f, input, cutoff) {
|
|
605
|
+
// A test run or probe in its container is work in flight, whatever the session record says (SW-856).
|
|
606
|
+
if (f.execs)
|
|
607
|
+
return "docker exec running";
|
|
590
608
|
// An open session that has sat idle past the cutoff does not count (SW-39 D3), nor, while others
|
|
591
609
|
// queue, one that has waited on a person that long: its status stays busy through the ask (SW-675).
|
|
592
610
|
const recent = (s) => (s.statusUpdatedAt ?? Infinity) > cutoff;
|
|
@@ -641,19 +659,27 @@ export function runningFeatures(runCmd = (cmd) => run(cmd, { capture: true, time
|
|
|
641
659
|
if (!list.size)
|
|
642
660
|
return [];
|
|
643
661
|
const feature = `or (index .Config.Labels "${LABEL_FEATURE}") (index .Config.Labels "${LEGACY_LABEL_FEATURE}")`;
|
|
644
|
-
const fmt = `{{${feature}}}\t{{index .Config.Labels "${LABEL_WORKDIR}"}}\t{{.State.StartedAt}}\t{{.HostConfig.MemoryReservation}}\t{{.HostConfig.NanoCpus}}`;
|
|
662
|
+
const fmt = `{{${feature}}}\t{{index .Config.Labels "${LABEL_WORKDIR}"}}\t{{.State.StartedAt}}\t{{.HostConfig.MemoryReservation}}\t{{.HostConfig.NanoCpus}}\t{{len .ExecIDs}}`;
|
|
645
663
|
const inspect = runCmd(["docker", "inspect", "--format", fmt, ...list]);
|
|
646
664
|
if (inspect.status !== 0)
|
|
647
665
|
return null;
|
|
648
666
|
const byFeature = new Map();
|
|
649
667
|
for (const line of inspect.stdout.split("\n").filter(Boolean)) {
|
|
650
|
-
const [feature, workspace, started, memory, nanoCpus] = line.split("\t");
|
|
668
|
+
const [feature, workspace, started, memory, nanoCpus, execs] = line.split("\t");
|
|
651
669
|
if (!feature || !workspace)
|
|
652
670
|
continue;
|
|
653
|
-
const f = byFeature.get(feature) ?? {
|
|
671
|
+
const f = byFeature.get(feature) ?? {
|
|
672
|
+
feature,
|
|
673
|
+
workspace,
|
|
674
|
+
startedAt: [],
|
|
675
|
+
memory: 0,
|
|
676
|
+
cpus: 0,
|
|
677
|
+
execs: 0,
|
|
678
|
+
};
|
|
654
679
|
f.startedAt.push(Date.parse(started));
|
|
655
680
|
f.memory += Number(memory) || 0;
|
|
656
681
|
f.cpus += (Number(nanoCpus) || 0) / 1e9;
|
|
682
|
+
f.execs += Number(execs) || 0;
|
|
657
683
|
byFeature.set(feature, f);
|
|
658
684
|
}
|
|
659
685
|
return [...byFeature.values()];
|
|
@@ -699,7 +725,7 @@ export async function stopIdle(minutes, opts = {}) {
|
|
|
699
725
|
}
|
|
700
726
|
const asking = input.sessions(f.workspace).some((s) => s.asking);
|
|
701
727
|
try {
|
|
702
|
-
await (opts.stop ?? stop)(f.feature);
|
|
728
|
+
await (opts.stop ?? stop)(f.feature, { why: `idle ≥ ${minutes} min` });
|
|
703
729
|
lines.push(asking
|
|
704
730
|
? `${f.feature}: stopped (waiting on a person ≥ ${minutes} min, others queued)`
|
|
705
731
|
: `${f.feature}: stopped (idle ≥ ${minutes} min)`);
|
|
@@ -733,11 +759,19 @@ const GRANTED_NUDGE = "Capacity freed: this workspace's containers are up. Conti
|
|
|
733
759
|
* The granted session's nudge. Preemption stops on-demand containers too, and the grant's `up` starts
|
|
734
760
|
* only the default set, so name each one left stopped and how to start it (SW-840).
|
|
735
761
|
*/
|
|
736
|
-
async function grantedNudge(feature) {
|
|
762
|
+
async function grantedNudge(feature, last) {
|
|
737
763
|
const stopped = (await getContainerStatus(feature)).filter((c) => c.status !== "running" && !isOneShotService(c.service, c.role));
|
|
764
|
+
// Say why it was down at all (SW-702): nothing else tells the session who stopped it.
|
|
765
|
+
const why = last
|
|
766
|
+
? `Werkmeister had stopped it (${last.why})` +
|
|
767
|
+
(last.killed.length
|
|
768
|
+
? `, killing ${last.killed.map((c) => `\`${c}\``).join(", ")}: rerun them`
|
|
769
|
+
: "") +
|
|
770
|
+
". "
|
|
771
|
+
: "";
|
|
738
772
|
if (!stopped.length)
|
|
739
|
-
return GRANTED_NUDGE;
|
|
740
|
-
return (
|
|
773
|
+
return why + GRANTED_NUDGE;
|
|
774
|
+
return (`${why}Capacity freed: this workspace started, but these containers are still stopped: ${stopped.map((c) => c.name).join(", ")}. ` +
|
|
741
775
|
`Start them with ${stopped.map((c) => `\`werkmeister feature up --service ${c.service}\``).join(", ")}, ` +
|
|
742
776
|
"then continue where you left off.");
|
|
743
777
|
}
|
|
@@ -768,7 +802,9 @@ export function runningLeases(runCmd) {
|
|
|
768
802
|
: state?.agent === "working"
|
|
769
803
|
? "working"
|
|
770
804
|
: "idle";
|
|
771
|
-
|
|
805
|
+
// A running `docker exec` (a session's test run, not only the gate's) is never stopped (SW-856).
|
|
806
|
+
const busy = f.execs > 0 ||
|
|
807
|
+
readPaused(f.workspace) !== null ||
|
|
772
808
|
(mtimeMs(lastExecMarker(f.workspace)) ?? 0) > gateCutoff;
|
|
773
809
|
const lease = {
|
|
774
810
|
feature: f.feature,
|
|
@@ -817,7 +853,7 @@ async function admitOrQueueHere(feature, workspace, byPerson, starting) {
|
|
|
817
853
|
const victimWs = readLeases()[victim]?.workspace || workspaceForFeature(victim);
|
|
818
854
|
try {
|
|
819
855
|
// stop, not pause: a paused container keeps its memory.
|
|
820
|
-
await stop(victim, { grant: false });
|
|
856
|
+
await stop(victim, { grant: false, why: `preempted to make room for '${feature}'` });
|
|
821
857
|
}
|
|
822
858
|
catch (e) {
|
|
823
859
|
// A failed stop can still have killed it (SW-738): queue it to come back, and wait rather than
|
|
@@ -867,46 +903,57 @@ async function grantWaitersNow() {
|
|
|
867
903
|
// A lease pruned by a process that can't see the folder must not cost the waiter its start (SW-848).
|
|
868
904
|
const workspace = readLeases()[feature]?.workspace || workspaceForFeature(feature);
|
|
869
905
|
if (!workspace) {
|
|
906
|
+
// No folder to nudge in: say it here, at least (SW-873).
|
|
907
|
+
warn(`'${feature}' left the waiting queue: its workspace can't be found`);
|
|
870
908
|
await dequeue(feature);
|
|
871
909
|
continue;
|
|
872
910
|
}
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
if (await startParked(feature, workspace, waiting.parked.service)) {
|
|
876
|
-
started.push(feature);
|
|
877
|
-
if (liveSessions(workspace).length)
|
|
878
|
-
writeNudge(workspace, await grantedNudge(feature));
|
|
879
|
-
}
|
|
880
|
-
continue;
|
|
881
|
-
}
|
|
882
|
-
// A service start's own poll admits it; `up` would start the wrong set and dequeue it.
|
|
883
|
-
if (waiting?.services)
|
|
884
|
-
continue;
|
|
885
|
-
let r;
|
|
911
|
+
// Before up(), which drops the record on any start, so only this grant's nudge reports it (SW-702).
|
|
912
|
+
const last = claimLastStop(workspace);
|
|
886
913
|
try {
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
clearWaiting(workspace);
|
|
896
|
-
// A never-fits rerun would fail the same way; its message already says what to change.
|
|
897
|
-
const neverFits = e instanceof SandboxError && e.code === "NEVER_FITS";
|
|
898
|
-
writeNudge(workspace, neverFits
|
|
899
|
-
? message
|
|
900
|
-
: `${message}\nIt left the queue: run \`werkmeister feature up\` to queue it again.`);
|
|
914
|
+
const waiting = readWaiting(workspace);
|
|
915
|
+
if (waiting?.parked) {
|
|
916
|
+
if (await startParked(feature, workspace, waiting.parked.service)) {
|
|
917
|
+
started.push(feature);
|
|
918
|
+
if (liveSessions(workspace).length)
|
|
919
|
+
writeNudge(workspace, await grantedNudge(feature, last));
|
|
920
|
+
}
|
|
921
|
+
continue;
|
|
901
922
|
}
|
|
902
|
-
|
|
923
|
+
// A service start's own poll admits it; `up` would start the wrong set and dequeue it.
|
|
924
|
+
if (waiting?.services)
|
|
925
|
+
continue;
|
|
926
|
+
let r;
|
|
927
|
+
try {
|
|
928
|
+
r = await up(feature, workspace, { byPerson: false });
|
|
929
|
+
}
|
|
930
|
+
catch (e) {
|
|
931
|
+
// One waiter's failed re-up (HOST_LOAD, NO_COMPOSE, ...) must not end the pass for the rest (SW-737).
|
|
932
|
+
const message = `Could not start '${feature}' from the waiting queue: ${e.message}`;
|
|
933
|
+
warn(message);
|
|
934
|
+
// Admission already dropped it (never fits, or a start that failed after admit): only its session can act (SW-682, SW-848).
|
|
935
|
+
if (!readQueue().includes(feature)) {
|
|
936
|
+
clearWaiting(workspace);
|
|
937
|
+
// A never-fits rerun would fail the same way; its message already says what to change.
|
|
938
|
+
const neverFits = e instanceof SandboxError && e.code === "NEVER_FITS";
|
|
939
|
+
writeNudge(workspace, neverFits
|
|
940
|
+
? message
|
|
941
|
+
: `${message}\nIt left the queue: run \`werkmeister feature up\` to queue it again.`);
|
|
942
|
+
}
|
|
943
|
+
continue;
|
|
944
|
+
}
|
|
945
|
+
// Backfill (SW-675): a later waiter may fit; admission itself holds the room for a starved head.
|
|
946
|
+
if (r.waiting !== undefined)
|
|
947
|
+
continue;
|
|
948
|
+
started.push(feature);
|
|
949
|
+
if (liveSessions(workspace).length)
|
|
950
|
+
writeNudge(workspace, await grantedNudge(feature, last));
|
|
951
|
+
}
|
|
952
|
+
finally {
|
|
953
|
+
// Still waiting: the grant that starts it reports the stop.
|
|
954
|
+
if (last && !started.includes(feature) && readQueue().includes(feature))
|
|
955
|
+
writeLastStop(workspace, last);
|
|
903
956
|
}
|
|
904
|
-
// Backfill (SW-675): a later waiter may fit; admission itself holds the room for a starved head.
|
|
905
|
-
if (r.waiting !== undefined)
|
|
906
|
-
continue;
|
|
907
|
-
started.push(feature);
|
|
908
|
-
if (liveSessions(workspace).length)
|
|
909
|
-
writeNudge(workspace, await grantedNudge(feature));
|
|
910
957
|
}
|
|
911
958
|
return started;
|
|
912
959
|
}
|
|
@@ -923,7 +970,9 @@ async function startParked(feature, workspace, service) {
|
|
|
923
970
|
});
|
|
924
971
|
}
|
|
925
972
|
catch (e) {
|
|
926
|
-
|
|
973
|
+
// No room keeps it parked; any other way out of the queue its session must hear (SW-873).
|
|
974
|
+
if (!(e instanceof SandboxError && e.code === "NO_CAPACITY") ||
|
|
975
|
+
!readQueue().includes(feature)) {
|
|
927
976
|
const message = `Could not start '${feature}' ${service} from the waiting queue: ${e.message}`;
|
|
928
977
|
warn(message);
|
|
929
978
|
// Its session was told it continues once started; it must hear that it won't (SW-701 review).
|
|
@@ -955,6 +1004,15 @@ async function grantWaitersQuietly() {
|
|
|
955
1004
|
warn(`Could not start the next waiting feature: ${e instanceof Error ? e.message : String(e)}`);
|
|
956
1005
|
}
|
|
957
1006
|
}
|
|
1007
|
+
/**
|
|
1008
|
+
* What a session hears when werkmeister stopped its containers under these commands (SW-856):
|
|
1009
|
+
* they died with them (exit 137), so the session reruns them rather than reading a test failure.
|
|
1010
|
+
*/
|
|
1011
|
+
export function killedNudge(why, killed) {
|
|
1012
|
+
return (`Werkmeister stopped this workspace's containers (${why}) while these ran in them: ` +
|
|
1013
|
+
`${killed.map((c) => `\`${c}\``).join(", ")}. They were killed (exit 137), not failed: start the ` +
|
|
1014
|
+
"containers with `werkmeister feature up` (`--service <name>` for an on-demand one), then rerun them.");
|
|
1015
|
+
}
|
|
958
1016
|
/**
|
|
959
1017
|
* Stop *feature*'s containers, leaving them and its slot in place so
|
|
960
1018
|
* `restart`/`up` can resume it.
|
|
@@ -962,9 +1020,26 @@ async function grantWaitersQuietly() {
|
|
|
962
1020
|
* Throws `NOT_RUNNING` when the feature holds no slot, and
|
|
963
1021
|
* `COMPOSE_STOP_FAILED` when a container fails to stop.
|
|
964
1022
|
*/
|
|
965
|
-
export async function stop(feature, { grant = true } = {}) {
|
|
1023
|
+
export async function stop(feature, { grant = true, why } = {}) {
|
|
966
1024
|
refuseInsideOwnContainer(feature, "stop");
|
|
967
1025
|
const slot = await requireSlot(feature);
|
|
1026
|
+
// *why* marks a stop nobody in the workspace asked for: name what it kills to the session (SW-856).
|
|
1027
|
+
const workspace = why ? workspaceForFeature(feature) : null;
|
|
1028
|
+
if (why && workspace) {
|
|
1029
|
+
const names = (await getContainerStatus(feature))
|
|
1030
|
+
.filter((c) => c.status === "running")
|
|
1031
|
+
.map((c) => c.name);
|
|
1032
|
+
const killed = await runningExecCommands(names);
|
|
1033
|
+
// Best effort: a folder gone, or unseen from a container (SW-860), must not block the stop.
|
|
1034
|
+
try {
|
|
1035
|
+
writeLastStop(workspace, { at: new Date().toISOString(), why, killed });
|
|
1036
|
+
if (killed.length)
|
|
1037
|
+
writeNudge(workspace, killedNudge(why, killed));
|
|
1038
|
+
}
|
|
1039
|
+
catch (e) {
|
|
1040
|
+
warn(`Could not tell '${feature}' why it stopped: ${e.message}`);
|
|
1041
|
+
}
|
|
1042
|
+
}
|
|
968
1043
|
info(`Stopping containers for '${feature}' (slot ${slot})...`);
|
|
969
1044
|
if (!(await composeStop(feature))) {
|
|
970
1045
|
throw new SandboxError(`Failed to stop containers for '${feature}'`, "COMPOSE_STOP_FAILED");
|
package/dist/pr_lifecycle.js
CHANGED
|
@@ -19,7 +19,7 @@ import { GATE_COMPOSE_FILE, renderGateCompose } from "./generate.js";
|
|
|
19
19
|
import { updateNoteStatus } from "./graduate.js";
|
|
20
20
|
import { eachIssue, issueStatuses, moveIssue } from "./issue_status.js";
|
|
21
21
|
import { groupOf, issueIdOf, issueRefsOf, issueTracker, projectOf } from "./issue_tracker.js";
|
|
22
|
-
import { grantWaiters, idleStopMinutes, lastExecMarker, liveSessions, readServicesFromWorkspace, runningFeatures, startService, stop as stopFeature, stopIdle, } from "./lifecycle.js";
|
|
22
|
+
import { GATE_CAPACITY_WAIT_MS, grantWaiters, idleStopMinutes, lastExecMarker, liveSessions, readServicesFromWorkspace, runningFeatures, startService, stop as stopFeature, stopIdle, } from "./lifecycle.js";
|
|
23
23
|
import { originSlug, prWatchScheduleState, } from "./machine_setup.js";
|
|
24
24
|
import { openDecisions } from "./notes.js";
|
|
25
25
|
import { prRulesPath } from "./pr_guard.js";
|
|
@@ -58,6 +58,8 @@ async function httpStatus(url) {
|
|
|
58
58
|
}
|
|
59
59
|
}
|
|
60
60
|
export function defaultDeps() {
|
|
61
|
+
// Built once per command, so the wait is a total across its starts, not per start (SW-869).
|
|
62
|
+
const capacityDeadline = Date.now() + GATE_CAPACITY_WAIT_MS;
|
|
61
63
|
return {
|
|
62
64
|
run: execRun,
|
|
63
65
|
httpStatus,
|
|
@@ -100,7 +102,10 @@ export function defaultDeps() {
|
|
|
100
102
|
const granted = await grantWaiters().catch((e) => [`waiting queue: ${e.message}`]);
|
|
101
103
|
return { lines: [...lines, ...granted.map((f) => `${f}: started from the waiting queue`)] };
|
|
102
104
|
},
|
|
103
|
-
startService: (ws, feature, service, composeName) => startService(feature, ws, service, composeName, {
|
|
105
|
+
startService: (ws, feature, service, composeName) => startService(feature, ws, service, composeName, {
|
|
106
|
+
skipCurrentSetups: true,
|
|
107
|
+
waitMs: Math.max(0, capacityDeadline - Date.now()),
|
|
108
|
+
}),
|
|
104
109
|
portHolders: (ws, feature, services) => portHolders(feature, ws, services),
|
|
105
110
|
gateCompose: (ws, feature, services) => renderGateCompose(feature, Number(readEnv(ws, "WERKMEISTER_SLOT")), readServicesFromWorkspace(ws), services, ws),
|
|
106
111
|
};
|
package/dist/scheduler.d.ts
CHANGED
|
@@ -19,7 +19,7 @@ export interface Lease {
|
|
|
19
19
|
feature: string;
|
|
20
20
|
cost: Cost;
|
|
21
21
|
rank: Rank;
|
|
22
|
-
/**
|
|
22
|
+
/** A `docker exec` running in it, mid-gate or paused: stopping it would break a run in flight. */
|
|
23
23
|
busy: boolean;
|
|
24
24
|
}
|
|
25
25
|
export interface Request {
|
package/dist/slots.d.ts
CHANGED
|
@@ -53,6 +53,8 @@ export declare function getAllSlotsFromDocker(): Promise<Record<string, SlotEntr
|
|
|
53
53
|
/** Reads Docker labels; injectable so tests need no daemon. */
|
|
54
54
|
export interface SlotDeps {
|
|
55
55
|
labels: () => Promise<Record<string, SlotEntry> | null>;
|
|
56
|
+
/** Docker's marker file, for tests; default `/.dockerenv`. */
|
|
57
|
+
dockerenv?: string;
|
|
56
58
|
}
|
|
57
59
|
/**
|
|
58
60
|
* Allocate *feature*'s slot and lease it before the lock is released.
|
package/dist/slots.js
CHANGED
|
@@ -25,6 +25,7 @@ import { workspaceForFeature } from "./compose.js";
|
|
|
25
25
|
import { baselineUpstream, loadPorts, loadServices, sandboxHome, slotPortRange } from "./config.js";
|
|
26
26
|
import { docker } from "./docker.js";
|
|
27
27
|
import { SandboxError } from "./errors.js";
|
|
28
|
+
import { insideContainer } from "./global_config.js";
|
|
28
29
|
import { sandboxLock } from "./locks.js";
|
|
29
30
|
import { error, info, success, warn } from "./reporter.js";
|
|
30
31
|
import { pickSlot } from "./scheduler.js";
|
|
@@ -175,11 +176,15 @@ const defaultSlotDeps = { labels: getAllSlotsFromDocker };
|
|
|
175
176
|
* label and a lease disagree on a slot's owner the label wins: those
|
|
176
177
|
* containers are what bind its ports.
|
|
177
178
|
*/
|
|
178
|
-
function heldSlots(dockerSlots) {
|
|
179
|
+
function heldSlots(dockerSlots, dockerenv) {
|
|
179
180
|
const leases = readLeases();
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
181
|
+
// A container mounts the real ~/.werkmeister but only its own workspace: every other folder
|
|
182
|
+
// reads gone from there, so only the host prunes (SW-860).
|
|
183
|
+
if (!insideContainer(dockerenv)) {
|
|
184
|
+
for (const [f, l] of Object.entries(leases)) {
|
|
185
|
+
if (l.workspace && !fs.existsSync(l.workspace))
|
|
186
|
+
delete leases[f];
|
|
187
|
+
}
|
|
183
188
|
}
|
|
184
189
|
for (const [f, e] of Object.entries(dockerSlots ?? {})) {
|
|
185
190
|
leases[f] ??= { slot: e.slot, workspace: workspaceForFeature(f) ?? "" };
|
|
@@ -210,7 +215,7 @@ export function allocateSlot(feature, preferredSlot, workspace, deps = defaultSl
|
|
|
210
215
|
function allocateSlotHere(feature, preferredSlot, workspace, deps) {
|
|
211
216
|
const count = slotCount();
|
|
212
217
|
return sandboxLock("allocate", async () => {
|
|
213
|
-
const { leases, held } = heldSlots(await deps.labels());
|
|
218
|
+
const { leases, held } = heldSlots(await deps.labels(), deps.dockerenv);
|
|
214
219
|
const slot = pickSlot(feature, held, count, preferredSlot ?? null);
|
|
215
220
|
if (slot === null) {
|
|
216
221
|
error(`All ${count} feature slots are occupied!`);
|
package/package.json
CHANGED