werkmeister-core 0.89.0 → 0.89.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +2 -0
- package/dist/harness/claude/claude_md.js +1 -1
- package/dist/pr_lifecycle.d.ts +5 -0
- package/dist/pr_lifecycle.js +17 -3
- package/dist/scheduler_state.d.ts +27 -6
- package/dist/scheduler_state.js +94 -20
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -501,7 +501,7 @@ or visit https://${proxy.host}/__werkmeister and select \`${feature}\` — the s
|
|
|
501
501
|
How to exercise a change to each service in this workspace.${onDemand.length
|
|
502
502
|
? ` A plain \`werkmeister feature up\` skips the on-demand ones, ${onDemand.map((s) => `\`${s}\``).join(", ")}: start one with \`werkmeister feature up --service <name>\`.`
|
|
503
503
|
: ""}${tempHome
|
|
504
|
-
? ` Each test command runs through \`.werkmeister/bin/with-temp-home\`, which gives it a fresh temporary HOME and deletes it on exit: never hand-roll \`HOME=…\`. Start every exercise command with the script itself, never with env assignments or a \`cd\`, so one permission rule matches it: \`${tempHome} <test command>\`. A command that needs an env var takes it after the script (\`${tempHome} env WERKMEISTER_ROOT=… <cmd>\`); no rule pre-approves that form, so it asks. Put \`--no-wait\` first (\`${tempHome} --no-wait <cmd>\`) to skip its host-load wait; the allow rules below cover that form too. That wait lasts up to 90 s; if load is still over then, the script exits 75 without running the command (retry later). It prints only on stderr, so a foreground call shows nothing until it ends and its only child is \`werkmeister internal wait-load\`: that is the wait, not a hang. The allow rule that covers them is \`Bash(${tempHome} <test command>:*)\` for each test command below: run those for a test without a prompt. A test command of more than one command (an \`&&\` chain, a \`{ …; }\` group, \`||\`, \`;\` or a pipe) is shown as \`${tempHome} sh -c '…'\`, so every part runs under the temp HOME, and its rule allows that exact line: run it as shown, with no extra arguments. Neither \`node --test\` through the script nor the script alone is allowed, since both run any file (SW-510). The temp HOME only keeps a run from writing to the real one by accident; it is not an isolation boundary. A pre-approved test command or built CLI still runs whatever the worktree holds, a test written in this run included, and that code can reach absolute paths, the caller's env, the network, docker and gh (SW-522).`
|
|
504
|
+
? ` Each test command runs through \`.werkmeister/bin/with-temp-home\`, which gives it a fresh temporary HOME and deletes it on exit: never hand-roll \`HOME=…\`. Start every exercise command with the script itself, never with env assignments or a \`cd\`, so one permission rule matches it: \`${tempHome} <test command>\`. A command that needs an env var takes it after the script (\`${tempHome} env WERKMEISTER_ROOT=… <cmd>\`); no rule pre-approves that form, so it asks. Put \`--no-wait\` first (\`${tempHome} --no-wait <cmd>\`) to skip its host-load and free-memory wait; the allow rules below cover that form too. That wait lasts up to 90 s; if load or memory is still over its limit then, the script exits 75 without running the command (retry later). It prints only on stderr, so a foreground call shows nothing until it ends and its only child is \`werkmeister internal wait-load\`: that is the wait, not a hang. The allow rule that covers them is \`Bash(${tempHome} <test command>:*)\` for each test command below: run those for a test without a prompt. A test command of more than one command (an \`&&\` chain, a \`{ …; }\` group, \`||\`, \`;\` or a pipe) is shown as \`${tempHome} sh -c '…'\`, so every part runs under the temp HOME, and its rule allows that exact line: run it as shown, with no extra arguments. Neither \`node --test\` through the script nor the script alone is allowed, since both run any file (SW-510). The temp HOME only keeps a run from writing to the real one by accident; it is not an isolation boundary. A pre-approved test command or built CLI still runs whatever the worktree holds, a test written in this run included, and that code can reach absolute paths, the caller's env, the network, docker and gh (SW-522).`
|
|
505
505
|
: ""}${linted
|
|
506
506
|
? " A test command ending in `&& <lint>` also runs the repo's lint, the same command the PR gate's lint row runs, so a format error fails here and not first at `werkmeister pr open`."
|
|
507
507
|
: ""}
|
package/dist/pr_lifecycle.d.ts
CHANGED
|
@@ -76,6 +76,11 @@ export interface Deps {
|
|
|
76
76
|
load: number[];
|
|
77
77
|
cores: number;
|
|
78
78
|
};
|
|
79
|
+
/** Bytes available and total, read once before the gate starts anything (SW-620). */
|
|
80
|
+
hostMemory: () => {
|
|
81
|
+
free: number;
|
|
82
|
+
total: number;
|
|
83
|
+
};
|
|
79
84
|
sleep: (ms: number) => Promise<void>;
|
|
80
85
|
/** One idle-reaper pass over every workspace (SW-39). */
|
|
81
86
|
reapIdle: () => Promise<{
|
package/dist/pr_lifecycle.js
CHANGED
|
@@ -28,7 +28,7 @@ import { loadNote } from "./resources.js";
|
|
|
28
28
|
import { commitAndPush, listRepoWorktrees, listWorkspaces, removeWorkspace, resolveSessionSource, resolveWorkspaceDir, sendToClaudeSession, sessionStartDir, WorkspaceError, worktreeSources,
|
|
29
29
|
// biome-ignore lint/suspicious/noImportCycles: pr_lifecycle↔sandbox.ts↔usage_snapshot; SW-411
|
|
30
30
|
} from "./sandbox.js";
|
|
31
|
-
import { portHolders, readHostLoad, waitForHostLoadNow } from "./scheduler_state.js";
|
|
31
|
+
import { lowMemory, portHolders, readHostLoad, readHostMemory, waitForHostLoadNow, } from "./scheduler_state.js";
|
|
32
32
|
import { getServicePort } from "./slots.js";
|
|
33
33
|
import { repoDirs } from "./standards.js";
|
|
34
34
|
import { TEMP_HOME_SCRIPT } from "./temp_home.js";
|
|
@@ -91,6 +91,7 @@ export function defaultDeps() {
|
|
|
91
91
|
stop: stopWorkspace,
|
|
92
92
|
prWatch: () => prWatchScheduleState(),
|
|
93
93
|
hostLoad: readHostLoad,
|
|
94
|
+
hostMemory: readHostMemory,
|
|
94
95
|
sleep: (ms) => delay(ms),
|
|
95
96
|
reapIdle: async () => {
|
|
96
97
|
const minutes = idleStopMinutes();
|
|
@@ -571,7 +572,8 @@ testsOnly = false) {
|
|
|
571
572
|
...(!ok && capacityShort && NO_CONTAINER.test(detail) && { retry: true }),
|
|
572
573
|
});
|
|
573
574
|
const startFailed = (name, e, prefix = "") => {
|
|
574
|
-
|
|
575
|
+
// HOST_LOAD: a setup start that waited out load or low memory is the host, not the change (SW-620).
|
|
576
|
+
const capacity = ["NO_CAPACITY", "HOST_LOAD"].includes(e.code ?? "");
|
|
575
577
|
if (capacity)
|
|
576
578
|
capacityShort = true;
|
|
577
579
|
checks.push({
|
|
@@ -884,6 +886,18 @@ testsOnly = false) {
|
|
|
884
886
|
}
|
|
885
887
|
return true;
|
|
886
888
|
};
|
|
889
|
+
// Admission prices reservations, not the host's live memory: Claude Code stops an idle background
|
|
890
|
+
// gate when the Mac runs low, before any row runs, so exit 75 before starting anything (SW-620).
|
|
891
|
+
const memory = lowMemory(deps.hostMemory);
|
|
892
|
+
if (memory) {
|
|
893
|
+
checks.push({
|
|
894
|
+
name: "memory",
|
|
895
|
+
ok: false,
|
|
896
|
+
detail: `low memory: ${memory}; no container started and no row run. Retry later.`,
|
|
897
|
+
retry: true,
|
|
898
|
+
});
|
|
899
|
+
return done();
|
|
900
|
+
}
|
|
887
901
|
// Running containers come from compose.yml, rendered from the machine registry; a
|
|
888
902
|
// touched service whose branch settings render differently runs from compose.gate.yml (SW-118).
|
|
889
903
|
let changed = [];
|
|
@@ -1112,7 +1126,7 @@ export function failedRowExcerpt(output, tail) {
|
|
|
1112
1126
|
}
|
|
1113
1127
|
export function formatGate(dir, g) {
|
|
1114
1128
|
return [
|
|
1115
|
-
`${dir}: gate ${g.ok ? "passed" : g.retry ?
|
|
1129
|
+
`${dir}: gate ${g.ok ? "passed" : g.retry ? `retry later (${g.checks.some((c) => c.name === "memory" && !c.ok) ? "low memory" : "no capacity"})` : "FAILED"}`,
|
|
1116
1130
|
...g.checks.map((c) => ` ${c.ok ? "✓" : "✗"} ${c.name}: ${c.detail.replace(/\n/g, "\n ")}`),
|
|
1117
1131
|
];
|
|
1118
1132
|
}
|
|
@@ -39,11 +39,30 @@ export declare function loadBudget(info?: () => Promise<{
|
|
|
39
39
|
}>): Promise<Budget>;
|
|
40
40
|
/** 1-min host load above which with-temp-home and a setup start wait (SW-209, SW-542, SW-630): `budget.host_load`, else 2 × cores; 0 is off. */
|
|
41
41
|
export declare function hostLoadLimit(cores: number): number;
|
|
42
|
+
/** % of memory available under which the gate and with-temp-home hold off (SW-620): `budget.free_memory`, else 15; 0 is off. */
|
|
43
|
+
export declare function freeMemoryLimit(): number;
|
|
44
|
+
/**
|
|
45
|
+
* Bytes available and total. On macOS os.freemem() counts only untouched pages (0.08 of 16 GiB on a
|
|
46
|
+
* healthy host), so it reads kern.memorystatus_level, the % the kernel calls available (SW-620).
|
|
47
|
+
*/
|
|
48
|
+
export declare function readHostMemory(platform?: NodeJS.Platform): {
|
|
49
|
+
free: number;
|
|
50
|
+
total: number;
|
|
51
|
+
};
|
|
52
|
+
/** "N% free, under budget.free_memory M%" when memory is low, else ""; unreadable memory is not low. */
|
|
53
|
+
export declare function lowMemory(hostMemory?: () => {
|
|
54
|
+
free: number;
|
|
55
|
+
total: number;
|
|
56
|
+
}): string;
|
|
42
57
|
export interface HostLoadDeps {
|
|
43
58
|
hostLoad: () => {
|
|
44
59
|
load: number[];
|
|
45
60
|
cores: number;
|
|
46
61
|
};
|
|
62
|
+
hostMemory: () => {
|
|
63
|
+
free: number;
|
|
64
|
+
total: number;
|
|
65
|
+
};
|
|
47
66
|
sleep: (ms: number) => Promise<unknown>;
|
|
48
67
|
}
|
|
49
68
|
export declare const readHostLoad: () => {
|
|
@@ -52,16 +71,18 @@ export declare const readHostLoad: () => {
|
|
|
52
71
|
};
|
|
53
72
|
/**
|
|
54
73
|
* The wait of `werkmeister internal wait-load`, which the generated with-temp-home runs first (SW-224):
|
|
55
|
-
* up to 90 s for the 1-minute load to fall under the limit
|
|
56
|
-
* each minute (SW-262); the note for the wrapper's stderr, or "".
|
|
57
|
-
*
|
|
58
|
-
*
|
|
74
|
+
* up to 90 s for the 1-minute load to fall under the limit and free memory to rise over its own (SW-620),
|
|
75
|
+
* saying so up front and printing progress each minute (SW-262); the note for the wrapper's stderr, or "".
|
|
76
|
+
* A spent wait's note starts "ran with". The gate stopped waiting in SW-542.
|
|
77
|
+
* Load not on macOS: its loadavg counts idle daemons in R state (450-550 on a quiet host) and never the
|
|
78
|
+
* Linux VM that runs the tests, so it is no signal to wait on (SW-556). Its memory is: Claude Code
|
|
79
|
+
* stops an idle background shell when the Mac runs low (SW-620).
|
|
59
80
|
*/
|
|
60
81
|
export declare function waitForHostLoadNow(deps?: HostLoadDeps, platform?: NodeJS.Platform, before?: string): Promise<string>;
|
|
61
82
|
/**
|
|
62
83
|
* Before compose starts a setup one-shot among *services* (undefined: what a whole `up` starts),
|
|
63
|
-
* wait on host load as with-temp-home does, and fail
|
|
64
|
-
* a setup install started at load ~413 ran 78 min, past compose's 20 min `--wait`, and ended 137.
|
|
84
|
+
* wait on host load and free memory as with-temp-home does, and fail once the wait is spent (SW-630,
|
|
85
|
+
* SW-620): a setup install started at load ~413 ran 78 min, past compose's 20 min `--wait`, and ended 137.
|
|
65
86
|
*
|
|
66
87
|
* Throws `HOST_LOAD`.
|
|
67
88
|
*/
|
package/dist/scheduler_state.js
CHANGED
|
@@ -11,6 +11,7 @@
|
|
|
11
11
|
* What is running, and what it costs, is never stored: `lifecycle.ts` reads it
|
|
12
12
|
* from Docker on every decision, because `up` exits once containers start.
|
|
13
13
|
*/
|
|
14
|
+
import { execFileSync } from "node:child_process";
|
|
14
15
|
import { createHash } from "node:crypto";
|
|
15
16
|
import * as fs from "node:fs";
|
|
16
17
|
import * as net from "node:net";
|
|
@@ -115,43 +116,106 @@ export function hostLoadLimit(cores) {
|
|
|
115
116
|
const v = isDict(cfg) ? Number(cfg["host_load"] ?? Number.NaN) : Number.NaN;
|
|
116
117
|
return Number.isFinite(v) && v >= 0 ? v : 2 * cores;
|
|
117
118
|
}
|
|
119
|
+
/** % of memory available under which the gate and with-temp-home hold off (SW-620): `budget.free_memory`, else 15; 0 is off. */
|
|
120
|
+
export function freeMemoryLimit() {
|
|
121
|
+
const cfg = loadGlobalConfig()["budget"];
|
|
122
|
+
const v = isDict(cfg) ? Number(cfg["free_memory"] ?? Number.NaN) : Number.NaN;
|
|
123
|
+
return Number.isFinite(v) && v >= 0 ? v : 15;
|
|
124
|
+
}
|
|
125
|
+
/**
|
|
126
|
+
* Bytes available and total. On macOS os.freemem() counts only untouched pages (0.08 of 16 GiB on a
|
|
127
|
+
* healthy host), so it reads kern.memorystatus_level, the % the kernel calls available (SW-620).
|
|
128
|
+
*/
|
|
129
|
+
export function readHostMemory(platform = process.platform) {
|
|
130
|
+
const total = os.totalmem();
|
|
131
|
+
if (platform !== "darwin")
|
|
132
|
+
return { free: os.freemem(), total };
|
|
133
|
+
const level = Number(execFileSync("sysctl", ["-n", "kern.memorystatus_level"], { encoding: "utf8" }));
|
|
134
|
+
return { free: (total * level) / 100, total };
|
|
135
|
+
}
|
|
136
|
+
/** "N% free, under budget.free_memory M%" when memory is low, else ""; unreadable memory is not low. */
|
|
137
|
+
export function lowMemory(hostMemory = readHostMemory) {
|
|
138
|
+
const limit = freeMemoryLimit();
|
|
139
|
+
if (limit === 0)
|
|
140
|
+
return "";
|
|
141
|
+
let pct;
|
|
142
|
+
try {
|
|
143
|
+
const { free, total } = hostMemory();
|
|
144
|
+
pct = (100 * free) / total;
|
|
145
|
+
}
|
|
146
|
+
catch {
|
|
147
|
+
return "";
|
|
148
|
+
}
|
|
149
|
+
return Number.isFinite(pct) && pct < limit
|
|
150
|
+
? `${pct.toFixed(0)}% free, under budget.free_memory ${limit}%`
|
|
151
|
+
: "";
|
|
152
|
+
}
|
|
118
153
|
const LOAD_POLL_MS = 15_000;
|
|
119
154
|
// with-temp-home's own wait: under an agent's default 2 min Bash timeout, so the call can't time out before its command starts (SW-469).
|
|
120
155
|
const WRAPPER_LOAD_WAIT_MS = 90_000;
|
|
121
156
|
const LOAD_PROGRESS_MS = 60_000;
|
|
122
157
|
export const readHostLoad = () => ({ load: os.loadavg(), cores: os.availableParallelism() });
|
|
123
|
-
const realHostLoad = {
|
|
158
|
+
const realHostLoad = {
|
|
159
|
+
hostLoad: readHostLoad,
|
|
160
|
+
hostMemory: readHostMemory,
|
|
161
|
+
sleep: (ms) => delay(ms),
|
|
162
|
+
};
|
|
124
163
|
const minutes = (ms) => Number((ms / 60_000).toFixed(1));
|
|
125
164
|
/**
|
|
126
165
|
* The wait of `werkmeister internal wait-load`, which the generated with-temp-home runs first (SW-224):
|
|
127
|
-
* up to 90 s for the 1-minute load to fall under the limit
|
|
128
|
-
* each minute (SW-262); the note for the wrapper's stderr, or "".
|
|
129
|
-
*
|
|
130
|
-
*
|
|
166
|
+
* up to 90 s for the 1-minute load to fall under the limit and free memory to rise over its own (SW-620),
|
|
167
|
+
* saying so up front and printing progress each minute (SW-262); the note for the wrapper's stderr, or "".
|
|
168
|
+
* A spent wait's note starts "ran with". The gate stopped waiting in SW-542.
|
|
169
|
+
* Load not on macOS: its loadavg counts idle daemons in R state (450-550 on a quiet host) and never the
|
|
170
|
+
* Linux VM that runs the tests, so it is no signal to wait on (SW-556). Its memory is: Claude Code
|
|
171
|
+
* stops an idle background shell when the Mac runs low (SW-620).
|
|
131
172
|
*/
|
|
132
173
|
export async function waitForHostLoadNow(deps = realHostLoad, platform = process.platform, before = "with-temp-home; --no-wait skips") {
|
|
133
174
|
// WERKMEISTER_NO_WAIT skips it inside the command too, not only in with-temp-home's shell (SW-425).
|
|
134
|
-
if (
|
|
175
|
+
if (envVar("NO_WAIT"))
|
|
135
176
|
return "";
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
177
|
+
const read = () => {
|
|
178
|
+
const { load, cores } = deps.hostLoad();
|
|
179
|
+
const limit = platform === "darwin" ? 0 : hostLoadLimit(cores);
|
|
180
|
+
return {
|
|
181
|
+
load: load[0],
|
|
182
|
+
limit,
|
|
183
|
+
loadOver: limit !== 0 && load[0] > limit,
|
|
184
|
+
memory: lowMemory(deps.hostMemory),
|
|
185
|
+
};
|
|
186
|
+
};
|
|
187
|
+
let now = read();
|
|
188
|
+
if (!now.loadOver && !now.memory)
|
|
139
189
|
return "";
|
|
140
|
-
|
|
190
|
+
const waitedFor = [now.loadOver && "host load", now.memory && "free memory"]
|
|
191
|
+
.filter(Boolean)
|
|
192
|
+
.join(" and ");
|
|
193
|
+
info(`waiting up to ${minutes(WRAPPER_LOAD_WAIT_MS)} min for ${[
|
|
194
|
+
now.loadOver && `host load ${now.load.toFixed(1)} to fall to ${now.limit} (budget.host_load)`,
|
|
195
|
+
now.memory && `free memory to rise (${now.memory})`,
|
|
196
|
+
]
|
|
197
|
+
.filter(Boolean)
|
|
198
|
+
.join(" and ")} before ${before}`);
|
|
199
|
+
const state = () => [
|
|
200
|
+
now.loadOver && `host load ${now.load.toFixed(1)} over ${now.limit}`,
|
|
201
|
+
now.memory && `low memory (${now.memory})`,
|
|
202
|
+
]
|
|
203
|
+
.filter(Boolean)
|
|
204
|
+
.join(", ");
|
|
141
205
|
for (let waited = LOAD_POLL_MS; waited <= WRAPPER_LOAD_WAIT_MS; waited += LOAD_POLL_MS) {
|
|
142
206
|
await deps.sleep(LOAD_POLL_MS);
|
|
143
|
-
|
|
144
|
-
if (
|
|
145
|
-
return `waited ${waited / 1000} s for
|
|
207
|
+
now = read();
|
|
208
|
+
if (!now.loadOver && !now.memory)
|
|
209
|
+
return `waited ${waited / 1000} s for ${waitedFor}`;
|
|
146
210
|
if (waited % LOAD_PROGRESS_MS === 0 && waited < WRAPPER_LOAD_WAIT_MS)
|
|
147
|
-
info(`still waiting
|
|
211
|
+
info(`still waiting: ${state()}, ${waited / 1000} s so far, up to ${minutes(WRAPPER_LOAD_WAIT_MS - waited)} min left`);
|
|
148
212
|
}
|
|
149
|
-
return `ran with
|
|
213
|
+
return `ran with ${state()}; the ${minutes(WRAPPER_LOAD_WAIT_MS)} min host-load wait is spent`;
|
|
150
214
|
}
|
|
151
215
|
/**
|
|
152
216
|
* Before compose starts a setup one-shot among *services* (undefined: what a whole `up` starts),
|
|
153
|
-
* wait on host load as with-temp-home does, and fail
|
|
154
|
-
* a setup install started at load ~413 ran 78 min, past compose's 20 min `--wait`, and ended 137.
|
|
217
|
+
* wait on host load and free memory as with-temp-home does, and fail once the wait is spent (SW-630,
|
|
218
|
+
* SW-620): a setup install started at load ~413 ran 78 min, past compose's 20 min `--wait`, and ended 137.
|
|
155
219
|
*
|
|
156
220
|
* Throws `HOST_LOAD`.
|
|
157
221
|
*/
|
|
@@ -167,9 +231,19 @@ export async function waitForSetupLoad(workspace, services, deps = realHostLoad,
|
|
|
167
231
|
if (!note.startsWith("ran with"))
|
|
168
232
|
return;
|
|
169
233
|
const { load, cores } = deps.hostLoad();
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
234
|
+
const limit = hostLoadLimit(cores);
|
|
235
|
+
const memory = lowMemory(deps.hostMemory);
|
|
236
|
+
const over = [
|
|
237
|
+
platform !== "darwin" &&
|
|
238
|
+
limit !== 0 &&
|
|
239
|
+
load[0] > limit &&
|
|
240
|
+
`host load ${load[0].toFixed(1)} is over ${limit} (budget.host_load)`,
|
|
241
|
+
memory && `memory is low (${memory})`,
|
|
242
|
+
]
|
|
243
|
+
.filter(Boolean)
|
|
244
|
+
.join(" and ");
|
|
245
|
+
throw new SandboxError(`${over || note} after the ${minutes(WRAPPER_LOAD_WAIT_MS)} min wait; ${what} not started. ` +
|
|
246
|
+
"Retry when load falls or memory frees, or set WERKMEISTER_NO_WAIT=1 to start anyway.", "HOST_LOAD");
|
|
173
247
|
}
|
|
174
248
|
function composeServices(workspace, composePath = path.join(workspace, COMPOSE_FILE)) {
|
|
175
249
|
try {
|
package/package.json
CHANGED