start-command 0.36.0 → 0.37.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +14 -0
- package/README.md +6 -0
- package/package.json +1 -1
- package/src/bin/cli.js +5 -0
- package/src/lib/args-parser-queries.js +14 -0
- package/src/lib/args-parser.js +20 -1
- package/src/lib/command-builder.js +3 -0
- package/src/lib/cpu-penalty-monitor.js +209 -87
- package/src/lib/cpu-penalty.js +30 -1
- package/src/lib/docker-cleanup.js +12 -5
- package/src/lib/docker-command-handoff.js +75 -0
- package/src/lib/docker-labels.js +33 -0
- package/src/lib/docker-runtime-args.js +8 -0
- package/src/lib/docker-snapshot-safety.js +164 -0
- package/src/lib/execution-recovery.js +8 -1
- package/src/lib/execution-resume.js +184 -37
- package/src/lib/failure-handler.js +15 -5
- package/src/lib/isolation.js +20 -4
- package/src/lib/log-sanitizer.js +267 -0
- package/src/lib/log-uploader.js +38 -19
- package/src/lib/query-commands.js +4 -3
- package/src/lib/resume-resources.js +15 -3
- package/src/lib/shell-utils.js +67 -17
- package/src/lib/store-lock.js +54 -3
- package/src/lib/usage.js +5 -1
- package/test/ci-workflow-invariants.js +15 -14
- package/test/docker-command-handoff.js +107 -0
- package/test/docker-runtime-options.js +4 -0
- package/test/docker-snapshot-safety.js +106 -0
- package/test/execution-resume.js +10 -8
- package/test/failure-handler.js +11 -0
- package/test/javascript-first-ci.js +216 -0
- package/test/log-sanitizer.js +306 -0
- package/test/regression-176.js +7 -0
- package/test/regression-187.js +9 -1
- package/test/regression-191.js +9 -1
- package/test/regression-193.js +11 -1
- package/test/regression-198.js +281 -0
- package/test/regression-199.js +79 -0
- package/test/regression-201.js +25 -0
- package/test/regression-202.js +92 -0
- package/test/regression-207.js +249 -0
- package/test/regression-91.js +23 -20
- package/test/shared-golden.js +77 -0
- package/test/status-query.js +97 -15
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,19 @@
|
|
|
1
1
|
# start-command
|
|
2
2
|
|
|
3
|
+
## 0.37.1
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- c67c92f: Observe Docker CPU limits applied after launch, restore the remembered base on lift and resume, and recover CPU penalty monitors after crashes, zombie/reused-PID locks and transient tick failures.
|
|
8
|
+
|
|
9
|
+
## 0.37.0
|
|
10
|
+
|
|
11
|
+
### Minor Changes
|
|
12
|
+
|
|
13
|
+
- 94db203: Add copy-free Docker command resumes with guarded legacy snapshots, persistent
|
|
14
|
+
task labels, wrapper help flags, private sanitized log uploads, and faithful shell
|
|
15
|
+
argument and exit-code handling. Enforce JavaScript-first CI and shared Rust parity.
|
|
16
|
+
|
|
3
17
|
## 0.36.0
|
|
4
18
|
|
|
5
19
|
### Minor Changes
|
package/README.md
CHANGED
|
@@ -24,6 +24,8 @@ $ ls -la
|
|
|
24
24
|
$ bun test
|
|
25
25
|
$ git status
|
|
26
26
|
$ --list
|
|
27
|
+
$ --help
|
|
28
|
+
$ -i docker -d --label task.tool=codex -- my-task
|
|
27
29
|
$ --attach <id>
|
|
28
30
|
$ --resume <id> -- <command>
|
|
29
31
|
$ --resume-all
|
|
@@ -34,6 +36,10 @@ See the project-wide [README](../README.md), [docs/USAGE.md](../docs/USAGE.md),
|
|
|
34
36
|
[docs/EXAMPLES.md](../docs/EXAMPLES.md) for the full user-facing guide and
|
|
35
37
|
checked examples.
|
|
36
38
|
|
|
39
|
+
`-h` also prints usage. Labels persist across Docker resumes. New detached
|
|
40
|
+
containers support copy-free command handoff; legacy snapshots require capacity
|
|
41
|
+
preflight. Log uploads are private and sanitized; manual `--no-sanitize` opts out.
|
|
42
|
+
|
|
37
43
|
## Development
|
|
38
44
|
|
|
39
45
|
```bash
|
package/package.json
CHANGED
package/src/bin/cli.js
CHANGED
|
@@ -192,6 +192,10 @@ try {
|
|
|
192
192
|
}
|
|
193
193
|
|
|
194
194
|
const { wrapperOptions, command: parsedCommand } = parsedArgs;
|
|
195
|
+
if (wrapperOptions.help) {
|
|
196
|
+
printUsage();
|
|
197
|
+
process.exit(0);
|
|
198
|
+
}
|
|
195
199
|
|
|
196
200
|
// Query/control modes (--status, --list, --attach, --resume, ...) address an
|
|
197
201
|
// execution that already exists, so they never need a command of their own.
|
|
@@ -479,6 +483,7 @@ async function runWithIsolation(
|
|
|
479
483
|
result = await runIsolated(environment, cmd, {
|
|
480
484
|
...options,
|
|
481
485
|
session: sessionName,
|
|
486
|
+
sessionId,
|
|
482
487
|
image: effectiveImage,
|
|
483
488
|
endpoint: options.endpoint,
|
|
484
489
|
detached: mode === 'detached',
|
|
@@ -26,6 +26,8 @@ function createQueryOptionDefaults() {
|
|
|
26
26
|
list: false, // List all tracked execution records
|
|
27
27
|
running: false, // Restrict --list to executions that are still running
|
|
28
28
|
uploadLog: null, // UUID/session name whose stored log should be uploaded
|
|
29
|
+
noSanitize: false,
|
|
30
|
+
removeOriginal: false,
|
|
29
31
|
outputFormat: null, // Output format (links-notation, json, text)
|
|
30
32
|
stop: null, // UUID/session name to stop gracefully
|
|
31
33
|
terminate: null, // UUID/session name to terminate immediately
|
|
@@ -55,6 +57,12 @@ const IDENTIFIER_OPTIONS = {
|
|
|
55
57
|
* Boolean flags with no argument.
|
|
56
58
|
*/
|
|
57
59
|
const FLAG_OPTIONS = {
|
|
60
|
+
'--no-sanitize': (options) => {
|
|
61
|
+
options.noSanitize = true;
|
|
62
|
+
},
|
|
63
|
+
'--remove-original': (options) => {
|
|
64
|
+
options.removeOriginal = true;
|
|
65
|
+
},
|
|
58
66
|
'--list': (options) => {
|
|
59
67
|
options.list = true;
|
|
60
68
|
},
|
|
@@ -185,6 +193,12 @@ const OUTPUT_FORMAT_MODES = ['--status', '--list', '--resume', '--resume-all'];
|
|
|
185
193
|
* @throws {Error} When the combination is not supported
|
|
186
194
|
*/
|
|
187
195
|
function validateQueryOptions(options) {
|
|
196
|
+
if (options.noSanitize && !hasValue(options.uploadLog)) {
|
|
197
|
+
throw new Error('--no-sanitize is only valid with --upload-log');
|
|
198
|
+
}
|
|
199
|
+
if (options.removeOriginal && !hasValue(options.resume)) {
|
|
200
|
+
throw new Error('--remove-original is only valid with --resume');
|
|
201
|
+
}
|
|
188
202
|
if (hasValue(options.outputFormat)) {
|
|
189
203
|
if (!VALID_OUTPUT_FORMATS.includes(options.outputFormat)) {
|
|
190
204
|
throw new Error(
|
package/src/lib/args-parser.js
CHANGED
|
@@ -193,6 +193,8 @@ function parseArgs(args) {
|
|
|
193
193
|
volumes: [], // Docker bind mounts/volumes (-v/--volume), applied to docker levels
|
|
194
194
|
mounts: [], // Docker --mount specs, applied to docker levels
|
|
195
195
|
env: [], // Docker environment variables (-e/--env, KEY=VALUE), applied to docker levels
|
|
196
|
+
labels: [], // Caller attribution, applied to every Docker level
|
|
197
|
+
help: false,
|
|
196
198
|
privileged: false, // Run docker container in privileged mode
|
|
197
199
|
network: null, // First Docker network name (compatibility accessor)
|
|
198
200
|
networks: [], // Ordered Docker network names
|
|
@@ -247,7 +249,9 @@ function parseArgs(args) {
|
|
|
247
249
|
}
|
|
248
250
|
|
|
249
251
|
// Validate options
|
|
250
|
-
|
|
252
|
+
if (!wrapperOptions.help) {
|
|
253
|
+
validateOptions(wrapperOptions);
|
|
254
|
+
}
|
|
251
255
|
|
|
252
256
|
return {
|
|
253
257
|
wrapperOptions,
|
|
@@ -288,6 +292,16 @@ function parseWrapperArgs(args, options) {
|
|
|
288
292
|
*/
|
|
289
293
|
function parseOption(args, index, options) {
|
|
290
294
|
const arg = args[index];
|
|
295
|
+
if (arg === '--help' || arg === '-h') {
|
|
296
|
+
options.help = true;
|
|
297
|
+
return 1;
|
|
298
|
+
}
|
|
299
|
+
if (arg === '--label' || arg.startsWith('--label=')) {
|
|
300
|
+
const label = arg === '--label' ? args[index + 1] : arg.slice(8);
|
|
301
|
+
require('./docker-labels').validateLabel(label);
|
|
302
|
+
options.labels.push(label);
|
|
303
|
+
return arg === '--label' ? 2 : 1;
|
|
304
|
+
}
|
|
291
305
|
|
|
292
306
|
// --isolated, --isolation, or -i
|
|
293
307
|
if (arg === '--isolated' || arg === '--isolation' || arg === '-i') {
|
|
@@ -568,6 +582,11 @@ function parseOption(args, index, options) {
|
|
|
568
582
|
* @throws {Error} If a docker-only option is set without docker isolation
|
|
569
583
|
*/
|
|
570
584
|
function validateDockerRuntimeOptionsRequireDocker(options) {
|
|
585
|
+
if (options.labels?.length > 0 && !options.resume) {
|
|
586
|
+
throw new Error(
|
|
587
|
+
'--label option is only valid with docker isolation or --resume'
|
|
588
|
+
);
|
|
589
|
+
}
|
|
571
590
|
if (options.volumes && options.volumes.length > 0) {
|
|
572
591
|
throw new Error(
|
|
573
592
|
'--volume option is only valid when isolation stack includes docker'
|
|
@@ -106,6 +106,9 @@ function buildNextLevelCommand(options, command) {
|
|
|
106
106
|
for (const envVar of options.env || []) {
|
|
107
107
|
parts.push(`--env "${envVar}"`);
|
|
108
108
|
}
|
|
109
|
+
for (const label of options.labels || []) {
|
|
110
|
+
parts.push(`--label ${require('./shell-utils').quoteShellArg(label)}`);
|
|
111
|
+
}
|
|
109
112
|
if (options.privileged) {
|
|
110
113
|
parts.push('--privileged');
|
|
111
114
|
}
|
|
@@ -2,10 +2,20 @@
|
|
|
2
2
|
const fs = require('fs');
|
|
3
3
|
const path = require('path');
|
|
4
4
|
const { spawnSync } = require('child_process');
|
|
5
|
-
const {
|
|
5
|
+
const {
|
|
6
|
+
initialState,
|
|
7
|
+
evaluate,
|
|
8
|
+
observeBase,
|
|
9
|
+
transitionLog,
|
|
10
|
+
} = require('./cpu-penalty');
|
|
6
11
|
const { LockManager } = require('./store-lock');
|
|
7
12
|
const { shellQuote } = require('./isolation-log-utils');
|
|
8
|
-
const {
|
|
13
|
+
const {
|
|
14
|
+
cpuCount,
|
|
15
|
+
cpuUpdateArgs,
|
|
16
|
+
normalizeResourceLimits,
|
|
17
|
+
readDockerResourceLimits,
|
|
18
|
+
} = require('./docker-resource-limits');
|
|
9
19
|
function statePath(record) {
|
|
10
20
|
return `${record.logPath}.cpu-penalty.json`;
|
|
11
21
|
}
|
|
@@ -40,9 +50,21 @@ function publicState(state) {
|
|
|
40
50
|
}
|
|
41
51
|
function run(command, args) {
|
|
42
52
|
const result = spawnSync(command, args, { encoding: 'utf8' });
|
|
43
|
-
return {
|
|
53
|
+
return {
|
|
54
|
+
success: result.status === 0,
|
|
55
|
+
stdout: result.stdout || '',
|
|
56
|
+
stderr: result.stderr || '',
|
|
57
|
+
};
|
|
44
58
|
}
|
|
45
|
-
async function monitor(
|
|
59
|
+
async function monitor(
|
|
60
|
+
uuid,
|
|
61
|
+
name,
|
|
62
|
+
attemptNumber,
|
|
63
|
+
standalone,
|
|
64
|
+
dependencies = {}
|
|
65
|
+
) {
|
|
66
|
+
const filesystem = dependencies.fs || fs;
|
|
67
|
+
const runner = dependencies.run || run;
|
|
46
68
|
const { ExecutionStore } = require('./execution-store');
|
|
47
69
|
const store = uuid
|
|
48
70
|
? new ExecutionStore(
|
|
@@ -60,36 +82,40 @@ async function monitor(uuid, name, attemptNumber, standalone) {
|
|
|
60
82
|
return;
|
|
61
83
|
}
|
|
62
84
|
const config = record.options.cpuPenaltyConfig;
|
|
63
|
-
|
|
85
|
+
if ((record.attempt?.number || 1) !== Number(attemptNumber)) {
|
|
86
|
+
return;
|
|
87
|
+
}
|
|
88
|
+
const lock = new LockManager(`${statePath(record)}.lock`, filesystem, true);
|
|
64
89
|
if (!lock.acquire(100)) {
|
|
65
90
|
return;
|
|
66
91
|
}
|
|
67
92
|
const docker = require('./docker-cleanup').getDockerCommand();
|
|
68
|
-
|
|
69
|
-
record.options.baseResourceLimits || record.options.resourceLimits
|
|
93
|
+
let baseFlags = normalizeResourceLimits(
|
|
94
|
+
record.options.baseResourceLimits || record.options.resourceLimits
|
|
95
|
+
);
|
|
70
96
|
const baseCpus = cpuCount(baseFlags);
|
|
71
97
|
let state = initialState(baseCpus, Date.now(), readState(record));
|
|
72
98
|
const interval = Math.min(
|
|
73
99
|
30000,
|
|
74
100
|
Math.max(50, Math.min(config.triggerWindowMs, config.releaseWindowMs) / 8)
|
|
75
101
|
);
|
|
76
|
-
const publish = () => {
|
|
102
|
+
const publish = (value = state) => {
|
|
77
103
|
const file = statePath(record),
|
|
78
104
|
tmp = `${file}.${process.pid}.tmp`;
|
|
79
105
|
try {
|
|
80
|
-
|
|
106
|
+
filesystem.writeFileSync(
|
|
81
107
|
tmp,
|
|
82
108
|
JSON.stringify({
|
|
83
|
-
...publicState(
|
|
109
|
+
...publicState(value),
|
|
84
110
|
containerName: name,
|
|
85
111
|
attemptNumber: Number(attemptNumber),
|
|
86
112
|
updatedAt: Date.now(),
|
|
87
113
|
})
|
|
88
114
|
);
|
|
89
|
-
|
|
115
|
+
filesystem.renameSync(tmp, file);
|
|
90
116
|
} finally {
|
|
91
117
|
try {
|
|
92
|
-
|
|
118
|
+
filesystem.unlinkSync(tmp);
|
|
93
119
|
} catch {
|
|
94
120
|
/* Optional diagnostics are best effort. */
|
|
95
121
|
}
|
|
@@ -97,95 +123,188 @@ async function monitor(uuid, name, attemptNumber, standalone) {
|
|
|
97
123
|
};
|
|
98
124
|
const log = (text) => {
|
|
99
125
|
try {
|
|
100
|
-
|
|
126
|
+
filesystem.appendFileSync(record.logPath, `${text}\n`);
|
|
101
127
|
} catch {
|
|
102
128
|
/* Optional diagnostics are best effort. */
|
|
103
129
|
}
|
|
104
130
|
};
|
|
105
|
-
const
|
|
131
|
+
const debug = (error) => {
|
|
132
|
+
if (process.env.START_DEBUG === '1') {
|
|
133
|
+
const message = `[DEBUG] CPU monitor: ${error.stack || error}`;
|
|
134
|
+
console.error(message);
|
|
135
|
+
log(message);
|
|
136
|
+
}
|
|
137
|
+
};
|
|
138
|
+
let stopping = false,
|
|
139
|
+
wake;
|
|
140
|
+
const stop = () => {
|
|
141
|
+
stopping = true;
|
|
142
|
+
wake?.();
|
|
143
|
+
};
|
|
144
|
+
process.on('SIGTERM', stop);
|
|
145
|
+
process.on('SIGINT', stop);
|
|
106
146
|
try {
|
|
107
|
-
|
|
108
|
-
while (true) {
|
|
109
|
-
const current = store ? store.get(uuid) : record;
|
|
110
|
-
if (
|
|
111
|
-
!current ||
|
|
112
|
-
current.options.sessionName !== name ||
|
|
113
|
-
(current.attempt?.number || 1) !== Number(attemptNumber)
|
|
114
|
-
) {
|
|
115
|
-
break;
|
|
116
|
-
}
|
|
117
|
-
const live = run(docker, ['inspect', '-f', '{{.State.Running}}', name]);
|
|
118
|
-
if (live.success && live.stdout.trim() === 'false') {
|
|
119
|
-
break;
|
|
120
|
-
}
|
|
121
|
-
if (!live.success && Date.now() - started > 30000) {
|
|
122
|
-
break;
|
|
123
|
-
}
|
|
124
|
-
const info = run(docker, ['info', '--format', '{{json .}}']);
|
|
125
|
-
const stats = run(docker, [
|
|
126
|
-
'stats',
|
|
127
|
-
'--no-stream',
|
|
128
|
-
'--format',
|
|
129
|
-
'{{json .}}',
|
|
130
|
-
]);
|
|
131
|
-
let daemonCpus = NaN,
|
|
132
|
-
cores = NaN;
|
|
147
|
+
while (!stopping) {
|
|
133
148
|
try {
|
|
134
|
-
|
|
135
|
-
|
|
149
|
+
const current = store ? store.get(uuid) : record;
|
|
150
|
+
if (
|
|
151
|
+
!current ||
|
|
152
|
+
current.options.sessionName !== name ||
|
|
153
|
+
(current.attempt?.number || 1) !== Number(attemptNumber)
|
|
154
|
+
) {
|
|
155
|
+
break;
|
|
156
|
+
}
|
|
157
|
+
const live = runner(docker, [
|
|
158
|
+
'inspect',
|
|
159
|
+
'-f',
|
|
160
|
+
'{{.State.Running}}',
|
|
161
|
+
name,
|
|
162
|
+
]);
|
|
163
|
+
if (live.success && live.stdout.trim() === 'false') {
|
|
164
|
+
break;
|
|
165
|
+
}
|
|
166
|
+
if (
|
|
167
|
+
!live.success &&
|
|
168
|
+
/No such (object|container)/i.test(live.stderr || '')
|
|
169
|
+
) {
|
|
170
|
+
break;
|
|
171
|
+
}
|
|
172
|
+
const observed = readDockerResourceLimits(name, runner);
|
|
173
|
+
if (observed !== null) {
|
|
174
|
+
baseFlags = observed;
|
|
175
|
+
state = observeBase(state, cpuCount(observed), Date.now());
|
|
136
176
|
}
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
177
|
+
const info = runner(docker, ['info', '--format', '{{json .}}']);
|
|
178
|
+
const stats = runner(docker, [
|
|
179
|
+
'stats',
|
|
180
|
+
'--no-stream',
|
|
181
|
+
'--format',
|
|
182
|
+
'{{json .}}',
|
|
183
|
+
]);
|
|
184
|
+
let daemonCpus = NaN,
|
|
185
|
+
cores = NaN;
|
|
186
|
+
try {
|
|
187
|
+
if (info.success) {
|
|
188
|
+
daemonCpus = JSON.parse(info.stdout).NCPU;
|
|
189
|
+
}
|
|
190
|
+
if (stats.success && live.success && observed !== null) {
|
|
191
|
+
const row = stats.stdout
|
|
192
|
+
.trim()
|
|
193
|
+
.split('\n')
|
|
194
|
+
.map((s) => JSON.parse(s))
|
|
195
|
+
.find((s) => s.Name === name);
|
|
196
|
+
if (row && /^\d+(?:\.\d+)?%$/.test(row.CPUPerc)) {
|
|
197
|
+
cores = Number(row.CPUPerc.slice(0, -1)) / 100;
|
|
198
|
+
}
|
|
145
199
|
}
|
|
200
|
+
} catch (error) {
|
|
201
|
+
debug(error);
|
|
146
202
|
}
|
|
147
|
-
|
|
148
|
-
|
|
203
|
+
debug(
|
|
204
|
+
`sample: phase=${state.phase} base=${state.baseCpus} limit=${state.limitCpus} cores=${cores} daemon=${daemonCpus}`
|
|
205
|
+
);
|
|
206
|
+
const decision = evaluate(
|
|
207
|
+
state,
|
|
208
|
+
{
|
|
209
|
+
now: Date.now(),
|
|
210
|
+
cores,
|
|
211
|
+
daemonCpus,
|
|
212
|
+
maxGapMs: Math.max(5000, interval * 3),
|
|
213
|
+
},
|
|
214
|
+
config
|
|
215
|
+
);
|
|
216
|
+
if (decision.action) {
|
|
217
|
+
const action = decision.action;
|
|
218
|
+
// Persist the base and intent before applying a cap. If this fails,
|
|
219
|
+
// leave Docker alone; a killed monitor can always recover its base.
|
|
220
|
+
if (action.kind === 'apply') {
|
|
221
|
+
publish(decision.state);
|
|
222
|
+
}
|
|
223
|
+
const update = runner(docker, [
|
|
224
|
+
'update',
|
|
225
|
+
...cpuUpdateArgs(baseFlags, action.cpus),
|
|
226
|
+
name,
|
|
227
|
+
]);
|
|
228
|
+
if (update.success) {
|
|
229
|
+
state = decision.state;
|
|
230
|
+
log(transitionLog(action, config));
|
|
231
|
+
} else {
|
|
232
|
+
debug(update.stderr || 'docker update failed');
|
|
233
|
+
state = {
|
|
234
|
+
...decision.state,
|
|
235
|
+
phase: state.phase,
|
|
236
|
+
since: state.since,
|
|
237
|
+
limitCpus: state.limitCpus,
|
|
238
|
+
penaltyCount: state.penaltyCount,
|
|
239
|
+
samples: [],
|
|
240
|
+
};
|
|
241
|
+
}
|
|
242
|
+
} else {
|
|
243
|
+
state = decision.state;
|
|
244
|
+
}
|
|
245
|
+
publish();
|
|
246
|
+
} catch (error) {
|
|
247
|
+
state.samples = [];
|
|
248
|
+
debug(`tick failed: ${error.stack || error}`);
|
|
149
249
|
}
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
{
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
);
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
250
|
+
await new Promise((resolve) => {
|
|
251
|
+
const timer = globalThis.setTimeout(resolve, interval);
|
|
252
|
+
wake = () => {
|
|
253
|
+
globalThis.clearTimeout(timer);
|
|
254
|
+
resolve();
|
|
255
|
+
};
|
|
256
|
+
if (stopping) {
|
|
257
|
+
wake();
|
|
258
|
+
}
|
|
259
|
+
});
|
|
260
|
+
}
|
|
261
|
+
} finally {
|
|
262
|
+
try {
|
|
263
|
+
const current = store ? store.get(uuid) : record;
|
|
264
|
+
if (
|
|
265
|
+
state.phase === 'penalized' &&
|
|
266
|
+
current?.options.sessionName === name &&
|
|
267
|
+
(current.attempt?.number || 1) === Number(attemptNumber)
|
|
268
|
+
) {
|
|
269
|
+
const live = runner(docker, [
|
|
270
|
+
'inspect',
|
|
271
|
+
'-f',
|
|
272
|
+
'{{.State.Running}}',
|
|
165
273
|
name,
|
|
166
274
|
]);
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
275
|
+
const observed = readDockerResourceLimits(name, runner);
|
|
276
|
+
const info = runner(docker, ['info', '--format', '{{json .}}']);
|
|
277
|
+
const capacity = info.success ? JSON.parse(info.stdout).NCPU : NaN;
|
|
278
|
+
if (
|
|
279
|
+
live.success &&
|
|
280
|
+
live.stdout.trim() === 'true' &&
|
|
281
|
+
observed !== null &&
|
|
282
|
+
cpuCount(observed) === state.limitCpus &&
|
|
283
|
+
capacity > 0
|
|
284
|
+
) {
|
|
285
|
+
const cpus = Math.min(state.baseCpus || capacity, capacity);
|
|
286
|
+
const update = runner(docker, [
|
|
287
|
+
'update',
|
|
288
|
+
...cpuUpdateArgs(observed, cpus),
|
|
289
|
+
name,
|
|
290
|
+
]);
|
|
291
|
+
if (update.success) {
|
|
292
|
+
state = {
|
|
293
|
+
...state,
|
|
294
|
+
phase: 'observing',
|
|
295
|
+
limitCpus: cpus,
|
|
296
|
+
since: Date.now(),
|
|
297
|
+
};
|
|
298
|
+
log(`[start] CPU penalty lifted: ${cpus} CPUs (monitor stopped)`);
|
|
299
|
+
publish();
|
|
300
|
+
}
|
|
181
301
|
}
|
|
182
|
-
} else {
|
|
183
|
-
state = decision.state;
|
|
184
302
|
}
|
|
185
|
-
|
|
186
|
-
|
|
303
|
+
} catch (error) {
|
|
304
|
+
debug(error);
|
|
187
305
|
}
|
|
188
|
-
|
|
306
|
+
process.off('SIGTERM', stop);
|
|
307
|
+
process.off('SIGINT', stop);
|
|
189
308
|
lock.release();
|
|
190
309
|
}
|
|
191
310
|
}
|
|
@@ -204,7 +323,9 @@ function startSnippet(uuid, name, attempt = 1, options = null) {
|
|
|
204
323
|
if (!uuid && !standalone) {
|
|
205
324
|
return '';
|
|
206
325
|
}
|
|
207
|
-
|
|
326
|
+
const command = `${shellQuote(process.execPath)} ${shellQuote(__filename)} ${shellQuote(uuid || '')} ${shellQuote(name)} ${Number(attempt)} ${shellQuote(standalone || '')}`;
|
|
327
|
+
// wait reaps a crashed child even in containers whose PID 1 never reaps.
|
|
328
|
+
return `( __start_cpu_child=; trap 'if [ -n "$__start_cpu_child" ]; then kill "$__start_cpu_child" 2>/dev/null; wait "$__start_cpu_child" 2>/dev/null; fi; exit 0' TERM INT HUP; while :; do ${command} >/dev/null 2>&1 & __start_cpu_child=$!; wait "$__start_cpu_child"; __start_cpu_code=$?; [ "$__start_cpu_code" -eq 0 ] && break; sleep 1 & __start_cpu_child=$!; wait "$__start_cpu_child"; done ) & __start_cpu_monitor=$!`;
|
|
208
329
|
}
|
|
209
330
|
function stopSnippet() {
|
|
210
331
|
return 'if [ -n "$__start_cpu_monitor" ]; then kill "$__start_cpu_monitor" 2>/dev/null; wait "$__start_cpu_monitor" 2>/dev/null; fi';
|
|
@@ -219,6 +340,7 @@ module.exports = {
|
|
|
219
340
|
};
|
|
220
341
|
if (require.main === module) {
|
|
221
342
|
monitor(...process.argv.slice(2)).catch((error) => {
|
|
343
|
+
process.exitCode = 1;
|
|
222
344
|
if (process.env.START_DEBUG === '1') {
|
|
223
345
|
console.error(`[DEBUG] CPU monitor: ${error.stack}`);
|
|
224
346
|
}
|
package/src/lib/cpu-penalty.js
CHANGED
|
@@ -70,10 +70,13 @@ function validate(options) {
|
|
|
70
70
|
options.cpuPenaltyConfig = { ...DEFAULTS, ...options.cpuPenaltyConfig };
|
|
71
71
|
}
|
|
72
72
|
function initialState(baseCpus = null, now = Date.now(), saved = null) {
|
|
73
|
+
baseCpus =
|
|
74
|
+
saved && Object.hasOwn(saved, 'baseCpus') ? saved.baseCpus : baseCpus;
|
|
73
75
|
return {
|
|
74
76
|
phase: saved?.phase === 'penalized' ? 'penalized' : 'observing',
|
|
75
77
|
since: saved?.phase === 'penalized' ? saved.since : now,
|
|
76
|
-
limitCpus:
|
|
78
|
+
limitCpus:
|
|
79
|
+
saved && Object.hasOwn(saved, 'limitCpus') ? saved.limitCpus : baseCpus,
|
|
77
80
|
baseCpus,
|
|
78
81
|
penaltyCount: saved?.penaltyCount || 0,
|
|
79
82
|
penalizedMs: saved?.penalizedMs || 0,
|
|
@@ -82,6 +85,30 @@ function initialState(baseCpus = null, now = Date.now(), saved = null) {
|
|
|
82
85
|
capacity: null,
|
|
83
86
|
};
|
|
84
87
|
}
|
|
88
|
+
// A quota different from our cap is a supervisor override, not our penalty.
|
|
89
|
+
function observeBase(state, baseCpus, now) {
|
|
90
|
+
if (baseCpus === state.limitCpus) {
|
|
91
|
+
return state;
|
|
92
|
+
}
|
|
93
|
+
return {
|
|
94
|
+
...state,
|
|
95
|
+
phase: 'observing',
|
|
96
|
+
baseCpus,
|
|
97
|
+
limitCpus: baseCpus,
|
|
98
|
+
since: now,
|
|
99
|
+
samples: [],
|
|
100
|
+
};
|
|
101
|
+
}
|
|
102
|
+
function transitionLog(action, config) {
|
|
103
|
+
const applying = action.kind === 'apply';
|
|
104
|
+
const window = applying ? config.triggerWindowMs : config.releaseWindowMs;
|
|
105
|
+
const duration = window % 60000 === 0 ? `${window / 60000}m` : `${window}ms`;
|
|
106
|
+
const measured =
|
|
107
|
+
action.average === null || action.average === undefined
|
|
108
|
+
? 'capacity changed'
|
|
109
|
+
: `${action.average.toFixed(2)} of ${applying ? action.capacity : config.cpus} CPUs for ${duration}`;
|
|
110
|
+
return `[start] CPU penalty ${applying ? 'applied' : 'lifted'}: ${action.cpus} CPUs (${measured})`;
|
|
111
|
+
}
|
|
85
112
|
function average(samples, now, window) {
|
|
86
113
|
const start = now - window;
|
|
87
114
|
if (
|
|
@@ -170,6 +197,8 @@ module.exports = {
|
|
|
170
197
|
validate,
|
|
171
198
|
duration,
|
|
172
199
|
initialState,
|
|
200
|
+
observeBase,
|
|
201
|
+
transitionLog,
|
|
173
202
|
average,
|
|
174
203
|
evaluate,
|
|
175
204
|
};
|
|
@@ -83,7 +83,7 @@ function getDockerContainerCleanupInstructions(containerName) {
|
|
|
83
83
|
`Container kept for investigation: ${containerName}`,
|
|
84
84
|
`Re-enter while running: $ --attach ${containerName}`,
|
|
85
85
|
`Continue the stored command: $ --resume ${containerName}`,
|
|
86
|
-
`Run another command
|
|
86
|
+
`Run another command (legacy containers snapshot the filesystem into a new container): $ --resume ${containerName} -- <command>`,
|
|
87
87
|
`Remove when done: docker rm -f ${containerName}`,
|
|
88
88
|
].join('\n');
|
|
89
89
|
}
|
|
@@ -306,7 +306,7 @@ function buildDockerKeptLogSnippet(containerName, quotedLogPath) {
|
|
|
306
306
|
`printf '\\nContainer kept for investigation: %s\\nReason: %s\\n` +
|
|
307
307
|
`Re-enter while running: $ --attach %s\\n` +
|
|
308
308
|
`Continue the stored command: $ --resume %s\\n` +
|
|
309
|
-
`Run another command
|
|
309
|
+
`Run another command (legacy containers snapshot the filesystem into a new container): $ --resume %s -- <command>\\n` +
|
|
310
310
|
`Remove when done: docker rm -f %s\\n' ` +
|
|
311
311
|
`${quotedName} "$__start_command_reason" ` +
|
|
312
312
|
`${quotedName} ${quotedName} ${quotedName} ${quotedName} >> ${quotedLogPath}`
|
|
@@ -361,6 +361,11 @@ function buildDetachedDockerCompletionScript(
|
|
|
361
361
|
watcherOptions = {}
|
|
362
362
|
) {
|
|
363
363
|
const quotedName = shellQuote(containerName);
|
|
364
|
+
const imageCleanup =
|
|
365
|
+
require('./docker-snapshot-safety').snapshotImageCleanupSnippet(
|
|
366
|
+
containerName,
|
|
367
|
+
logPath ? `>> ${shellQuote(logPath)} 2>&1` : '>/dev/null 2>&1'
|
|
368
|
+
);
|
|
364
369
|
// The container's cgroup disappears with it, so its memory counters are
|
|
365
370
|
// sampled while it runs (issue #182).
|
|
366
371
|
const cpuMonitor = require('./cpu-penalty-monitor');
|
|
@@ -389,7 +394,7 @@ function buildDetachedDockerCompletionScript(
|
|
|
389
394
|
quotedLogPath
|
|
390
395
|
);
|
|
391
396
|
const memory = buildCgroupMemoryLogSnippet(quotedLogPath);
|
|
392
|
-
const remove =
|
|
397
|
+
const remove = `${imageCleanup.capture}; if docker rm -f ${quotedName} >> ${quotedLogPath} 2>&1; then ${imageCleanup.remove}; fi; ${removalNote}; ${memory}`;
|
|
393
398
|
const keep = `${postMortem}; ${memory}; ${buildDockerKeptLogSnippet(containerName, quotedLogPath)}`;
|
|
394
399
|
|
|
395
400
|
const since = watcherOptions.since
|
|
@@ -438,13 +443,15 @@ function buildDetachedDockerCompletionScript(
|
|
|
438
443
|
parts.push(buildCgroupSamplerStopSnippet());
|
|
439
444
|
parts.push(buildDockerStateSnippet(containerName));
|
|
440
445
|
if (policy === DOCKER_CONTAINER_CLEANUP_POLICY.ALWAYS) {
|
|
441
|
-
exited.push(
|
|
446
|
+
exited.push(
|
|
447
|
+
`${imageCleanup.capture}; if docker rm -f ${quotedName} >/dev/null 2>&1; then ${imageCleanup.remove}; fi`
|
|
448
|
+
);
|
|
442
449
|
} else if (
|
|
443
450
|
policy === DOCKER_CONTAINER_CLEANUP_POLICY.DEFAULT ||
|
|
444
451
|
policy === DOCKER_CONTAINER_CLEANUP_POLICY.KEEP_ON_FAIL
|
|
445
452
|
) {
|
|
446
453
|
exited.push(
|
|
447
|
-
`if ${buildSuccessfulNonOomCondition()}; then docker rm -f ${quotedName} >/dev/null 2>&1
|
|
454
|
+
`if ${buildSuccessfulNonOomCondition()}; then ${imageCleanup.capture}; if docker rm -f ${quotedName} >/dev/null 2>&1; then ${imageCleanup.remove}; fi; fi`
|
|
448
455
|
);
|
|
449
456
|
}
|
|
450
457
|
}
|