start-command 0.35.3 → 0.36.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/README.md +10 -0
- package/package.json +1 -1
- package/src/bin/cli.js +43 -7
- package/src/lib/args-parser.js +12 -0
- package/src/lib/atomic-write.js +34 -0
- package/src/lib/attached-diagnostics.js +90 -0
- package/src/lib/cgroup-memory.js +43 -11
- package/src/lib/command-builder.js +40 -0
- package/src/lib/cpu-penalty-monitor.js +226 -0
- package/src/lib/cpu-penalty.js +175 -0
- package/src/lib/detached-finalize.js +47 -7
- package/src/lib/detached-output.js +61 -0
- package/src/lib/docker-cleanup.js +43 -4
- package/src/lib/docker-resource-limits.js +28 -0
- package/src/lib/docker-resource-options.js +180 -0
- package/src/lib/docker-runtime-args.js +15 -0
- package/src/lib/execution-attempt.js +153 -0
- package/src/lib/execution-recovery.js +191 -15
- package/src/lib/execution-resume-all.js +23 -1
- package/src/lib/execution-resume.js +203 -40
- package/src/lib/execution-store.js +56 -121
- package/src/lib/exit-evidence.js +123 -0
- package/src/lib/exit-reason.js +24 -6
- package/src/lib/isolation-log-utils.js +3 -3
- package/src/lib/isolation.js +36 -9
- package/src/lib/launch-owner.js +25 -0
- package/src/lib/launch-persistence.js +54 -0
- package/src/lib/query-commands.js +14 -3
- package/src/lib/resume-resources.js +88 -0
- package/src/lib/status-formatter.js +66 -6
- package/src/lib/store-lock.js +133 -0
- package/src/lib/usage.js +11 -0
- package/test/docker-runtime-options.js +6 -0
- package/test/execution-resume.js +10 -3
- package/test/exit-reason.js +6 -1
- package/test/isolation.js +5 -4
- package/test/regression-174.js +3 -1
- package/test/regression-176.js +5 -1
- package/test/regression-180.js +17 -10
- package/test/regression-182.js +7 -3
- package/test/regression-185.js +4 -9
- package/test/regression-187.js +547 -0
- package/test/regression-189.js +224 -0
- package/test/regression-190.js +111 -0
- package/test/regression-191.js +152 -0
- package/test/regression-192.js +66 -0
- package/test/regression-193.js +241 -0
- package/test/regression-194.js +182 -0
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,17 @@
|
|
|
1
1
|
# start-command
|
|
2
2
|
|
|
3
|
+
## 0.36.0
|
|
4
|
+
|
|
5
|
+
### Minor Changes
|
|
6
|
+
|
|
7
|
+
- dbf7ba6: Add Docker launch/recovery resource limits with daemon percentages and random ranges, opt-in delayed CPU penalties, and remote/DinD memory diagnostics. Reserve launches before side effects, atomically publish store locks and database writes, and require exit-time evidence for OOM verdicts while recognizing attributed Docker daemon restarts.
|
|
8
|
+
|
|
9
|
+
## 0.35.4
|
|
10
|
+
|
|
11
|
+
### Patch Changes
|
|
12
|
+
|
|
13
|
+
- 6544236: Scope explicit resume status, memory evidence and log reads to execution attempts; preserve earlier evidence in attempt history and report launch, watcher, output and terminal lifecycle separately.
|
|
14
|
+
|
|
3
15
|
## 0.35.3
|
|
4
16
|
|
|
5
17
|
### Patch Changes
|
package/README.md
CHANGED
|
@@ -56,3 +56,13 @@ GitHub Releases. The release title carries the `[JavaScript]` prefix, e.g.
|
|
|
56
56
|
## License
|
|
57
57
|
|
|
58
58
|
Released into the public domain under the [Unlicense](../LICENSE).
|
|
59
|
+
|
|
60
|
+
### Docker resource and recovery controls
|
|
61
|
+
|
|
62
|
+
Launch with `--memory 64m`, `--memory-swap 64m` and `--cpus 1`, or use daemon
|
|
63
|
+
percentages and uniform ranges such as `--memory '70%-80%'`. Memory-swap defaults
|
|
64
|
+
to memory. `--cpu-penalty` enables delayed caps and reports state through status.
|
|
65
|
+
Manual `--resume <id> --memory 128m` updates before starting;
|
|
66
|
+
`--on-kill-resume-memory` changes memory only after a fresh qualifying OOM.
|
|
67
|
+
See the [full resource and recovery guide](../README.md#docker-launch-limits-recovery-limits-and-cpu-penalty)
|
|
68
|
+
and [incident research](../docs/case-studies/issue-195/README.md).
|
package/package.json
CHANGED
package/src/bin/cli.js
CHANGED
|
@@ -241,6 +241,18 @@ const sessionId = wrapperOptions.sessionId || generateUUID();
|
|
|
241
241
|
process.exit(queryExitCode === null ? 0 : queryExitCode);
|
|
242
242
|
}
|
|
243
243
|
|
|
244
|
+
if (wrapperOptions.isolated === 'docker') {
|
|
245
|
+
const {
|
|
246
|
+
resolveResourceOptions,
|
|
247
|
+
} = require('../lib/docker-resource-options');
|
|
248
|
+
Object.assign(
|
|
249
|
+
wrapperOptions,
|
|
250
|
+
resolveResourceOptions(
|
|
251
|
+
wrapperOptions,
|
|
252
|
+
require('../lib/execution-control').runCommand
|
|
253
|
+
)
|
|
254
|
+
);
|
|
255
|
+
}
|
|
244
256
|
// Check if running in isolation mode or with user isolation
|
|
245
257
|
if (hasIsolation(wrapperOptions) || wrapperOptions.user) {
|
|
246
258
|
await runWithIsolation(
|
|
@@ -419,14 +431,21 @@ async function runWithIsolation(
|
|
|
419
431
|
// Save initial execution record and set global reference for signal cleanup
|
|
420
432
|
if (executionRecord && store) {
|
|
421
433
|
currentExecutionRecord = executionRecord;
|
|
434
|
+
require('../lib/launch-owner').markLaunch(executionRecord);
|
|
422
435
|
try {
|
|
423
436
|
store.save(executionRecord);
|
|
424
437
|
} catch (err) {
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
438
|
+
console.error(
|
|
439
|
+
JSON.stringify({
|
|
440
|
+
code: 'LAUNCH_RESERVATION_FAILED',
|
|
441
|
+
uuid: executionRecord.uuid,
|
|
442
|
+
containerName: sessionName,
|
|
443
|
+
error: err.message,
|
|
444
|
+
})
|
|
445
|
+
);
|
|
446
|
+
currentExecutionRecord = null;
|
|
447
|
+
process.exitCode = 1;
|
|
448
|
+
return 1;
|
|
430
449
|
}
|
|
431
450
|
}
|
|
432
451
|
|
|
@@ -458,6 +477,7 @@ async function runWithIsolation(
|
|
|
458
477
|
// Note: Isolation environments currently use native spawn/execSync
|
|
459
478
|
// Future: Add command-stream support with raw() function for multiplexers
|
|
460
479
|
result = await runIsolated(environment, cmd, {
|
|
480
|
+
...options,
|
|
461
481
|
session: sessionName,
|
|
462
482
|
image: effectiveImage,
|
|
463
483
|
endpoint: options.endpoint,
|
|
@@ -502,6 +522,11 @@ async function runWithIsolation(
|
|
|
502
522
|
|
|
503
523
|
// Update execution record: detached keeps "executing" (resolved at query time)
|
|
504
524
|
if (executionRecord && store) {
|
|
525
|
+
executionRecord.options.launchPending = false;
|
|
526
|
+
if (result.diagnostics) {
|
|
527
|
+
executionRecord.cgroupMemory = result.diagnostics.cgroupMemory;
|
|
528
|
+
executionRecord.options.exitEvidence = result.diagnostics.exitEvidence;
|
|
529
|
+
}
|
|
505
530
|
if (result.containerId) {
|
|
506
531
|
executionRecord.options.containerId = result.containerId;
|
|
507
532
|
}
|
|
@@ -515,11 +540,22 @@ async function runWithIsolation(
|
|
|
515
540
|
: executionRecord
|
|
516
541
|
);
|
|
517
542
|
} catch (err) {
|
|
518
|
-
if (
|
|
543
|
+
if (environment === 'docker' && mode === 'detached' && result.success) {
|
|
519
544
|
console.error(
|
|
520
|
-
|
|
545
|
+
require('../lib/launch-persistence').failLaunchedPersistence(
|
|
546
|
+
store,
|
|
547
|
+
executionRecord,
|
|
548
|
+
require('../lib/execution-control').runCommand,
|
|
549
|
+
err
|
|
550
|
+
)
|
|
521
551
|
);
|
|
552
|
+
currentExecutionRecord = null;
|
|
553
|
+
process.exitCode = 1;
|
|
554
|
+
return 1;
|
|
522
555
|
}
|
|
556
|
+
console.error(
|
|
557
|
+
`[Tracking] Could not update execution record: ${err.message}`
|
|
558
|
+
);
|
|
523
559
|
}
|
|
524
560
|
currentExecutionRecord = null;
|
|
525
561
|
}
|
package/src/lib/args-parser.js
CHANGED
|
@@ -44,6 +44,8 @@
|
|
|
44
44
|
const { getDefaultDockerImage } = require('./docker-utils');
|
|
45
45
|
const dockerNetworkOptions = require('./docker-network-options');
|
|
46
46
|
const dockerRecoveryOptions = require('./docker-recovery-options');
|
|
47
|
+
const cpuPenalty = require('./cpu-penalty');
|
|
48
|
+
const dockerResourceOptions = require('./docker-resource-options');
|
|
47
49
|
const { parseSequence, isSequence } = require('./sequence-parser');
|
|
48
50
|
const { buildCommandString } = require('./shell-utils');
|
|
49
51
|
const {
|
|
@@ -414,6 +416,14 @@ function parseOption(args, index, options) {
|
|
|
414
416
|
return networkOption;
|
|
415
417
|
}
|
|
416
418
|
|
|
419
|
+
const cpuConsumed = cpuPenalty.parse(args, index, options);
|
|
420
|
+
if (cpuConsumed) {
|
|
421
|
+
return cpuConsumed;
|
|
422
|
+
}
|
|
423
|
+
const resourceOption = dockerResourceOptions.parse(args, index, options);
|
|
424
|
+
if (resourceOption) {
|
|
425
|
+
return resourceOption;
|
|
426
|
+
}
|
|
417
427
|
const recoveryOption = dockerRecoveryOptions.parse(args, index, options);
|
|
418
428
|
if (recoveryOption) {
|
|
419
429
|
return recoveryOption;
|
|
@@ -756,6 +766,8 @@ function validateOptions(options) {
|
|
|
756
766
|
}
|
|
757
767
|
|
|
758
768
|
dockerRecoveryOptions.validate(options);
|
|
769
|
+
dockerResourceOptions.validate(options);
|
|
770
|
+
cpuPenalty.validate(options);
|
|
759
771
|
|
|
760
772
|
// Session name is only valid with isolation
|
|
761
773
|
if (options.session && !options.isolated) {
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
/** Keep the previous database intact when a write fails, including ENOSPC. */
|
|
2
|
+
const fs = require('fs');
|
|
3
|
+
const path = require('path');
|
|
4
|
+
const crypto = require('crypto');
|
|
5
|
+
|
|
6
|
+
function atomicWrite(file, content) {
|
|
7
|
+
const temporary = `${file}.${process.pid}.${crypto.randomUUID()}.tmp`;
|
|
8
|
+
try {
|
|
9
|
+
fs.writeFileSync(temporary, content, { flag: 'wx' });
|
|
10
|
+
const handle = fs.openSync(temporary, 'r+');
|
|
11
|
+
try {
|
|
12
|
+
fs.fsyncSync(handle);
|
|
13
|
+
} finally {
|
|
14
|
+
fs.closeSync(handle);
|
|
15
|
+
}
|
|
16
|
+
fs.renameSync(temporary, file);
|
|
17
|
+
if (process.platform !== 'win32') {
|
|
18
|
+
const directory = fs.openSync(path.dirname(file), 'r');
|
|
19
|
+
try {
|
|
20
|
+
fs.fsyncSync(directory);
|
|
21
|
+
} finally {
|
|
22
|
+
fs.closeSync(directory);
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
} finally {
|
|
26
|
+
try {
|
|
27
|
+
fs.unlinkSync(temporary);
|
|
28
|
+
} catch {
|
|
29
|
+
/* Renamed or never created. */
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
module.exports = { atomicWrite };
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
/** Keep the same memory and CPU monitors alive during an attached Docker run. */
|
|
2
|
+
const { spawn, spawnSync } = require('child_process');
|
|
3
|
+
const fs = require('fs');
|
|
4
|
+
const os = require('os');
|
|
5
|
+
const path = require('path');
|
|
6
|
+
const crypto = require('crypto');
|
|
7
|
+
const cgroup = require('./cgroup-memory');
|
|
8
|
+
const cpu = require('./cpu-penalty-monitor');
|
|
9
|
+
const evidence = require('./exit-evidence');
|
|
10
|
+
const { shellQuote } = require('./isolation-log-utils');
|
|
11
|
+
|
|
12
|
+
function startAttachedDiagnostics(name, options) {
|
|
13
|
+
const file = path.join(os.tmpdir(), `start-attached-${crypto.randomUUID()}`);
|
|
14
|
+
let worker;
|
|
15
|
+
{
|
|
16
|
+
const script = [
|
|
17
|
+
cgroup.buildCgroupSamplerStartSnippet(name),
|
|
18
|
+
cpu.startSnippet(options.executionId, name, 1, options),
|
|
19
|
+
'read __start_stop',
|
|
20
|
+
cgroup.buildCgroupSamplerStopSnippet(),
|
|
21
|
+
cpu.stopSnippet(),
|
|
22
|
+
`printf '%s' "$__start_command_cgroup" > ${shellQuote(file)}`,
|
|
23
|
+
`printf '%s' "$__start_command_memory_unavailable" > ${shellQuote(`${file}.reason`)}`,
|
|
24
|
+
]
|
|
25
|
+
.filter(Boolean)
|
|
26
|
+
.join('; ');
|
|
27
|
+
worker = spawn('sh', ['-c', script], {
|
|
28
|
+
stdio: ['pipe', 'ignore', 'ignore'],
|
|
29
|
+
});
|
|
30
|
+
worker.on('error', () => {});
|
|
31
|
+
// Subscribe before a fast command finishes, including shell startup failure.
|
|
32
|
+
worker.done = new Promise((resolve) => worker.once('close', resolve));
|
|
33
|
+
worker.stdin.on('error', () => {});
|
|
34
|
+
}
|
|
35
|
+
return {
|
|
36
|
+
async finish(state) {
|
|
37
|
+
if (worker) {
|
|
38
|
+
worker.stdin.end();
|
|
39
|
+
await worker.done;
|
|
40
|
+
}
|
|
41
|
+
let sample = '';
|
|
42
|
+
let unavailable = 'sampler could not start (sh unavailable)';
|
|
43
|
+
try {
|
|
44
|
+
sample = fs.readFileSync(file, 'utf8');
|
|
45
|
+
unavailable =
|
|
46
|
+
fs.readFileSync(`${file}.reason`, 'utf8') ||
|
|
47
|
+
'sampler stopped before a reading';
|
|
48
|
+
} catch {
|
|
49
|
+
/* Optional diagnostics are best effort. */
|
|
50
|
+
}
|
|
51
|
+
try {
|
|
52
|
+
fs.unlinkSync(file);
|
|
53
|
+
fs.unlinkSync(`${file}.reason`);
|
|
54
|
+
} catch {
|
|
55
|
+
/* Optional diagnostics are best effort. */
|
|
56
|
+
}
|
|
57
|
+
const docker = require('./docker-cleanup').getDockerCommand();
|
|
58
|
+
const limit = spawnSync(
|
|
59
|
+
docker,
|
|
60
|
+
['inspect', '-f', '{{.HostConfig.Memory}}', name],
|
|
61
|
+
{ encoding: 'utf8' }
|
|
62
|
+
);
|
|
63
|
+
const line = evidence.collectExitEvidence({
|
|
64
|
+
containerName: name,
|
|
65
|
+
sample,
|
|
66
|
+
finishedAt: state?.finishedAt,
|
|
67
|
+
exitCode: state?.exitCode,
|
|
68
|
+
oomKilled: state?.oomKilled,
|
|
69
|
+
});
|
|
70
|
+
if (options.logPath) {
|
|
71
|
+
try {
|
|
72
|
+
fs.appendFileSync(
|
|
73
|
+
options.logPath,
|
|
74
|
+
`${cgroup.formatCgroupMemoryLogLine(sample, limit.status === 0 ? limit.stdout.trim() : 'unknown', unavailable)}\n${line}\n`
|
|
75
|
+
);
|
|
76
|
+
} catch (error) {
|
|
77
|
+
if (process.env.START_DEBUG === '1') {
|
|
78
|
+
console.error(`[DEBUG] Attached diagnostics: ${error.message}`);
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
return {
|
|
83
|
+
cgroupMemory: cgroup.parseCgroupMemorySample(sample),
|
|
84
|
+
exitEvidence: evidence.fromLog(line),
|
|
85
|
+
};
|
|
86
|
+
},
|
|
87
|
+
};
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
module.exports = { startAttachedDiagnostics };
|
package/src/lib/cgroup-memory.js
CHANGED
|
@@ -67,6 +67,7 @@ function buildCgroupFunctionsSnippet() {
|
|
|
67
67
|
'__start_command_cgroup_read() { __scr_events=$(cat "$1/memory.events" 2>/dev/null) || return 1; ' +
|
|
68
68
|
"__scr_oom=$(printf '%s\\n' \"$__scr_events\" | sed -n 's/^oom //p'); " +
|
|
69
69
|
"__scr_kill=$(printf '%s\\n' \"$__scr_events\" | sed -n 's/^oom_kill //p'); " +
|
|
70
|
+
'case "$__scr_oom:$__scr_kill" in *[!0-9:]*|:*|*:) return 1;; esac; ' +
|
|
70
71
|
'__scr_max=$(cat "$1/memory.max" 2>/dev/null); __scr_peak=$(cat "$1/memory.peak" 2>/dev/null); ' +
|
|
71
72
|
'printf \'%s %s %s %s %s\\n\' "${__scr_max:--}" "${__scr_peak:--}" ' +
|
|
72
73
|
'"${__scr_oom:--}" "${__scr_kill:--}" "$1"; }',
|
|
@@ -82,13 +83,33 @@ function buildCgroupFunctionsSnippet() {
|
|
|
82
83
|
*/
|
|
83
84
|
function buildCgroupSamplerStartSnippet(containerName) {
|
|
84
85
|
const v = CGROUP_SHELL_VARS;
|
|
86
|
+
const name = shellQuote(containerName);
|
|
87
|
+
const remoteRead = '__start_command_cgroup_read /sys/fs/cgroup';
|
|
85
88
|
return [
|
|
86
89
|
buildCgroupFunctionsSnippet(),
|
|
87
90
|
`${v.file}="\${TMPDIR:-/tmp}/start-command-cgroup.$$"`,
|
|
88
|
-
`rm -f "$${v.file}" "$${v.file}.tmp"`,
|
|
89
|
-
`
|
|
90
|
-
|
|
91
|
-
|
|
91
|
+
`rm -f "$${v.file}" "$${v.file}.tmp" "$${v.file}.reason"`,
|
|
92
|
+
`__start_command_memory_limit=$(docker inspect -f '{{.HostConfig.Memory}}' ${name} 2>/dev/null)`,
|
|
93
|
+
`__start_command_sample_name=${name}`,
|
|
94
|
+
// The first counter is a baseline, including after watcher restart. Only
|
|
95
|
+
// changes between adjacent successful samples supply exit-time evidence.
|
|
96
|
+
`( export START_COMMAND_CGROUP_SAMPLING=1; __scs_previous=''; __scs_previous_at=0; __scs_changed=0; ` +
|
|
97
|
+
`__scs_dir=$(__start_command_cgroup_dir ${name}); while :; do ` +
|
|
98
|
+
'__scs_line=""; if [ -n "$__scs_dir" ]; then __scs_line=$(__start_command_cgroup_read "$__scs_dir"); fi; ' +
|
|
99
|
+
`if [ -z "$__scs_line" ]; then __scs_ns=$(docker inspect -f '{{.HostConfig.CgroupnsMode}}' ${name} 2>/dev/null); ` +
|
|
100
|
+
`if [ "$__scs_ns" = private ]; then ` +
|
|
101
|
+
`__scs_remote=$(docker exec ${name} sh -c ${shellQuote(`${buildCgroupFunctionsSnippet()}; ${remoteRead}`)} 2>&1); __scs_exit=$?; ` +
|
|
102
|
+
'if [ "$__scs_exit" = 0 ]; then __scs_line=$__scs_remote; ' +
|
|
103
|
+
'else __scs_detail=$(printf "%s" "$__scs_remote" | sed -n "1p" | cut -c 1-240); ' +
|
|
104
|
+
`printf '%s\n' "local cgroup unavailable; Docker exec exit=$__scs_exit: \${__scs_detail:-private cgroup v2 counters unreadable}" > "$${v.file}.reason"; fi; ` +
|
|
105
|
+
`else printf '%s\n' "local cgroup unavailable; container cgroup namespace=\${__scs_ns:-unavailable}, private required" > "$${v.file}.reason"; fi; fi; ` +
|
|
106
|
+
'__scs_now=$(date +%s); if [ -n "$__scs_line" ]; then ' +
|
|
107
|
+
'__scs_kill=$(printf "%s" "$__scs_line" | cut -d " " -f 4); ' +
|
|
108
|
+
'if [ -n "$__scs_previous" ] && [ "$__scs_kill" -gt "$__scs_previous" ] 2>/dev/null && [ "$((__scs_now-__scs_previous_at))" -le 3 ]; then __scs_changed=$__scs_now; fi; ' +
|
|
109
|
+
'__scs_previous=$__scs_kill; __scs_previous_at=$__scs_now; ' +
|
|
110
|
+
'__scs_fields=$(printf "%s" "$__scs_line" | cut -d " " -f 1-4); ' +
|
|
111
|
+
`printf '%s %s %s %s\\n' "$__scs_fields" "$__scs_changed" "$__scs_now" "$__scs_dir" > "$${v.file}.tmp" && mv -f "$${v.file}.tmp" "$${v.file}"; fi; ` +
|
|
112
|
+
`if [ "$(docker inspect -f '{{.State.Running}}' ${name} 2>/dev/null)" = false ]; then break; fi; ` +
|
|
92
113
|
`sleep ${CGROUP_SAMPLE_INTERVAL_SECONDS}; done ) >/dev/null 2>&1 & ${v.sampler}=$!`,
|
|
93
114
|
].join('; ');
|
|
94
115
|
}
|
|
@@ -106,10 +127,16 @@ function buildCgroupSamplerStopSnippet() {
|
|
|
106
127
|
`kill "$${v.sampler}" 2>/dev/null`,
|
|
107
128
|
`wait "$${v.sampler}" 2>/dev/null`,
|
|
108
129
|
`${v.sample}=$(cat "$${v.file}" 2>/dev/null)`,
|
|
109
|
-
`
|
|
110
|
-
`
|
|
111
|
-
`
|
|
112
|
-
|
|
130
|
+
`__start_command_memory_unavailable=$(cat "$${v.file}.reason" 2>/dev/null)`,
|
|
131
|
+
`__scs_dir=$(printf '%s' "$${v.sample}" | cut -s -d ' ' -f 7-)`,
|
|
132
|
+
`if [ -n "$__scs_dir" ] && __scs_line=$(__start_command_cgroup_read "$__scs_dir"); then ` +
|
|
133
|
+
`__scs_old=$(printf '%s' "$${v.sample}" | cut -d ' ' -f 4); __scs_at=$(printf '%s' "$${v.sample}" | cut -d ' ' -f 6); ` +
|
|
134
|
+
`__scs_changed=$(printf '%s' "$${v.sample}" | cut -d ' ' -f 5); __scs_now=$(date +%s); ` +
|
|
135
|
+
`__scs_new=$(printf '%s' "$__scs_line" | cut -d ' ' -f 4); ` +
|
|
136
|
+
`if [ "$__scs_new" -gt "$__scs_old" ] 2>/dev/null && [ "$((__scs_now-__scs_at))" -le 3 ]; then __scs_changed=$__scs_now; fi; ` +
|
|
137
|
+
`${v.sample}="$(printf '%s' "$__scs_line" | cut -d ' ' -f 1-4) $__scs_changed $__scs_now"; fi`,
|
|
138
|
+
`rm -f "$${v.file}" "$${v.file}.tmp" "$${v.file}.reason"`,
|
|
139
|
+
`${v.sample}=$(printf '%s' "$${v.sample}" | cut -d ' ' -f 1-6)`,
|
|
113
140
|
].join('; ');
|
|
114
141
|
}
|
|
115
142
|
|
|
@@ -128,7 +155,8 @@ function buildCgroupMemoryLogSnippet(quotedLogPath) {
|
|
|
128
155
|
`__scm_note=' (${OOM_SCOPE_NOTES[OOM_SCOPE.UNKNOWN]})'; fi; ` +
|
|
129
156
|
`printf 'Memory: memory.max=%s memory.peak=%s oom=%s oom_kill=%s%s\\n' ` +
|
|
130
157
|
`"$1" "$2" "$3" "$4" "$__scm_note"; }; ` +
|
|
131
|
-
`if [ -n "$${v.sample}" ]; then __start_command_cgroup_log $${v.sample} >> ${quotedLogPath};
|
|
158
|
+
`if [ -n "$${v.sample}" ]; then __start_command_cgroup_log $${v.sample} >> ${quotedLogPath}; ` +
|
|
159
|
+
`else printf 'Memory: unavailable (%s) memory.limit=%s (HostConfig)\\n' "\${__start_command_memory_unavailable:-sampler stopped before a reading; local cgroup unavailable}" "\${__start_command_memory_limit:-unknown}" >> ${quotedLogPath}; fi`
|
|
132
160
|
);
|
|
133
161
|
}
|
|
134
162
|
|
|
@@ -138,10 +166,14 @@ function buildCgroupMemoryLogSnippet(quotedLogPath) {
|
|
|
138
166
|
* @param {string|null|undefined} text - `<memory.max> <memory.peak> <oom> <oom_kill>`
|
|
139
167
|
* @returns {?string} Line without a trailing newline, or null without a sample
|
|
140
168
|
*/
|
|
141
|
-
function formatCgroupMemoryLogLine(
|
|
169
|
+
function formatCgroupMemoryLogLine(
|
|
170
|
+
text,
|
|
171
|
+
memoryLimit = 'unknown',
|
|
172
|
+
unavailable = 'sampler unavailable; local cgroup hidden or unreadable'
|
|
173
|
+
) {
|
|
142
174
|
const counters = parseCgroupMemorySample(text);
|
|
143
175
|
if (!counters) {
|
|
144
|
-
return
|
|
176
|
+
return `Memory: unavailable (${unavailable}) memory.limit=${memoryLimit} (HostConfig)`;
|
|
145
177
|
}
|
|
146
178
|
const [max, peak, oom, kill] = String(text).trim().split(/\s+/);
|
|
147
179
|
const scope = describeCgroupOomScope(counters);
|
|
@@ -57,6 +57,46 @@ function buildNextLevelCommand(options, command) {
|
|
|
57
57
|
// Docker runtime options are flat (not per-level); forward them only when a
|
|
58
58
|
// remaining level still uses docker so the nested $ invocation can apply them.
|
|
59
59
|
if (remainingStack.includes('docker')) {
|
|
60
|
+
for (const [key, flag] of [
|
|
61
|
+
['memory', '--memory'],
|
|
62
|
+
['memorySwap', '--memory-swap'],
|
|
63
|
+
['cpus', '--cpus'],
|
|
64
|
+
['onKillResumeMemory', '--on-kill-resume-memory'],
|
|
65
|
+
]) {
|
|
66
|
+
if (options[key] !== null && options[key] !== undefined) {
|
|
67
|
+
parts.push(
|
|
68
|
+
`${flag} ${require('./shell-utils').quoteShellArg(String(options[key]))}`
|
|
69
|
+
);
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
if (options.cpuPenaltyConfig) {
|
|
73
|
+
parts.push('--cpu-penalty');
|
|
74
|
+
for (const [key, flag] of [
|
|
75
|
+
['cpus', '--cpu-penalty-cpus'],
|
|
76
|
+
['trigger', '--cpu-penalty-trigger'],
|
|
77
|
+
['triggerWindowMs', '--cpu-penalty-trigger-window'],
|
|
78
|
+
['release', '--cpu-penalty-release'],
|
|
79
|
+
['releaseWindowMs', '--cpu-penalty-release-window'],
|
|
80
|
+
]) {
|
|
81
|
+
parts.push(
|
|
82
|
+
`${flag} ${options.cpuPenaltyConfig[key]}${key.endsWith('Ms') ? 'ms' : ''}`
|
|
83
|
+
);
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
if (options.onKillResume !== null && options.onKillResume !== undefined) {
|
|
87
|
+
parts.push(`--on-kill-resume ${options.onKillResume}`);
|
|
88
|
+
}
|
|
89
|
+
if (
|
|
90
|
+
options.onKillResumeDelay !== null &&
|
|
91
|
+
options.onKillResumeDelay !== undefined
|
|
92
|
+
) {
|
|
93
|
+
parts.push(`--on-kill-resume-delay ${options.onKillResumeDelay}`);
|
|
94
|
+
}
|
|
95
|
+
if (options.recoveryCommand) {
|
|
96
|
+
parts.push(
|
|
97
|
+
`--recovery-command ${require('./shell-utils').quoteShellArg(options.recoveryCommand)}`
|
|
98
|
+
);
|
|
99
|
+
}
|
|
60
100
|
for (const volume of options.volumes || []) {
|
|
61
101
|
parts.push(`--volume "${volume}"`);
|
|
62
102
|
}
|
|
@@ -0,0 +1,226 @@
|
|
|
1
|
+
/** A watcher-owned monitor. Sidecars avoid writing the store for every sample. */
|
|
2
|
+
const fs = require('fs');
|
|
3
|
+
const path = require('path');
|
|
4
|
+
const { spawnSync } = require('child_process');
|
|
5
|
+
const { initialState, evaluate } = require('./cpu-penalty');
|
|
6
|
+
const { LockManager } = require('./store-lock');
|
|
7
|
+
const { shellQuote } = require('./isolation-log-utils');
|
|
8
|
+
const { cpuCount, cpuUpdateArgs } = require('./docker-resource-limits');
|
|
9
|
+
function statePath(record) {
|
|
10
|
+
return `${record.logPath}.cpu-penalty.json`;
|
|
11
|
+
}
|
|
12
|
+
function readState(record) {
|
|
13
|
+
if (!record.options?.cpuPenaltyConfig || !record.logPath) {
|
|
14
|
+
return null;
|
|
15
|
+
}
|
|
16
|
+
try {
|
|
17
|
+
const state = JSON.parse(fs.readFileSync(statePath(record), 'utf8'));
|
|
18
|
+
if (
|
|
19
|
+
state.containerName !== record.options.sessionName ||
|
|
20
|
+
state.attemptNumber !== (record.attempt?.number || 1)
|
|
21
|
+
) {
|
|
22
|
+
return null;
|
|
23
|
+
}
|
|
24
|
+
if (state.phase === 'penalized' && Number.isFinite(state.updatedAt)) {
|
|
25
|
+
const until = record.endTime ? Date.parse(record.endTime) : Date.now();
|
|
26
|
+
state.penalizedMs += Math.max(0, until - state.updatedAt);
|
|
27
|
+
state.updatedAt = Math.max(until, state.updatedAt);
|
|
28
|
+
}
|
|
29
|
+
return state;
|
|
30
|
+
} catch {
|
|
31
|
+
return null;
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
function publicState(state) {
|
|
35
|
+
const result = { ...state };
|
|
36
|
+
delete result.samples;
|
|
37
|
+
delete result.lastAt;
|
|
38
|
+
delete result.capacity;
|
|
39
|
+
return result;
|
|
40
|
+
}
|
|
41
|
+
function run(command, args) {
|
|
42
|
+
const result = spawnSync(command, args, { encoding: 'utf8' });
|
|
43
|
+
return { success: result.status === 0, stdout: result.stdout || '' };
|
|
44
|
+
}
|
|
45
|
+
async function monitor(uuid, name, attemptNumber, standalone) {
|
|
46
|
+
const { ExecutionStore } = require('./execution-store');
|
|
47
|
+
const store = uuid
|
|
48
|
+
? new ExecutionStore(
|
|
49
|
+
process.env.START_APP_FOLDER
|
|
50
|
+
? { appFolder: path.resolve(process.env.START_APP_FOLDER) }
|
|
51
|
+
: {}
|
|
52
|
+
)
|
|
53
|
+
: null;
|
|
54
|
+
const record = store ? store.get(uuid) : JSON.parse(standalone || 'null');
|
|
55
|
+
if (
|
|
56
|
+
!record?.options.cpuPenaltyConfig ||
|
|
57
|
+
!record.logPath ||
|
|
58
|
+
record.options.sessionName !== name
|
|
59
|
+
) {
|
|
60
|
+
return;
|
|
61
|
+
}
|
|
62
|
+
const config = record.options.cpuPenaltyConfig;
|
|
63
|
+
const lock = new LockManager(`${statePath(record)}.lock`);
|
|
64
|
+
if (!lock.acquire(100)) {
|
|
65
|
+
return;
|
|
66
|
+
}
|
|
67
|
+
const docker = require('./docker-cleanup').getDockerCommand();
|
|
68
|
+
const baseFlags =
|
|
69
|
+
record.options.baseResourceLimits || record.options.resourceLimits || [];
|
|
70
|
+
const baseCpus = cpuCount(baseFlags);
|
|
71
|
+
let state = initialState(baseCpus, Date.now(), readState(record));
|
|
72
|
+
const interval = Math.min(
|
|
73
|
+
30000,
|
|
74
|
+
Math.max(50, Math.min(config.triggerWindowMs, config.releaseWindowMs) / 8)
|
|
75
|
+
);
|
|
76
|
+
const publish = () => {
|
|
77
|
+
const file = statePath(record),
|
|
78
|
+
tmp = `${file}.${process.pid}.tmp`;
|
|
79
|
+
try {
|
|
80
|
+
fs.writeFileSync(
|
|
81
|
+
tmp,
|
|
82
|
+
JSON.stringify({
|
|
83
|
+
...publicState(state),
|
|
84
|
+
containerName: name,
|
|
85
|
+
attemptNumber: Number(attemptNumber),
|
|
86
|
+
updatedAt: Date.now(),
|
|
87
|
+
})
|
|
88
|
+
);
|
|
89
|
+
fs.renameSync(tmp, file);
|
|
90
|
+
} finally {
|
|
91
|
+
try {
|
|
92
|
+
fs.unlinkSync(tmp);
|
|
93
|
+
} catch {
|
|
94
|
+
/* Optional diagnostics are best effort. */
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
};
|
|
98
|
+
const log = (text) => {
|
|
99
|
+
try {
|
|
100
|
+
fs.appendFileSync(record.logPath, `${text}\n`);
|
|
101
|
+
} catch {
|
|
102
|
+
/* Optional diagnostics are best effort. */
|
|
103
|
+
}
|
|
104
|
+
};
|
|
105
|
+
const started = Date.now();
|
|
106
|
+
try {
|
|
107
|
+
publish();
|
|
108
|
+
while (true) {
|
|
109
|
+
const current = store ? store.get(uuid) : record;
|
|
110
|
+
if (
|
|
111
|
+
!current ||
|
|
112
|
+
current.options.sessionName !== name ||
|
|
113
|
+
(current.attempt?.number || 1) !== Number(attemptNumber)
|
|
114
|
+
) {
|
|
115
|
+
break;
|
|
116
|
+
}
|
|
117
|
+
const live = run(docker, ['inspect', '-f', '{{.State.Running}}', name]);
|
|
118
|
+
if (live.success && live.stdout.trim() === 'false') {
|
|
119
|
+
break;
|
|
120
|
+
}
|
|
121
|
+
if (!live.success && Date.now() - started > 30000) {
|
|
122
|
+
break;
|
|
123
|
+
}
|
|
124
|
+
const info = run(docker, ['info', '--format', '{{json .}}']);
|
|
125
|
+
const stats = run(docker, [
|
|
126
|
+
'stats',
|
|
127
|
+
'--no-stream',
|
|
128
|
+
'--format',
|
|
129
|
+
'{{json .}}',
|
|
130
|
+
]);
|
|
131
|
+
let daemonCpus = NaN,
|
|
132
|
+
cores = NaN;
|
|
133
|
+
try {
|
|
134
|
+
if (info.success) {
|
|
135
|
+
daemonCpus = JSON.parse(info.stdout).NCPU;
|
|
136
|
+
}
|
|
137
|
+
if (stats.success) {
|
|
138
|
+
const row = stats.stdout
|
|
139
|
+
.trim()
|
|
140
|
+
.split('\n')
|
|
141
|
+
.map((s) => JSON.parse(s))
|
|
142
|
+
.find((s) => s.Name === name);
|
|
143
|
+
if (row && /^\d+(?:\.\d+)?%$/.test(row.CPUPerc)) {
|
|
144
|
+
cores = Number(row.CPUPerc.slice(0, -1)) / 100;
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
} catch {
|
|
148
|
+
/* Optional diagnostics are best effort. */
|
|
149
|
+
}
|
|
150
|
+
const decision = evaluate(
|
|
151
|
+
state,
|
|
152
|
+
{
|
|
153
|
+
now: Date.now(),
|
|
154
|
+
cores,
|
|
155
|
+
daemonCpus,
|
|
156
|
+
maxGapMs: Math.max(5000, interval * 3),
|
|
157
|
+
},
|
|
158
|
+
config
|
|
159
|
+
);
|
|
160
|
+
if (decision.action) {
|
|
161
|
+
const action = decision.action;
|
|
162
|
+
const update = run(docker, [
|
|
163
|
+
'update',
|
|
164
|
+
...cpuUpdateArgs(baseFlags, action.cpus),
|
|
165
|
+
name,
|
|
166
|
+
]);
|
|
167
|
+
if (update.success) {
|
|
168
|
+
state = decision.state;
|
|
169
|
+
log(
|
|
170
|
+
`[start] CPU penalty ${action.kind === 'apply' ? 'applied' : 'lifted'}: ${action.cpus} CPUs (${action.average === null || action.average === undefined ? 'capacity changed' : `${action.average.toFixed(2)} of ${action.capacity} CPUs`})`
|
|
171
|
+
);
|
|
172
|
+
} else {
|
|
173
|
+
state = {
|
|
174
|
+
...decision.state,
|
|
175
|
+
phase: state.phase,
|
|
176
|
+
since: state.since,
|
|
177
|
+
limitCpus: state.limitCpus,
|
|
178
|
+
penaltyCount: state.penaltyCount,
|
|
179
|
+
samples: [],
|
|
180
|
+
};
|
|
181
|
+
}
|
|
182
|
+
} else {
|
|
183
|
+
state = decision.state;
|
|
184
|
+
}
|
|
185
|
+
publish();
|
|
186
|
+
await new Promise((resolve) => globalThis.setTimeout(resolve, interval));
|
|
187
|
+
}
|
|
188
|
+
} finally {
|
|
189
|
+
lock.release();
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
function startSnippet(uuid, name, attempt = 1, options = null) {
|
|
193
|
+
const standalone =
|
|
194
|
+
!uuid && options?.cpuPenaltyConfig && options.logPath
|
|
195
|
+
? JSON.stringify({
|
|
196
|
+
logPath: options.logPath,
|
|
197
|
+
options: {
|
|
198
|
+
sessionName: name,
|
|
199
|
+
cpuPenaltyConfig: options.cpuPenaltyConfig,
|
|
200
|
+
baseResourceLimits: options.resourceLimits,
|
|
201
|
+
},
|
|
202
|
+
})
|
|
203
|
+
: null;
|
|
204
|
+
if (!uuid && !standalone) {
|
|
205
|
+
return '';
|
|
206
|
+
}
|
|
207
|
+
return `${shellQuote(process.execPath)} ${shellQuote(__filename)} ${shellQuote(uuid || '')} ${shellQuote(name)} ${Number(attempt)} ${shellQuote(standalone || '')} >/dev/null 2>&1 & __start_cpu_monitor=$!`;
|
|
208
|
+
}
|
|
209
|
+
function stopSnippet() {
|
|
210
|
+
return 'if [ -n "$__start_cpu_monitor" ]; then kill "$__start_cpu_monitor" 2>/dev/null; wait "$__start_cpu_monitor" 2>/dev/null; fi';
|
|
211
|
+
}
|
|
212
|
+
module.exports = {
|
|
213
|
+
statePath,
|
|
214
|
+
readState,
|
|
215
|
+
publicState,
|
|
216
|
+
monitor,
|
|
217
|
+
startSnippet,
|
|
218
|
+
stopSnippet,
|
|
219
|
+
};
|
|
220
|
+
if (require.main === module) {
|
|
221
|
+
monitor(...process.argv.slice(2)).catch((error) => {
|
|
222
|
+
if (process.env.START_DEBUG === '1') {
|
|
223
|
+
console.error(`[DEBUG] CPU monitor: ${error.stack}`);
|
|
224
|
+
}
|
|
225
|
+
});
|
|
226
|
+
}
|