start-command 0.35.4 → 0.37.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +14 -0
- package/README.md +16 -0
- package/package.json +1 -1
- package/src/bin/cli.js +48 -7
- package/src/lib/args-parser-queries.js +14 -0
- package/src/lib/args-parser.js +32 -1
- package/src/lib/atomic-write.js +34 -0
- package/src/lib/attached-diagnostics.js +90 -0
- package/src/lib/cgroup-memory.js +43 -11
- package/src/lib/command-builder.js +43 -0
- package/src/lib/cpu-penalty-monitor.js +226 -0
- package/src/lib/cpu-penalty.js +175 -0
- package/src/lib/detached-finalize.js +16 -0
- package/src/lib/docker-cleanup.js +35 -7
- package/src/lib/docker-command-handoff.js +75 -0
- package/src/lib/docker-labels.js +33 -0
- package/src/lib/docker-resource-limits.js +28 -0
- package/src/lib/docker-resource-options.js +180 -0
- package/src/lib/docker-runtime-args.js +23 -0
- package/src/lib/docker-snapshot-safety.js +164 -0
- package/src/lib/execution-recovery.js +153 -30
- package/src/lib/execution-resume-all.js +11 -0
- package/src/lib/execution-resume.js +314 -54
- package/src/lib/execution-store.js +25 -121
- package/src/lib/exit-evidence.js +123 -0
- package/src/lib/exit-reason.js +24 -6
- package/src/lib/failure-handler.js +15 -5
- package/src/lib/isolation.js +48 -7
- package/src/lib/launch-owner.js +25 -0
- package/src/lib/launch-persistence.js +54 -0
- package/src/lib/log-sanitizer.js +267 -0
- package/src/lib/log-uploader.js +38 -19
- package/src/lib/query-commands.js +18 -6
- package/src/lib/resume-resources.js +88 -0
- package/src/lib/shell-utils.js +67 -17
- package/src/lib/status-formatter.js +28 -0
- package/src/lib/store-lock.js +133 -0
- package/src/lib/usage.js +16 -1
- package/test/ci-workflow-invariants.js +15 -14
- package/test/docker-command-handoff.js +107 -0
- package/test/docker-runtime-options.js +10 -0
- package/test/docker-snapshot-safety.js +106 -0
- package/test/execution-resume.js +19 -11
- package/test/exit-reason.js +6 -1
- package/test/failure-handler.js +11 -0
- package/test/isolation.js +5 -4
- package/test/javascript-first-ci.js +216 -0
- package/test/log-sanitizer.js +306 -0
- package/test/regression-174.js +3 -1
- package/test/regression-176.js +7 -0
- package/test/regression-180.js +17 -10
- package/test/regression-182.js +7 -3
- package/test/regression-185.js +4 -9
- package/test/regression-187.js +11 -2
- package/test/regression-189.js +224 -0
- package/test/regression-190.js +111 -0
- package/test/regression-191.js +160 -0
- package/test/regression-192.js +66 -0
- package/test/regression-193.js +251 -0
- package/test/regression-194.js +182 -0
- package/test/regression-198.js +281 -0
- package/test/regression-199.js +79 -0
- package/test/regression-201.js +25 -0
- package/test/regression-202.js +92 -0
- package/test/regression-91.js +23 -20
- package/test/shared-golden.js +54 -0
- package/test/status-query.js +97 -15
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,19 @@
|
|
|
1
1
|
# start-command
|
|
2
2
|
|
|
3
|
+
## 0.37.0
|
|
4
|
+
|
|
5
|
+
### Minor Changes
|
|
6
|
+
|
|
7
|
+
- 94db203: Add copy-free Docker command resumes with guarded legacy snapshots, persistent
|
|
8
|
+
task labels, wrapper help flags, private sanitized log uploads, and faithful shell
|
|
9
|
+
argument and exit-code handling. Enforce JavaScript-first CI and shared Rust parity.
|
|
10
|
+
|
|
11
|
+
## 0.36.0
|
|
12
|
+
|
|
13
|
+
### Minor Changes
|
|
14
|
+
|
|
15
|
+
- dbf7ba6: Add Docker launch/recovery resource limits with daemon percentages and random ranges, opt-in delayed CPU penalties, and remote/DinD memory diagnostics. Reserve launches before side effects, atomically publish store locks and database writes, and require exit-time evidence for OOM verdicts while recognizing attributed Docker daemon restarts.
|
|
16
|
+
|
|
3
17
|
## 0.35.4
|
|
4
18
|
|
|
5
19
|
### Patch Changes
|
package/README.md
CHANGED
|
@@ -24,6 +24,8 @@ $ ls -la
|
|
|
24
24
|
$ bun test
|
|
25
25
|
$ git status
|
|
26
26
|
$ --list
|
|
27
|
+
$ --help
|
|
28
|
+
$ -i docker -d --label task.tool=codex -- my-task
|
|
27
29
|
$ --attach <id>
|
|
28
30
|
$ --resume <id> -- <command>
|
|
29
31
|
$ --resume-all
|
|
@@ -34,6 +36,10 @@ See the project-wide [README](../README.md), [docs/USAGE.md](../docs/USAGE.md),
|
|
|
34
36
|
[docs/EXAMPLES.md](../docs/EXAMPLES.md) for the full user-facing guide and
|
|
35
37
|
checked examples.
|
|
36
38
|
|
|
39
|
+
`-h` also prints usage. Labels persist across Docker resumes. New detached
|
|
40
|
+
containers support copy-free command handoff; legacy snapshots require capacity
|
|
41
|
+
preflight. Log uploads are private and sanitized; manual `--no-sanitize` opts out.
|
|
42
|
+
|
|
37
43
|
## Development
|
|
38
44
|
|
|
39
45
|
```bash
|
|
@@ -56,3 +62,13 @@ GitHub Releases. The release title carries the `[JavaScript]` prefix, e.g.
|
|
|
56
62
|
## License
|
|
57
63
|
|
|
58
64
|
Released into the public domain under the [Unlicense](../LICENSE).
|
|
65
|
+
|
|
66
|
+
### Docker resource and recovery controls
|
|
67
|
+
|
|
68
|
+
Launch with `--memory 64m`, `--memory-swap 64m` and `--cpus 1`, or use daemon
|
|
69
|
+
percentages and uniform ranges such as `--memory '70%-80%'`. Memory-swap defaults
|
|
70
|
+
to memory. `--cpu-penalty` enables delayed caps and reports state through status.
|
|
71
|
+
Manual `--resume <id> --memory 128m` updates before starting;
|
|
72
|
+
`--on-kill-resume-memory` changes memory only after a fresh qualifying OOM.
|
|
73
|
+
See the [full resource and recovery guide](../README.md#docker-launch-limits-recovery-limits-and-cpu-penalty)
|
|
74
|
+
and [incident research](../docs/case-studies/issue-195/README.md).
|
package/package.json
CHANGED
package/src/bin/cli.js
CHANGED
|
@@ -192,6 +192,10 @@ try {
|
|
|
192
192
|
}
|
|
193
193
|
|
|
194
194
|
const { wrapperOptions, command: parsedCommand } = parsedArgs;
|
|
195
|
+
if (wrapperOptions.help) {
|
|
196
|
+
printUsage();
|
|
197
|
+
process.exit(0);
|
|
198
|
+
}
|
|
195
199
|
|
|
196
200
|
// Query/control modes (--status, --list, --attach, --resume, ...) address an
|
|
197
201
|
// execution that already exists, so they never need a command of their own.
|
|
@@ -241,6 +245,18 @@ const sessionId = wrapperOptions.sessionId || generateUUID();
|
|
|
241
245
|
process.exit(queryExitCode === null ? 0 : queryExitCode);
|
|
242
246
|
}
|
|
243
247
|
|
|
248
|
+
if (wrapperOptions.isolated === 'docker') {
|
|
249
|
+
const {
|
|
250
|
+
resolveResourceOptions,
|
|
251
|
+
} = require('../lib/docker-resource-options');
|
|
252
|
+
Object.assign(
|
|
253
|
+
wrapperOptions,
|
|
254
|
+
resolveResourceOptions(
|
|
255
|
+
wrapperOptions,
|
|
256
|
+
require('../lib/execution-control').runCommand
|
|
257
|
+
)
|
|
258
|
+
);
|
|
259
|
+
}
|
|
244
260
|
// Check if running in isolation mode or with user isolation
|
|
245
261
|
if (hasIsolation(wrapperOptions) || wrapperOptions.user) {
|
|
246
262
|
await runWithIsolation(
|
|
@@ -419,14 +435,21 @@ async function runWithIsolation(
|
|
|
419
435
|
// Save initial execution record and set global reference for signal cleanup
|
|
420
436
|
if (executionRecord && store) {
|
|
421
437
|
currentExecutionRecord = executionRecord;
|
|
438
|
+
require('../lib/launch-owner').markLaunch(executionRecord);
|
|
422
439
|
try {
|
|
423
440
|
store.save(executionRecord);
|
|
424
441
|
} catch (err) {
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
442
|
+
console.error(
|
|
443
|
+
JSON.stringify({
|
|
444
|
+
code: 'LAUNCH_RESERVATION_FAILED',
|
|
445
|
+
uuid: executionRecord.uuid,
|
|
446
|
+
containerName: sessionName,
|
|
447
|
+
error: err.message,
|
|
448
|
+
})
|
|
449
|
+
);
|
|
450
|
+
currentExecutionRecord = null;
|
|
451
|
+
process.exitCode = 1;
|
|
452
|
+
return 1;
|
|
430
453
|
}
|
|
431
454
|
}
|
|
432
455
|
|
|
@@ -458,7 +481,9 @@ async function runWithIsolation(
|
|
|
458
481
|
// Note: Isolation environments currently use native spawn/execSync
|
|
459
482
|
// Future: Add command-stream support with raw() function for multiplexers
|
|
460
483
|
result = await runIsolated(environment, cmd, {
|
|
484
|
+
...options,
|
|
461
485
|
session: sessionName,
|
|
486
|
+
sessionId,
|
|
462
487
|
image: effectiveImage,
|
|
463
488
|
endpoint: options.endpoint,
|
|
464
489
|
detached: mode === 'detached',
|
|
@@ -502,6 +527,11 @@ async function runWithIsolation(
|
|
|
502
527
|
|
|
503
528
|
// Update execution record: detached keeps "executing" (resolved at query time)
|
|
504
529
|
if (executionRecord && store) {
|
|
530
|
+
executionRecord.options.launchPending = false;
|
|
531
|
+
if (result.diagnostics) {
|
|
532
|
+
executionRecord.cgroupMemory = result.diagnostics.cgroupMemory;
|
|
533
|
+
executionRecord.options.exitEvidence = result.diagnostics.exitEvidence;
|
|
534
|
+
}
|
|
505
535
|
if (result.containerId) {
|
|
506
536
|
executionRecord.options.containerId = result.containerId;
|
|
507
537
|
}
|
|
@@ -515,11 +545,22 @@ async function runWithIsolation(
|
|
|
515
545
|
: executionRecord
|
|
516
546
|
);
|
|
517
547
|
} catch (err) {
|
|
518
|
-
if (
|
|
548
|
+
if (environment === 'docker' && mode === 'detached' && result.success) {
|
|
519
549
|
console.error(
|
|
520
|
-
|
|
550
|
+
require('../lib/launch-persistence').failLaunchedPersistence(
|
|
551
|
+
store,
|
|
552
|
+
executionRecord,
|
|
553
|
+
require('../lib/execution-control').runCommand,
|
|
554
|
+
err
|
|
555
|
+
)
|
|
521
556
|
);
|
|
557
|
+
currentExecutionRecord = null;
|
|
558
|
+
process.exitCode = 1;
|
|
559
|
+
return 1;
|
|
522
560
|
}
|
|
561
|
+
console.error(
|
|
562
|
+
`[Tracking] Could not update execution record: ${err.message}`
|
|
563
|
+
);
|
|
523
564
|
}
|
|
524
565
|
currentExecutionRecord = null;
|
|
525
566
|
}
|
|
@@ -26,6 +26,8 @@ function createQueryOptionDefaults() {
|
|
|
26
26
|
list: false, // List all tracked execution records
|
|
27
27
|
running: false, // Restrict --list to executions that are still running
|
|
28
28
|
uploadLog: null, // UUID/session name whose stored log should be uploaded
|
|
29
|
+
noSanitize: false,
|
|
30
|
+
removeOriginal: false,
|
|
29
31
|
outputFormat: null, // Output format (links-notation, json, text)
|
|
30
32
|
stop: null, // UUID/session name to stop gracefully
|
|
31
33
|
terminate: null, // UUID/session name to terminate immediately
|
|
@@ -55,6 +57,12 @@ const IDENTIFIER_OPTIONS = {
|
|
|
55
57
|
* Boolean flags with no argument.
|
|
56
58
|
*/
|
|
57
59
|
const FLAG_OPTIONS = {
|
|
60
|
+
'--no-sanitize': (options) => {
|
|
61
|
+
options.noSanitize = true;
|
|
62
|
+
},
|
|
63
|
+
'--remove-original': (options) => {
|
|
64
|
+
options.removeOriginal = true;
|
|
65
|
+
},
|
|
58
66
|
'--list': (options) => {
|
|
59
67
|
options.list = true;
|
|
60
68
|
},
|
|
@@ -185,6 +193,12 @@ const OUTPUT_FORMAT_MODES = ['--status', '--list', '--resume', '--resume-all'];
|
|
|
185
193
|
* @throws {Error} When the combination is not supported
|
|
186
194
|
*/
|
|
187
195
|
function validateQueryOptions(options) {
|
|
196
|
+
if (options.noSanitize && !hasValue(options.uploadLog)) {
|
|
197
|
+
throw new Error('--no-sanitize is only valid with --upload-log');
|
|
198
|
+
}
|
|
199
|
+
if (options.removeOriginal && !hasValue(options.resume)) {
|
|
200
|
+
throw new Error('--remove-original is only valid with --resume');
|
|
201
|
+
}
|
|
188
202
|
if (hasValue(options.outputFormat)) {
|
|
189
203
|
if (!VALID_OUTPUT_FORMATS.includes(options.outputFormat)) {
|
|
190
204
|
throw new Error(
|
package/src/lib/args-parser.js
CHANGED
|
@@ -44,6 +44,8 @@
|
|
|
44
44
|
const { getDefaultDockerImage } = require('./docker-utils');
|
|
45
45
|
const dockerNetworkOptions = require('./docker-network-options');
|
|
46
46
|
const dockerRecoveryOptions = require('./docker-recovery-options');
|
|
47
|
+
const cpuPenalty = require('./cpu-penalty');
|
|
48
|
+
const dockerResourceOptions = require('./docker-resource-options');
|
|
47
49
|
const { parseSequence, isSequence } = require('./sequence-parser');
|
|
48
50
|
const { buildCommandString } = require('./shell-utils');
|
|
49
51
|
const {
|
|
@@ -191,6 +193,8 @@ function parseArgs(args) {
|
|
|
191
193
|
volumes: [], // Docker bind mounts/volumes (-v/--volume), applied to docker levels
|
|
192
194
|
mounts: [], // Docker --mount specs, applied to docker levels
|
|
193
195
|
env: [], // Docker environment variables (-e/--env, KEY=VALUE), applied to docker levels
|
|
196
|
+
labels: [], // Caller attribution, applied to every Docker level
|
|
197
|
+
help: false,
|
|
194
198
|
privileged: false, // Run docker container in privileged mode
|
|
195
199
|
network: null, // First Docker network name (compatibility accessor)
|
|
196
200
|
networks: [], // Ordered Docker network names
|
|
@@ -245,7 +249,9 @@ function parseArgs(args) {
|
|
|
245
249
|
}
|
|
246
250
|
|
|
247
251
|
// Validate options
|
|
248
|
-
|
|
252
|
+
if (!wrapperOptions.help) {
|
|
253
|
+
validateOptions(wrapperOptions);
|
|
254
|
+
}
|
|
249
255
|
|
|
250
256
|
return {
|
|
251
257
|
wrapperOptions,
|
|
@@ -286,6 +292,16 @@ function parseWrapperArgs(args, options) {
|
|
|
286
292
|
*/
|
|
287
293
|
function parseOption(args, index, options) {
|
|
288
294
|
const arg = args[index];
|
|
295
|
+
if (arg === '--help' || arg === '-h') {
|
|
296
|
+
options.help = true;
|
|
297
|
+
return 1;
|
|
298
|
+
}
|
|
299
|
+
if (arg === '--label' || arg.startsWith('--label=')) {
|
|
300
|
+
const label = arg === '--label' ? args[index + 1] : arg.slice(8);
|
|
301
|
+
require('./docker-labels').validateLabel(label);
|
|
302
|
+
options.labels.push(label);
|
|
303
|
+
return arg === '--label' ? 2 : 1;
|
|
304
|
+
}
|
|
289
305
|
|
|
290
306
|
// --isolated, --isolation, or -i
|
|
291
307
|
if (arg === '--isolated' || arg === '--isolation' || arg === '-i') {
|
|
@@ -414,6 +430,14 @@ function parseOption(args, index, options) {
|
|
|
414
430
|
return networkOption;
|
|
415
431
|
}
|
|
416
432
|
|
|
433
|
+
const cpuConsumed = cpuPenalty.parse(args, index, options);
|
|
434
|
+
if (cpuConsumed) {
|
|
435
|
+
return cpuConsumed;
|
|
436
|
+
}
|
|
437
|
+
const resourceOption = dockerResourceOptions.parse(args, index, options);
|
|
438
|
+
if (resourceOption) {
|
|
439
|
+
return resourceOption;
|
|
440
|
+
}
|
|
417
441
|
const recoveryOption = dockerRecoveryOptions.parse(args, index, options);
|
|
418
442
|
if (recoveryOption) {
|
|
419
443
|
return recoveryOption;
|
|
@@ -558,6 +582,11 @@ function parseOption(args, index, options) {
|
|
|
558
582
|
* @throws {Error} If a docker-only option is set without docker isolation
|
|
559
583
|
*/
|
|
560
584
|
function validateDockerRuntimeOptionsRequireDocker(options) {
|
|
585
|
+
if (options.labels?.length > 0 && !options.resume) {
|
|
586
|
+
throw new Error(
|
|
587
|
+
'--label option is only valid with docker isolation or --resume'
|
|
588
|
+
);
|
|
589
|
+
}
|
|
561
590
|
if (options.volumes && options.volumes.length > 0) {
|
|
562
591
|
throw new Error(
|
|
563
592
|
'--volume option is only valid when isolation stack includes docker'
|
|
@@ -756,6 +785,8 @@ function validateOptions(options) {
|
|
|
756
785
|
}
|
|
757
786
|
|
|
758
787
|
dockerRecoveryOptions.validate(options);
|
|
788
|
+
dockerResourceOptions.validate(options);
|
|
789
|
+
cpuPenalty.validate(options);
|
|
759
790
|
|
|
760
791
|
// Session name is only valid with isolation
|
|
761
792
|
if (options.session && !options.isolated) {
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
/** Keep the previous database intact when a write fails, including ENOSPC. */
|
|
2
|
+
const fs = require('fs');
|
|
3
|
+
const path = require('path');
|
|
4
|
+
const crypto = require('crypto');
|
|
5
|
+
|
|
6
|
+
function atomicWrite(file, content) {
|
|
7
|
+
const temporary = `${file}.${process.pid}.${crypto.randomUUID()}.tmp`;
|
|
8
|
+
try {
|
|
9
|
+
fs.writeFileSync(temporary, content, { flag: 'wx' });
|
|
10
|
+
const handle = fs.openSync(temporary, 'r+');
|
|
11
|
+
try {
|
|
12
|
+
fs.fsyncSync(handle);
|
|
13
|
+
} finally {
|
|
14
|
+
fs.closeSync(handle);
|
|
15
|
+
}
|
|
16
|
+
fs.renameSync(temporary, file);
|
|
17
|
+
if (process.platform !== 'win32') {
|
|
18
|
+
const directory = fs.openSync(path.dirname(file), 'r');
|
|
19
|
+
try {
|
|
20
|
+
fs.fsyncSync(directory);
|
|
21
|
+
} finally {
|
|
22
|
+
fs.closeSync(directory);
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
} finally {
|
|
26
|
+
try {
|
|
27
|
+
fs.unlinkSync(temporary);
|
|
28
|
+
} catch {
|
|
29
|
+
/* Renamed or never created. */
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
module.exports = { atomicWrite };
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
/** Keep the same memory and CPU monitors alive during an attached Docker run. */
|
|
2
|
+
const { spawn, spawnSync } = require('child_process');
|
|
3
|
+
const fs = require('fs');
|
|
4
|
+
const os = require('os');
|
|
5
|
+
const path = require('path');
|
|
6
|
+
const crypto = require('crypto');
|
|
7
|
+
const cgroup = require('./cgroup-memory');
|
|
8
|
+
const cpu = require('./cpu-penalty-monitor');
|
|
9
|
+
const evidence = require('./exit-evidence');
|
|
10
|
+
const { shellQuote } = require('./isolation-log-utils');
|
|
11
|
+
|
|
12
|
+
function startAttachedDiagnostics(name, options) {
|
|
13
|
+
const file = path.join(os.tmpdir(), `start-attached-${crypto.randomUUID()}`);
|
|
14
|
+
let worker;
|
|
15
|
+
{
|
|
16
|
+
const script = [
|
|
17
|
+
cgroup.buildCgroupSamplerStartSnippet(name),
|
|
18
|
+
cpu.startSnippet(options.executionId, name, 1, options),
|
|
19
|
+
'read __start_stop',
|
|
20
|
+
cgroup.buildCgroupSamplerStopSnippet(),
|
|
21
|
+
cpu.stopSnippet(),
|
|
22
|
+
`printf '%s' "$__start_command_cgroup" > ${shellQuote(file)}`,
|
|
23
|
+
`printf '%s' "$__start_command_memory_unavailable" > ${shellQuote(`${file}.reason`)}`,
|
|
24
|
+
]
|
|
25
|
+
.filter(Boolean)
|
|
26
|
+
.join('; ');
|
|
27
|
+
worker = spawn('sh', ['-c', script], {
|
|
28
|
+
stdio: ['pipe', 'ignore', 'ignore'],
|
|
29
|
+
});
|
|
30
|
+
worker.on('error', () => {});
|
|
31
|
+
// Subscribe before a fast command finishes, including shell startup failure.
|
|
32
|
+
worker.done = new Promise((resolve) => worker.once('close', resolve));
|
|
33
|
+
worker.stdin.on('error', () => {});
|
|
34
|
+
}
|
|
35
|
+
return {
|
|
36
|
+
async finish(state) {
|
|
37
|
+
if (worker) {
|
|
38
|
+
worker.stdin.end();
|
|
39
|
+
await worker.done;
|
|
40
|
+
}
|
|
41
|
+
let sample = '';
|
|
42
|
+
let unavailable = 'sampler could not start (sh unavailable)';
|
|
43
|
+
try {
|
|
44
|
+
sample = fs.readFileSync(file, 'utf8');
|
|
45
|
+
unavailable =
|
|
46
|
+
fs.readFileSync(`${file}.reason`, 'utf8') ||
|
|
47
|
+
'sampler stopped before a reading';
|
|
48
|
+
} catch {
|
|
49
|
+
/* Optional diagnostics are best effort. */
|
|
50
|
+
}
|
|
51
|
+
try {
|
|
52
|
+
fs.unlinkSync(file);
|
|
53
|
+
fs.unlinkSync(`${file}.reason`);
|
|
54
|
+
} catch {
|
|
55
|
+
/* Optional diagnostics are best effort. */
|
|
56
|
+
}
|
|
57
|
+
const docker = require('./docker-cleanup').getDockerCommand();
|
|
58
|
+
const limit = spawnSync(
|
|
59
|
+
docker,
|
|
60
|
+
['inspect', '-f', '{{.HostConfig.Memory}}', name],
|
|
61
|
+
{ encoding: 'utf8' }
|
|
62
|
+
);
|
|
63
|
+
const line = evidence.collectExitEvidence({
|
|
64
|
+
containerName: name,
|
|
65
|
+
sample,
|
|
66
|
+
finishedAt: state?.finishedAt,
|
|
67
|
+
exitCode: state?.exitCode,
|
|
68
|
+
oomKilled: state?.oomKilled,
|
|
69
|
+
});
|
|
70
|
+
if (options.logPath) {
|
|
71
|
+
try {
|
|
72
|
+
fs.appendFileSync(
|
|
73
|
+
options.logPath,
|
|
74
|
+
`${cgroup.formatCgroupMemoryLogLine(sample, limit.status === 0 ? limit.stdout.trim() : 'unknown', unavailable)}\n${line}\n`
|
|
75
|
+
);
|
|
76
|
+
} catch (error) {
|
|
77
|
+
if (process.env.START_DEBUG === '1') {
|
|
78
|
+
console.error(`[DEBUG] Attached diagnostics: ${error.message}`);
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
return {
|
|
83
|
+
cgroupMemory: cgroup.parseCgroupMemorySample(sample),
|
|
84
|
+
exitEvidence: evidence.fromLog(line),
|
|
85
|
+
};
|
|
86
|
+
},
|
|
87
|
+
};
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
module.exports = { startAttachedDiagnostics };
|
package/src/lib/cgroup-memory.js
CHANGED
|
@@ -67,6 +67,7 @@ function buildCgroupFunctionsSnippet() {
|
|
|
67
67
|
'__start_command_cgroup_read() { __scr_events=$(cat "$1/memory.events" 2>/dev/null) || return 1; ' +
|
|
68
68
|
"__scr_oom=$(printf '%s\\n' \"$__scr_events\" | sed -n 's/^oom //p'); " +
|
|
69
69
|
"__scr_kill=$(printf '%s\\n' \"$__scr_events\" | sed -n 's/^oom_kill //p'); " +
|
|
70
|
+
'case "$__scr_oom:$__scr_kill" in *[!0-9:]*|:*|*:) return 1;; esac; ' +
|
|
70
71
|
'__scr_max=$(cat "$1/memory.max" 2>/dev/null); __scr_peak=$(cat "$1/memory.peak" 2>/dev/null); ' +
|
|
71
72
|
'printf \'%s %s %s %s %s\\n\' "${__scr_max:--}" "${__scr_peak:--}" ' +
|
|
72
73
|
'"${__scr_oom:--}" "${__scr_kill:--}" "$1"; }',
|
|
@@ -82,13 +83,33 @@ function buildCgroupFunctionsSnippet() {
|
|
|
82
83
|
*/
|
|
83
84
|
function buildCgroupSamplerStartSnippet(containerName) {
|
|
84
85
|
const v = CGROUP_SHELL_VARS;
|
|
86
|
+
const name = shellQuote(containerName);
|
|
87
|
+
const remoteRead = '__start_command_cgroup_read /sys/fs/cgroup';
|
|
85
88
|
return [
|
|
86
89
|
buildCgroupFunctionsSnippet(),
|
|
87
90
|
`${v.file}="\${TMPDIR:-/tmp}/start-command-cgroup.$$"`,
|
|
88
|
-
`rm -f "$${v.file}" "$${v.file}.tmp"`,
|
|
89
|
-
`
|
|
90
|
-
|
|
91
|
-
|
|
91
|
+
`rm -f "$${v.file}" "$${v.file}.tmp" "$${v.file}.reason"`,
|
|
92
|
+
`__start_command_memory_limit=$(docker inspect -f '{{.HostConfig.Memory}}' ${name} 2>/dev/null)`,
|
|
93
|
+
`__start_command_sample_name=${name}`,
|
|
94
|
+
// The first counter is a baseline, including after watcher restart. Only
|
|
95
|
+
// changes between adjacent successful samples supply exit-time evidence.
|
|
96
|
+
`( export START_COMMAND_CGROUP_SAMPLING=1; __scs_previous=''; __scs_previous_at=0; __scs_changed=0; ` +
|
|
97
|
+
`__scs_dir=$(__start_command_cgroup_dir ${name}); while :; do ` +
|
|
98
|
+
'__scs_line=""; if [ -n "$__scs_dir" ]; then __scs_line=$(__start_command_cgroup_read "$__scs_dir"); fi; ' +
|
|
99
|
+
`if [ -z "$__scs_line" ]; then __scs_ns=$(docker inspect -f '{{.HostConfig.CgroupnsMode}}' ${name} 2>/dev/null); ` +
|
|
100
|
+
`if [ "$__scs_ns" = private ]; then ` +
|
|
101
|
+
`__scs_remote=$(docker exec ${name} sh -c ${shellQuote(`${buildCgroupFunctionsSnippet()}; ${remoteRead}`)} 2>&1); __scs_exit=$?; ` +
|
|
102
|
+
'if [ "$__scs_exit" = 0 ]; then __scs_line=$__scs_remote; ' +
|
|
103
|
+
'else __scs_detail=$(printf "%s" "$__scs_remote" | sed -n "1p" | cut -c 1-240); ' +
|
|
104
|
+
`printf '%s\n' "local cgroup unavailable; Docker exec exit=$__scs_exit: \${__scs_detail:-private cgroup v2 counters unreadable}" > "$${v.file}.reason"; fi; ` +
|
|
105
|
+
`else printf '%s\n' "local cgroup unavailable; container cgroup namespace=\${__scs_ns:-unavailable}, private required" > "$${v.file}.reason"; fi; fi; ` +
|
|
106
|
+
'__scs_now=$(date +%s); if [ -n "$__scs_line" ]; then ' +
|
|
107
|
+
'__scs_kill=$(printf "%s" "$__scs_line" | cut -d " " -f 4); ' +
|
|
108
|
+
'if [ -n "$__scs_previous" ] && [ "$__scs_kill" -gt "$__scs_previous" ] 2>/dev/null && [ "$((__scs_now-__scs_previous_at))" -le 3 ]; then __scs_changed=$__scs_now; fi; ' +
|
|
109
|
+
'__scs_previous=$__scs_kill; __scs_previous_at=$__scs_now; ' +
|
|
110
|
+
'__scs_fields=$(printf "%s" "$__scs_line" | cut -d " " -f 1-4); ' +
|
|
111
|
+
`printf '%s %s %s %s\\n' "$__scs_fields" "$__scs_changed" "$__scs_now" "$__scs_dir" > "$${v.file}.tmp" && mv -f "$${v.file}.tmp" "$${v.file}"; fi; ` +
|
|
112
|
+
`if [ "$(docker inspect -f '{{.State.Running}}' ${name} 2>/dev/null)" = false ]; then break; fi; ` +
|
|
92
113
|
`sleep ${CGROUP_SAMPLE_INTERVAL_SECONDS}; done ) >/dev/null 2>&1 & ${v.sampler}=$!`,
|
|
93
114
|
].join('; ');
|
|
94
115
|
}
|
|
@@ -106,10 +127,16 @@ function buildCgroupSamplerStopSnippet() {
|
|
|
106
127
|
`kill "$${v.sampler}" 2>/dev/null`,
|
|
107
128
|
`wait "$${v.sampler}" 2>/dev/null`,
|
|
108
129
|
`${v.sample}=$(cat "$${v.file}" 2>/dev/null)`,
|
|
109
|
-
`
|
|
110
|
-
`
|
|
111
|
-
`
|
|
112
|
-
|
|
130
|
+
`__start_command_memory_unavailable=$(cat "$${v.file}.reason" 2>/dev/null)`,
|
|
131
|
+
`__scs_dir=$(printf '%s' "$${v.sample}" | cut -s -d ' ' -f 7-)`,
|
|
132
|
+
`if [ -n "$__scs_dir" ] && __scs_line=$(__start_command_cgroup_read "$__scs_dir"); then ` +
|
|
133
|
+
`__scs_old=$(printf '%s' "$${v.sample}" | cut -d ' ' -f 4); __scs_at=$(printf '%s' "$${v.sample}" | cut -d ' ' -f 6); ` +
|
|
134
|
+
`__scs_changed=$(printf '%s' "$${v.sample}" | cut -d ' ' -f 5); __scs_now=$(date +%s); ` +
|
|
135
|
+
`__scs_new=$(printf '%s' "$__scs_line" | cut -d ' ' -f 4); ` +
|
|
136
|
+
`if [ "$__scs_new" -gt "$__scs_old" ] 2>/dev/null && [ "$((__scs_now-__scs_at))" -le 3 ]; then __scs_changed=$__scs_now; fi; ` +
|
|
137
|
+
`${v.sample}="$(printf '%s' "$__scs_line" | cut -d ' ' -f 1-4) $__scs_changed $__scs_now"; fi`,
|
|
138
|
+
`rm -f "$${v.file}" "$${v.file}.tmp" "$${v.file}.reason"`,
|
|
139
|
+
`${v.sample}=$(printf '%s' "$${v.sample}" | cut -d ' ' -f 1-6)`,
|
|
113
140
|
].join('; ');
|
|
114
141
|
}
|
|
115
142
|
|
|
@@ -128,7 +155,8 @@ function buildCgroupMemoryLogSnippet(quotedLogPath) {
|
|
|
128
155
|
`__scm_note=' (${OOM_SCOPE_NOTES[OOM_SCOPE.UNKNOWN]})'; fi; ` +
|
|
129
156
|
`printf 'Memory: memory.max=%s memory.peak=%s oom=%s oom_kill=%s%s\\n' ` +
|
|
130
157
|
`"$1" "$2" "$3" "$4" "$__scm_note"; }; ` +
|
|
131
|
-
`if [ -n "$${v.sample}" ]; then __start_command_cgroup_log $${v.sample} >> ${quotedLogPath};
|
|
158
|
+
`if [ -n "$${v.sample}" ]; then __start_command_cgroup_log $${v.sample} >> ${quotedLogPath}; ` +
|
|
159
|
+
`else printf 'Memory: unavailable (%s) memory.limit=%s (HostConfig)\\n' "\${__start_command_memory_unavailable:-sampler stopped before a reading; local cgroup unavailable}" "\${__start_command_memory_limit:-unknown}" >> ${quotedLogPath}; fi`
|
|
132
160
|
);
|
|
133
161
|
}
|
|
134
162
|
|
|
@@ -138,10 +166,14 @@ function buildCgroupMemoryLogSnippet(quotedLogPath) {
|
|
|
138
166
|
* @param {string|null|undefined} text - `<memory.max> <memory.peak> <oom> <oom_kill>`
|
|
139
167
|
* @returns {?string} Line without a trailing newline, or null without a sample
|
|
140
168
|
*/
|
|
141
|
-
function formatCgroupMemoryLogLine(
|
|
169
|
+
function formatCgroupMemoryLogLine(
|
|
170
|
+
text,
|
|
171
|
+
memoryLimit = 'unknown',
|
|
172
|
+
unavailable = 'sampler unavailable; local cgroup hidden or unreadable'
|
|
173
|
+
) {
|
|
142
174
|
const counters = parseCgroupMemorySample(text);
|
|
143
175
|
if (!counters) {
|
|
144
|
-
return
|
|
176
|
+
return `Memory: unavailable (${unavailable}) memory.limit=${memoryLimit} (HostConfig)`;
|
|
145
177
|
}
|
|
146
178
|
const [max, peak, oom, kill] = String(text).trim().split(/\s+/);
|
|
147
179
|
const scope = describeCgroupOomScope(counters);
|
|
@@ -57,6 +57,46 @@ function buildNextLevelCommand(options, command) {
|
|
|
57
57
|
// Docker runtime options are flat (not per-level); forward them only when a
|
|
58
58
|
// remaining level still uses docker so the nested $ invocation can apply them.
|
|
59
59
|
if (remainingStack.includes('docker')) {
|
|
60
|
+
for (const [key, flag] of [
|
|
61
|
+
['memory', '--memory'],
|
|
62
|
+
['memorySwap', '--memory-swap'],
|
|
63
|
+
['cpus', '--cpus'],
|
|
64
|
+
['onKillResumeMemory', '--on-kill-resume-memory'],
|
|
65
|
+
]) {
|
|
66
|
+
if (options[key] !== null && options[key] !== undefined) {
|
|
67
|
+
parts.push(
|
|
68
|
+
`${flag} ${require('./shell-utils').quoteShellArg(String(options[key]))}`
|
|
69
|
+
);
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
if (options.cpuPenaltyConfig) {
|
|
73
|
+
parts.push('--cpu-penalty');
|
|
74
|
+
for (const [key, flag] of [
|
|
75
|
+
['cpus', '--cpu-penalty-cpus'],
|
|
76
|
+
['trigger', '--cpu-penalty-trigger'],
|
|
77
|
+
['triggerWindowMs', '--cpu-penalty-trigger-window'],
|
|
78
|
+
['release', '--cpu-penalty-release'],
|
|
79
|
+
['releaseWindowMs', '--cpu-penalty-release-window'],
|
|
80
|
+
]) {
|
|
81
|
+
parts.push(
|
|
82
|
+
`${flag} ${options.cpuPenaltyConfig[key]}${key.endsWith('Ms') ? 'ms' : ''}`
|
|
83
|
+
);
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
if (options.onKillResume !== null && options.onKillResume !== undefined) {
|
|
87
|
+
parts.push(`--on-kill-resume ${options.onKillResume}`);
|
|
88
|
+
}
|
|
89
|
+
if (
|
|
90
|
+
options.onKillResumeDelay !== null &&
|
|
91
|
+
options.onKillResumeDelay !== undefined
|
|
92
|
+
) {
|
|
93
|
+
parts.push(`--on-kill-resume-delay ${options.onKillResumeDelay}`);
|
|
94
|
+
}
|
|
95
|
+
if (options.recoveryCommand) {
|
|
96
|
+
parts.push(
|
|
97
|
+
`--recovery-command ${require('./shell-utils').quoteShellArg(options.recoveryCommand)}`
|
|
98
|
+
);
|
|
99
|
+
}
|
|
60
100
|
for (const volume of options.volumes || []) {
|
|
61
101
|
parts.push(`--volume "${volume}"`);
|
|
62
102
|
}
|
|
@@ -66,6 +106,9 @@ function buildNextLevelCommand(options, command) {
|
|
|
66
106
|
for (const envVar of options.env || []) {
|
|
67
107
|
parts.push(`--env "${envVar}"`);
|
|
68
108
|
}
|
|
109
|
+
for (const label of options.labels || []) {
|
|
110
|
+
parts.push(`--label ${require('./shell-utils').quoteShellArg(label)}`);
|
|
111
|
+
}
|
|
69
112
|
if (options.privileged) {
|
|
70
113
|
parts.push('--privileged');
|
|
71
114
|
}
|