start-command 0.34.1 → 0.35.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/package.json +3 -3
- package/src/lib/args-parser.js +10 -0
- package/src/lib/docker-cleanup.js +28 -5
- package/src/lib/docker-recovery-options.js +84 -0
- package/src/lib/docker-resource-limits.js +211 -0
- package/src/lib/docker-runtime-args.js +141 -0
- package/src/lib/execution-control.js +39 -0
- package/src/lib/execution-recovery.js +364 -0
- package/src/lib/execution-resume.js +68 -4
- package/src/lib/isolation.js +20 -92
- package/src/lib/usage.js +3 -0
- package/test/docker-runtime-options.js +6 -0
- package/test/regression-176.js +897 -0
- package/test/regression-178.js +232 -0
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,17 @@
|
|
|
1
1
|
# start-command
|
|
2
2
|
|
|
3
|
+
## 0.35.1
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- 27737e3: Do not resume `--on-kill-resume` executions whose main process exited 0–127 on its own while Docker's sticky `OOMKilled` flag was set. Docker sets `OOMKilled` when _any_ process in the container was OOM-killed (a compiler, a test runner, a child `node`) and keeps it until the next start, so a run that survived that and later exited 0 or 1 was resumed as if it had been killed. A run now counts as killed only on exit 137, or on `OOMKilled` without a usable exit code; `oomKilled: true` is still reported in `--status` and the post-mortem.
|
|
8
|
+
|
|
9
|
+
## 0.35.0
|
|
10
|
+
|
|
11
|
+
### Minor Changes
|
|
12
|
+
|
|
13
|
+
- 9679323: Keep Docker resource limits across `--resume <id> -- <command>`, and add `--on-kill-resume <N>` / `--recovery-command <cmd>`. `docker commit` does not capture HostConfig, so the snapshot resume now reads the stopped container's limits with `docker inspect` (including ones applied later with `docker update`) and re-applies the non-default ones (`--memory`, `--memory-swap`, `--memory-reservation`, `--cpus`/`--cpu-quota`/`--cpu-period`, `--cpu-shares`, `--cpuset-cpus`, `--cpuset-mems`, `--pids-limit`, `--shm-size`, `--storage-opt`, `--ulimit`) to the `-resume-N` container, prints them as `[Isolation] Resource limits: ...` and stores them as `resourceLimits`. With `--on-kill-resume <N>` a detached Docker execution whose main process is killed (exit 137 or `OOMKilled`) is resumed in the same container up to N times, running `--recovery-command` (or the original command) with `START_COMMAND_RECOVERY_ATTEMPT` set. The UUID and log file are kept, each attempt is separated by a `[Recovery k/N]` line, and `--status` shows `recoveryAttempts` and `recoveryHistory`. `--stop` cancels further recovery.
|
|
14
|
+
|
|
3
15
|
## 0.34.1
|
|
4
16
|
|
|
5
17
|
### Patch Changes
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "start-command",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.35.1",
|
|
4
4
|
"description": "Gamification of coding, execute any command with ability to auto-report issues on GitHub",
|
|
5
5
|
"main": "src/bin/cli.js",
|
|
6
6
|
"exports": {
|
|
@@ -47,7 +47,7 @@
|
|
|
47
47
|
},
|
|
48
48
|
"homepage": "https://github.com/link-foundation/start#readme",
|
|
49
49
|
"devDependencies": {
|
|
50
|
-
"@changesets/cli": "^
|
|
50
|
+
"@changesets/cli": "^3.0.3",
|
|
51
51
|
"@eslint/js": "^10.0.1",
|
|
52
52
|
"eslint": "^10.3.0",
|
|
53
53
|
"eslint-config-prettier": "^10.1.8",
|
|
@@ -72,7 +72,7 @@
|
|
|
72
72
|
"lino-objects-codec": "^0.4.0"
|
|
73
73
|
},
|
|
74
74
|
"overrides": {
|
|
75
|
-
"brace-expansion": "^5.0.
|
|
75
|
+
"brace-expansion": "^5.0.12",
|
|
76
76
|
"flatted": "^3.4.2",
|
|
77
77
|
"js-yaml": "^4.3.2"
|
|
78
78
|
}
|
package/src/lib/args-parser.js
CHANGED
|
@@ -43,6 +43,7 @@
|
|
|
43
43
|
|
|
44
44
|
const { getDefaultDockerImage } = require('./docker-utils');
|
|
45
45
|
const dockerNetworkOptions = require('./docker-network-options');
|
|
46
|
+
const dockerRecoveryOptions = require('./docker-recovery-options');
|
|
46
47
|
const { parseSequence, isSequence } = require('./sequence-parser');
|
|
47
48
|
const { buildCommandString } = require('./shell-utils');
|
|
48
49
|
const {
|
|
@@ -194,6 +195,8 @@ function parseArgs(args) {
|
|
|
194
195
|
network: null, // First Docker network name (compatibility accessor)
|
|
195
196
|
networks: [], // Ordered Docker network names
|
|
196
197
|
networkAliases: [], // Docker network-scoped aliases
|
|
198
|
+
onKillResume: null, // Resume up to N times when the docker main process is killed (issue #176)
|
|
199
|
+
recoveryCommand: null, // Command run in the same container on such a resume (issue #176)
|
|
197
200
|
endpoint: null, // SSH endpoint (current level, e.g., user@host)
|
|
198
201
|
endpointStack: null, // SSH endpoints for each level (with nulls for non-ssh levels)
|
|
199
202
|
user: false, // Create isolated user
|
|
@@ -410,6 +413,11 @@ function parseOption(args, index, options) {
|
|
|
410
413
|
return networkOption;
|
|
411
414
|
}
|
|
412
415
|
|
|
416
|
+
const recoveryOption = dockerRecoveryOptions.parse(args, index, options);
|
|
417
|
+
if (recoveryOption) {
|
|
418
|
+
return recoveryOption;
|
|
419
|
+
}
|
|
420
|
+
|
|
413
421
|
// --endpoint (for ssh) - supports sequence for stacked isolation
|
|
414
422
|
if (arg === '--endpoint') {
|
|
415
423
|
if (index + 1 < args.length && !args[index + 1].startsWith('-')) {
|
|
@@ -746,6 +754,8 @@ function validateOptions(options) {
|
|
|
746
754
|
validateDockerRuntimeOptionsRequireDocker(options);
|
|
747
755
|
}
|
|
748
756
|
|
|
757
|
+
dockerRecoveryOptions.validate(options);
|
|
758
|
+
|
|
749
759
|
// Session name is only valid with isolation
|
|
750
760
|
if (options.session && !options.isolated) {
|
|
751
761
|
throw new Error('--session option is only valid with --isolated');
|
|
@@ -320,17 +320,27 @@ function buildSuccessfulNonOomCondition() {
|
|
|
320
320
|
* container that is somehow still running afterwards is never removed, gets no
|
|
321
321
|
* `Exit Code:` footer and is never finalized — its record stays `executing`.
|
|
322
322
|
*
|
|
323
|
+
* With `recoverOnKill` (issue #176), a killed main process (exit 137, or
|
|
324
|
+
* `OOMKilled` without a usable exit code, issue #178) is first handed to the
|
|
325
|
+
* recovery entry point. When it resumes the container, this watcher stops
|
|
326
|
+
* here — a new one follows the resumed run — and cleanup, footer and
|
|
327
|
+
* finalization are left to whichever watcher sees the last run end.
|
|
328
|
+
*
|
|
323
329
|
* @param {string} containerName - Docker container name
|
|
324
330
|
* @param {string} policy - One of DOCKER_CONTAINER_CLEANUP_POLICY
|
|
325
331
|
* @param {string|null} logPath - Log file to append to, or null
|
|
326
332
|
* @param {string|null} [executionId] - Execution UUID to finalize, or null
|
|
333
|
+
* @param {{since?: string, recoverOnKill?: boolean}} [watcherOptions] -
|
|
334
|
+
* `since` limits `docker logs` to output after a restart, so a resumed run
|
|
335
|
+
* does not copy the previous run's output into the log again
|
|
327
336
|
* @returns {string} The shell script
|
|
328
337
|
*/
|
|
329
338
|
function buildDetachedDockerCompletionScript(
|
|
330
339
|
containerName,
|
|
331
340
|
policy,
|
|
332
341
|
logPath,
|
|
333
|
-
executionId = null
|
|
342
|
+
executionId = null,
|
|
343
|
+
watcherOptions = {}
|
|
334
344
|
) {
|
|
335
345
|
const quotedName = shellQuote(containerName);
|
|
336
346
|
const parts = [];
|
|
@@ -351,7 +361,10 @@ function buildDetachedDockerCompletionScript(
|
|
|
351
361
|
const remove = `docker rm -f ${quotedName} >> ${quotedLogPath} 2>&1 || true; ${removalNote}`;
|
|
352
362
|
const keep = `${postMortem}; ${buildDockerKeptLogSnippet(containerName, quotedLogPath)}`;
|
|
353
363
|
|
|
354
|
-
|
|
364
|
+
const since = watcherOptions.since
|
|
365
|
+
? ` --since ${shellQuote(watcherOptions.since)}`
|
|
366
|
+
: '';
|
|
367
|
+
parts.push(`docker logs -f${since} ${quotedName} >> ${quotedLogPath} 2>&1`);
|
|
355
368
|
parts.push(buildDockerWaitForExitSnippet(containerName, quotedLogPath));
|
|
356
369
|
parts.push(buildDockerStateSnippet(containerName));
|
|
357
370
|
if (policy === DOCKER_CONTAINER_CLEANUP_POLICY.ALWAYS) {
|
|
@@ -394,9 +407,15 @@ function buildDetachedDockerCompletionScript(
|
|
|
394
407
|
const stillRunning = logPath
|
|
395
408
|
? buildDockerStillRunningNoteSnippet(containerName, shellQuote(logPath))
|
|
396
409
|
: ':';
|
|
410
|
+
let recovery = '';
|
|
411
|
+
if (executionId && watcherOptions.recoverOnKill) {
|
|
412
|
+
// Lazily required: the recovery module starts watchers itself.
|
|
413
|
+
const { buildRecoverySnippet } = require('./execution-recovery');
|
|
414
|
+
recovery = `elif ${buildRecoverySnippet(executionId)}; then :; `;
|
|
415
|
+
}
|
|
397
416
|
parts.push(
|
|
398
417
|
`if [ "$${SHELL_VARS.running}" = true ]; then ${stillRunning}; ` +
|
|
399
|
-
|
|
418
|
+
`${recovery}else ${exited.length ? exited.join('; ') : ':'}; fi`
|
|
400
419
|
);
|
|
401
420
|
|
|
402
421
|
return parts.join('; ');
|
|
@@ -408,13 +427,16 @@ function buildDetachedDockerCompletionScript(
|
|
|
408
427
|
* @param {string} policy - One of DOCKER_CONTAINER_CLEANUP_POLICY
|
|
409
428
|
* @param {string|null} logPath - Log file to append to, or null
|
|
410
429
|
* @param {string|null} [executionId] - Execution UUID to finalize, or null
|
|
430
|
+
* @param {{since?: string, recoverOnKill?: boolean}} [watcherOptions] - See
|
|
431
|
+
* `buildDetachedDockerCompletionScript`
|
|
411
432
|
* @returns {void}
|
|
412
433
|
*/
|
|
413
434
|
function startDetachedDockerCompletionWatcher(
|
|
414
435
|
containerName,
|
|
415
436
|
policy,
|
|
416
437
|
logPath,
|
|
417
|
-
executionId = null
|
|
438
|
+
executionId = null,
|
|
439
|
+
watcherOptions = {}
|
|
418
440
|
) {
|
|
419
441
|
const watcher = spawn(
|
|
420
442
|
'sh',
|
|
@@ -424,7 +446,8 @@ function startDetachedDockerCompletionWatcher(
|
|
|
424
446
|
containerName,
|
|
425
447
|
policy,
|
|
426
448
|
logPath,
|
|
427
|
-
executionId
|
|
449
|
+
executionId,
|
|
450
|
+
watcherOptions
|
|
428
451
|
),
|
|
429
452
|
],
|
|
430
453
|
{
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Parse and validate the launch-time recovery options (issue #176):
|
|
3
|
+
* --on-kill-resume <N> resume up to N times when the main process is
|
|
4
|
+
* killed (exit 137 / OOMKilled)
|
|
5
|
+
* --recovery-command <cmd> command to run in the same container on resume
|
|
6
|
+
* (without it, the original command is re-run)
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
function parseOnKillResumeValue(value) {
|
|
10
|
+
const text = String(value).trim();
|
|
11
|
+
const count = Number(text);
|
|
12
|
+
if (!/^\d+$/.test(text) || !Number.isSafeInteger(count) || count < 1) {
|
|
13
|
+
throw new Error(
|
|
14
|
+
`Invalid --on-kill-resume value: "${value}". Expected a positive integer.`
|
|
15
|
+
);
|
|
16
|
+
}
|
|
17
|
+
return count;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
function parseDockerRecoveryOption(args, index, options) {
|
|
21
|
+
const arg = args[index];
|
|
22
|
+
if (arg === '--on-kill-resume' || arg === '--recovery-command') {
|
|
23
|
+
if (index + 1 >= args.length || args[index + 1].startsWith('-')) {
|
|
24
|
+
const value = arg === '--on-kill-resume' ? 'count' : 'command';
|
|
25
|
+
throw new Error(`Option ${arg} requires a ${value} argument`);
|
|
26
|
+
}
|
|
27
|
+
if (arg === '--on-kill-resume') {
|
|
28
|
+
options.onKillResume = parseOnKillResumeValue(args[index + 1]);
|
|
29
|
+
} else {
|
|
30
|
+
options.recoveryCommand = args[index + 1];
|
|
31
|
+
}
|
|
32
|
+
return 2;
|
|
33
|
+
}
|
|
34
|
+
if (arg.startsWith('--on-kill-resume=')) {
|
|
35
|
+
options.onKillResume = parseOnKillResumeValue(
|
|
36
|
+
arg.slice('--on-kill-resume='.length)
|
|
37
|
+
);
|
|
38
|
+
return 1;
|
|
39
|
+
}
|
|
40
|
+
if (arg.startsWith('--recovery-command=')) {
|
|
41
|
+
options.recoveryCommand = arg.slice('--recovery-command='.length);
|
|
42
|
+
return 1;
|
|
43
|
+
}
|
|
44
|
+
return 0;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* Recovery is driven by the detached docker completion watcher, so it needs a
|
|
49
|
+
* detached session whose only isolation level is docker. A recovery command on
|
|
50
|
+
* its own implies one attempt.
|
|
51
|
+
* @param {object} options - Parsed options
|
|
52
|
+
* @throws {Error} If the recovery options are used where they cannot work
|
|
53
|
+
*/
|
|
54
|
+
function validateDockerRecoveryOptions(options) {
|
|
55
|
+
if (
|
|
56
|
+
options.recoveryCommand !== null &&
|
|
57
|
+
options.recoveryCommand !== undefined
|
|
58
|
+
) {
|
|
59
|
+
if (String(options.recoveryCommand).trim() === '') {
|
|
60
|
+
throw new Error('--recovery-command requires a non-empty command');
|
|
61
|
+
}
|
|
62
|
+
options.onKillResume ??= 1;
|
|
63
|
+
}
|
|
64
|
+
if (!options.onKillResume) {
|
|
65
|
+
return;
|
|
66
|
+
}
|
|
67
|
+
const flag = options.recoveryCommand
|
|
68
|
+
? '--recovery-command'
|
|
69
|
+
: '--on-kill-resume';
|
|
70
|
+
const stack = options.isolatedStack || [options.isolated];
|
|
71
|
+
if (!options.isolated || stack.length !== 1 || stack[0] !== 'docker') {
|
|
72
|
+
throw new Error(
|
|
73
|
+
`${flag} option is only valid with --isolated docker as the only isolation level`
|
|
74
|
+
);
|
|
75
|
+
}
|
|
76
|
+
if (!options.detached) {
|
|
77
|
+
throw new Error(`${flag} option requires --detached`);
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
module.exports = {
|
|
82
|
+
parse: parseDockerRecoveryOption,
|
|
83
|
+
validate: validateDockerRecoveryOptions,
|
|
84
|
+
};
|
|
@@ -0,0 +1,211 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Docker resource limits that must survive a resume (issue #176).
|
|
3
|
+
*
|
|
4
|
+
* `docker commit` captures a container's filesystem and image config, but not
|
|
5
|
+
* its HostConfig. A snapshot-based resume (`$ --resume <id> -- <cmd>`) used to
|
|
6
|
+
* start the replacement container without the memory, CPU and PIDs limits the
|
|
7
|
+
* original one had — limits a supervisor typically applied with
|
|
8
|
+
* `docker update`, so they were never part of the stored launch options.
|
|
9
|
+
*
|
|
10
|
+
* This module reads the limits back out of `docker inspect` and translates the
|
|
11
|
+
* non-default ones into `docker run` flags. The flags are kept in the
|
|
12
|
+
* `--flag=value` form, so the same list is stored in the execution record,
|
|
13
|
+
* shown in `[Isolation]` status lines, and spliced into `docker run` as is.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
const { getDockerCommand } = require('./docker-cleanup');
|
|
17
|
+
|
|
18
|
+
/** Docker's default `/dev/shm` size; only a different size is re-applied. */
|
|
19
|
+
const DEFAULT_SHM_SIZE = 64 * 1024 * 1024;
|
|
20
|
+
|
|
21
|
+
const KIB = 1024;
|
|
22
|
+
const MIB = KIB * 1024;
|
|
23
|
+
const GIB = MIB * 1024;
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* Format a byte count with the largest binary unit `docker run` accepts that
|
|
27
|
+
* represents it exactly (`268435456` → `256m`).
|
|
28
|
+
* @param {number} bytes - Byte count
|
|
29
|
+
* @returns {string} Docker memory size
|
|
30
|
+
*/
|
|
31
|
+
function formatDockerBytes(bytes) {
|
|
32
|
+
for (const [unit, size] of [
|
|
33
|
+
['g', GIB],
|
|
34
|
+
['m', MIB],
|
|
35
|
+
['k', KIB],
|
|
36
|
+
]) {
|
|
37
|
+
if (bytes >= size && bytes % size === 0) {
|
|
38
|
+
return `${bytes / size}${unit}`;
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
return String(bytes);
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Format `NanoCpus` as the decimal `--cpus` value (`500000000` → `0.5`).
|
|
46
|
+
* @param {number} nanoCpus - CPU quota in units of 1e-9 CPUs
|
|
47
|
+
* @returns {string} Docker CPU count
|
|
48
|
+
*/
|
|
49
|
+
function formatDockerCpus(nanoCpus) {
|
|
50
|
+
return String(Number((nanoCpus / 1e9).toFixed(9)));
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
function positive(value) {
|
|
54
|
+
const number = Number(value);
|
|
55
|
+
return Number.isFinite(number) && number > 0 ? number : null;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
function nonEmpty(value) {
|
|
59
|
+
return typeof value === 'string' && value.trim() !== '' ? value.trim() : null;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* Translate an inspected `HostConfig` into `docker run` flags.
|
|
64
|
+
*
|
|
65
|
+
* Only limits that differ from Docker's defaults are emitted, and only in
|
|
66
|
+
* combinations `docker run` accepts: `--memory-swap` needs `--memory`, and
|
|
67
|
+
* `--cpus` cannot be combined with `--cpu-quota`/`--cpu-period`.
|
|
68
|
+
*
|
|
69
|
+
* @param {?object} hostConfig - `docker inspect` `.HostConfig`
|
|
70
|
+
* @returns {string[]} Flags in `--flag=value` form
|
|
71
|
+
*/
|
|
72
|
+
function parseDockerResourceLimits(hostConfig) {
|
|
73
|
+
if (!hostConfig || typeof hostConfig !== 'object') {
|
|
74
|
+
return [];
|
|
75
|
+
}
|
|
76
|
+
const h = hostConfig;
|
|
77
|
+
const limits = [];
|
|
78
|
+
|
|
79
|
+
const memory = positive(h.Memory);
|
|
80
|
+
if (memory) {
|
|
81
|
+
limits.push(`--memory=${formatDockerBytes(memory)}`);
|
|
82
|
+
const swap = Number(h.MemorySwap);
|
|
83
|
+
if (swap === -1) {
|
|
84
|
+
limits.push('--memory-swap=-1');
|
|
85
|
+
} else if (positive(swap)) {
|
|
86
|
+
limits.push(`--memory-swap=${formatDockerBytes(swap)}`);
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
const reservation = positive(h.MemoryReservation);
|
|
90
|
+
if (reservation) {
|
|
91
|
+
limits.push(`--memory-reservation=${formatDockerBytes(reservation)}`);
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
const nanoCpus = positive(h.NanoCpus);
|
|
95
|
+
if (nanoCpus) {
|
|
96
|
+
limits.push(`--cpus=${formatDockerCpus(nanoCpus)}`);
|
|
97
|
+
} else {
|
|
98
|
+
const quota = positive(h.CpuQuota);
|
|
99
|
+
if (quota) {
|
|
100
|
+
limits.push(`--cpu-quota=${quota}`);
|
|
101
|
+
}
|
|
102
|
+
const period = positive(h.CpuPeriod);
|
|
103
|
+
if (period) {
|
|
104
|
+
limits.push(`--cpu-period=${period}`);
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
const shares = positive(h.CpuShares);
|
|
108
|
+
if (shares) {
|
|
109
|
+
limits.push(`--cpu-shares=${shares}`);
|
|
110
|
+
}
|
|
111
|
+
const cpusetCpus = nonEmpty(h.CpusetCpus);
|
|
112
|
+
if (cpusetCpus) {
|
|
113
|
+
limits.push(`--cpuset-cpus=${cpusetCpus}`);
|
|
114
|
+
}
|
|
115
|
+
const cpusetMems = nonEmpty(h.CpusetMems);
|
|
116
|
+
if (cpusetMems) {
|
|
117
|
+
limits.push(`--cpuset-mems=${cpusetMems}`);
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
const pids = positive(h.PidsLimit);
|
|
121
|
+
if (pids) {
|
|
122
|
+
limits.push(`--pids-limit=${pids}`);
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
const shm = positive(h.ShmSize);
|
|
126
|
+
if (shm && shm !== DEFAULT_SHM_SIZE) {
|
|
127
|
+
limits.push(`--shm-size=${formatDockerBytes(shm)}`);
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
if (h.StorageOpt && typeof h.StorageOpt === 'object') {
|
|
131
|
+
for (const key of Object.keys(h.StorageOpt).sort()) {
|
|
132
|
+
const value = nonEmpty(String(h.StorageOpt[key] ?? ''));
|
|
133
|
+
if (value) {
|
|
134
|
+
limits.push(`--storage-opt=${key}=${value}`);
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
for (const ulimit of Array.isArray(h.Ulimits) ? h.Ulimits : []) {
|
|
140
|
+
if (ulimit && nonEmpty(ulimit.Name)) {
|
|
141
|
+
limits.push(`--ulimit=${ulimit.Name}=${ulimit.Soft}:${ulimit.Hard}`);
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
return limits;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/**
|
|
149
|
+
* Read a container's resource limits with `docker inspect`.
|
|
150
|
+
* @param {string} containerName - Container to inspect
|
|
151
|
+
* @param {Function} runner - `(command, args) => {success, stdout}`
|
|
152
|
+
* @returns {?string[]} Flags, or null when the container could not be read
|
|
153
|
+
*/
|
|
154
|
+
function readDockerResourceLimits(containerName, runner) {
|
|
155
|
+
let result;
|
|
156
|
+
try {
|
|
157
|
+
result = runner(getDockerCommand(), [
|
|
158
|
+
'inspect',
|
|
159
|
+
'--format',
|
|
160
|
+
'{{json .HostConfig}}',
|
|
161
|
+
containerName,
|
|
162
|
+
]);
|
|
163
|
+
} catch {
|
|
164
|
+
return null;
|
|
165
|
+
}
|
|
166
|
+
if (!result || !result.success) {
|
|
167
|
+
return null;
|
|
168
|
+
}
|
|
169
|
+
try {
|
|
170
|
+
return parseDockerResourceLimits(JSON.parse(String(result.stdout).trim()));
|
|
171
|
+
} catch {
|
|
172
|
+
return null;
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
/**
|
|
177
|
+
* Normalize a stored `resourceLimits` value to a list of flags.
|
|
178
|
+
* @param {*} value - Value from an execution record or options object
|
|
179
|
+
* @returns {string[]} Flags
|
|
180
|
+
*/
|
|
181
|
+
function normalizeResourceLimits(value) {
|
|
182
|
+
if (Array.isArray(value)) {
|
|
183
|
+
return value.map(String).filter((flag) => flag.startsWith('--'));
|
|
184
|
+
}
|
|
185
|
+
if (typeof value === 'string' && value.trim() !== '') {
|
|
186
|
+
return value.split(/\s+/).filter((flag) => flag.startsWith('--'));
|
|
187
|
+
}
|
|
188
|
+
return [];
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
/**
|
|
192
|
+
* `[Isolation]` status line for resource limits, or null when there are none.
|
|
193
|
+
* @param {*} value - Stored `resourceLimits`
|
|
194
|
+
* @returns {?string} Status line
|
|
195
|
+
*/
|
|
196
|
+
function buildResourceLimitsStatusLine(value) {
|
|
197
|
+
const limits = normalizeResourceLimits(value);
|
|
198
|
+
return limits.length > 0
|
|
199
|
+
? `[Isolation] Resource limits: ${limits.join(' ')}`
|
|
200
|
+
: null;
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
module.exports = {
|
|
204
|
+
DEFAULT_SHM_SIZE,
|
|
205
|
+
buildResourceLimitsStatusLine,
|
|
206
|
+
formatDockerBytes,
|
|
207
|
+
formatDockerCpus,
|
|
208
|
+
normalizeResourceLimits,
|
|
209
|
+
parseDockerResourceLimits,
|
|
210
|
+
readDockerResourceLimits,
|
|
211
|
+
};
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Docker runtime options shared by `docker run`, `[Isolation]` status lines
|
|
3
|
+
* and execution-record metadata: privileged mode, environment, volumes,
|
|
4
|
+
* mounts, networks (issue #150) and resource limits (issue #176).
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
const { getDockerNetworks } = require('./docker-network-lifecycle');
|
|
8
|
+
const {
|
|
9
|
+
buildResourceLimitsStatusLine,
|
|
10
|
+
normalizeResourceLimits,
|
|
11
|
+
} = require('./docker-resource-limits');
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* Number of kill recoveries requested at launch; a recovery command on its own
|
|
15
|
+
* implies one (issue #176).
|
|
16
|
+
* @param {object} options - Options (onKillResume, recoveryCommand)
|
|
17
|
+
* @returns {?number} Attempt limit, or null when recovery is off
|
|
18
|
+
*/
|
|
19
|
+
function getOnKillResume(options = {}) {
|
|
20
|
+
const count = Number(options.onKillResume) || 0;
|
|
21
|
+
if (count > 0) {
|
|
22
|
+
return count;
|
|
23
|
+
}
|
|
24
|
+
return options.recoveryCommand ? 1 : null;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* Build the docker run runtime argument list contributed by configurable
|
|
29
|
+
* container options: privileged mode, environment variables, volumes/bind
|
|
30
|
+
* mounts, --mount specs, network configuration and resource limits. Returned
|
|
31
|
+
* in a stable order so they can be spliced into the `docker run` argv before
|
|
32
|
+
* the image name.
|
|
33
|
+
* @param {object} options - Options (privileged, env, volumes, mounts, network, networkAliases, resourceLimits)
|
|
34
|
+
* @returns {string[]} Docker CLI arguments
|
|
35
|
+
*/
|
|
36
|
+
function buildDockerRuntimeArgs(options = {}) {
|
|
37
|
+
const args = [];
|
|
38
|
+
if (options.privileged) {
|
|
39
|
+
args.push('--privileged');
|
|
40
|
+
}
|
|
41
|
+
for (const envVar of options.env || []) {
|
|
42
|
+
args.push('-e', envVar);
|
|
43
|
+
}
|
|
44
|
+
for (const volume of options.volumes || []) {
|
|
45
|
+
args.push('-v', volume);
|
|
46
|
+
}
|
|
47
|
+
for (const mount of options.mounts || []) {
|
|
48
|
+
args.push('--mount', mount);
|
|
49
|
+
}
|
|
50
|
+
const [firstNetwork] = getDockerNetworks(options);
|
|
51
|
+
if (firstNetwork) {
|
|
52
|
+
args.push('--network', firstNetwork);
|
|
53
|
+
}
|
|
54
|
+
for (const alias of options.networkAliases || []) {
|
|
55
|
+
args.push('--network-alias', alias);
|
|
56
|
+
}
|
|
57
|
+
args.push(...normalizeResourceLimits(options.resourceLimits));
|
|
58
|
+
return args;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* Build the human-readable `[Isolation]` status lines for docker runtime
|
|
63
|
+
* options (volumes, mounts, env, privileged, networks, resource limits and
|
|
64
|
+
* the launch-time recovery command). Empty collections and a falsy privileged
|
|
65
|
+
* flag contribute no lines.
|
|
66
|
+
* @param {object} options - Options (volumes, mounts, env, privileged, network, networks, networkAliases, resourceLimits, onKillResume, recoveryCommand)
|
|
67
|
+
* @returns {string[]} Status lines for the start block / log header
|
|
68
|
+
*/
|
|
69
|
+
function buildDockerRuntimeStatusLines(options = {}) {
|
|
70
|
+
const lines = [];
|
|
71
|
+
if (options.volumes && options.volumes.length > 0) {
|
|
72
|
+
lines.push(`[Isolation] Volumes: ${options.volumes.join(', ')}`);
|
|
73
|
+
}
|
|
74
|
+
if (options.mounts && options.mounts.length > 0) {
|
|
75
|
+
lines.push(`[Isolation] Mounts: ${options.mounts.join(', ')}`);
|
|
76
|
+
}
|
|
77
|
+
if (options.env && options.env.length > 0) {
|
|
78
|
+
lines.push(`[Isolation] Env: ${options.env.join(', ')}`);
|
|
79
|
+
}
|
|
80
|
+
if (options.privileged) {
|
|
81
|
+
lines.push(`[Isolation] Privileged: true`);
|
|
82
|
+
}
|
|
83
|
+
const networks = getDockerNetworks(options);
|
|
84
|
+
if (networks.length > 0) {
|
|
85
|
+
lines.push(`[Isolation] Network: ${networks[0]}`);
|
|
86
|
+
}
|
|
87
|
+
if (networks.length > 1) {
|
|
88
|
+
lines.push(`[Isolation] Networks: ${networks.join(', ')}`);
|
|
89
|
+
}
|
|
90
|
+
if (options.networkAliases && options.networkAliases.length > 0) {
|
|
91
|
+
lines.push(
|
|
92
|
+
`[Isolation] Network aliases: ${options.networkAliases.join(', ')}`
|
|
93
|
+
);
|
|
94
|
+
}
|
|
95
|
+
const limitsLine = buildResourceLimitsStatusLine(options.resourceLimits);
|
|
96
|
+
if (limitsLine) {
|
|
97
|
+
lines.push(limitsLine);
|
|
98
|
+
}
|
|
99
|
+
const onKillResume = getOnKillResume(options);
|
|
100
|
+
if (onKillResume) {
|
|
101
|
+
const what = options.recoveryCommand || 'the original command';
|
|
102
|
+
lines.push(
|
|
103
|
+
`[Isolation] On kill: resume up to ${onKillResume} time(s) with ${what}`
|
|
104
|
+
);
|
|
105
|
+
}
|
|
106
|
+
return lines;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/**
|
|
110
|
+
* Build the execution-record metadata for docker runtime options, normalizing
|
|
111
|
+
* empty collections and a falsy privileged flag to `null`.
|
|
112
|
+
* @param {object} options - Options (volumes, mounts, env, privileged, network, networks, networkAliases, resourceLimits, onKillResume, recoveryCommand)
|
|
113
|
+
* @returns {object} Record metadata
|
|
114
|
+
*/
|
|
115
|
+
function buildDockerRuntimeMetadata(options = {}) {
|
|
116
|
+
const networks = getDockerNetworks(options);
|
|
117
|
+
const resourceLimits = normalizeResourceLimits(options.resourceLimits);
|
|
118
|
+
return {
|
|
119
|
+
volumes:
|
|
120
|
+
options.volumes && options.volumes.length > 0 ? options.volumes : null,
|
|
121
|
+
mounts: options.mounts && options.mounts.length > 0 ? options.mounts : null,
|
|
122
|
+
env: options.env && options.env.length > 0 ? options.env : null,
|
|
123
|
+
privileged: options.privileged || null,
|
|
124
|
+
network: networks[0] || null,
|
|
125
|
+
networks: networks.length > 0 ? networks : null,
|
|
126
|
+
networkAliases:
|
|
127
|
+
options.networkAliases && options.networkAliases.length > 0
|
|
128
|
+
? options.networkAliases
|
|
129
|
+
: null,
|
|
130
|
+
resourceLimits: resourceLimits.length > 0 ? resourceLimits : null,
|
|
131
|
+
onKillResume: getOnKillResume(options),
|
|
132
|
+
recoveryCommand: options.recoveryCommand || null,
|
|
133
|
+
};
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
module.exports = {
|
|
137
|
+
buildDockerRuntimeArgs,
|
|
138
|
+
buildDockerRuntimeStatusLines,
|
|
139
|
+
buildDockerRuntimeMetadata,
|
|
140
|
+
getOnKillResume,
|
|
141
|
+
};
|
|
@@ -268,6 +268,40 @@ function formatControlResultAsLinksNotation(result) {
|
|
|
268
268
|
return lines.join('\n');
|
|
269
269
|
}
|
|
270
270
|
|
|
271
|
+
/**
|
|
272
|
+
* Record that a stop was requested, for sessions with launch-time kill
|
|
273
|
+
* recovery (issue #176).
|
|
274
|
+
* @param {object} store - Execution store
|
|
275
|
+
* @param {object} record - Execution record
|
|
276
|
+
* @returns {string|undefined} The previous marker, to restore on failure
|
|
277
|
+
*/
|
|
278
|
+
function markStopRequested(store, record) {
|
|
279
|
+
const options = record.options || {};
|
|
280
|
+
if (!options.onKillResume) {
|
|
281
|
+
return undefined;
|
|
282
|
+
}
|
|
283
|
+
const previous = options.stopRequestedAt;
|
|
284
|
+
record.options = { ...options, stopRequestedAt: new Date().toISOString() };
|
|
285
|
+
try {
|
|
286
|
+
store.save(record);
|
|
287
|
+
} catch {
|
|
288
|
+
// Best effort: without the marker the session may be recovered once more.
|
|
289
|
+
}
|
|
290
|
+
return previous;
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
function restoreStopRequested(store, record, previous) {
|
|
294
|
+
if (!record.options || !record.options.onKillResume) {
|
|
295
|
+
return;
|
|
296
|
+
}
|
|
297
|
+
record.options = { ...record.options, stopRequestedAt: previous };
|
|
298
|
+
try {
|
|
299
|
+
store.save(record);
|
|
300
|
+
} catch {
|
|
301
|
+
// Best effort, see markStopRequested.
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
|
|
271
305
|
function controlExecution(store, identifier, action, runner = runCommand) {
|
|
272
306
|
if (!store) {
|
|
273
307
|
return { success: false, error: 'Execution tracking is disabled.' };
|
|
@@ -286,8 +320,13 @@ function controlExecution(store, identifier, action, runner = runCommand) {
|
|
|
286
320
|
return { success: false, error: control.error };
|
|
287
321
|
}
|
|
288
322
|
|
|
323
|
+
// A deliberate stop must not be mistaken for a kill that launch-time
|
|
324
|
+
// recovery should undo (issue #176): `docker stop` escalates to SIGKILL,
|
|
325
|
+
// which exits 137 just like the OOM killer.
|
|
326
|
+
const previousStopRequestedAt = markStopRequested(store, record);
|
|
289
327
|
const result = runner(control.command, control.args);
|
|
290
328
|
if (!result.success) {
|
|
329
|
+
restoreStopRequested(store, record, previousStopRequestedAt);
|
|
291
330
|
const detail =
|
|
292
331
|
result.stderr || result.error || `exit code ${result.status}`;
|
|
293
332
|
return {
|