start-command 0.38.0 → 0.39.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +20 -0
- package/README.md +4 -2
- package/bunfig.toml +4 -0
- package/eslint.config.mjs +1 -0
- package/package.json +4 -4
- package/src/lib/args-parser.js +9 -0
- package/src/lib/cgroup-memory.js +95 -23
- package/src/lib/command-builder.js +4 -0
- package/src/lib/container-removal.js +4 -5
- package/src/lib/detached-finalize.js +1 -0
- package/src/lib/docker-init-option.js +65 -0
- package/src/lib/docker-inspect-batch.js +176 -0
- package/src/lib/docker-resource-options.js +21 -2
- package/src/lib/docker-runtime-args.js +12 -3
- package/src/lib/execution-control.js +8 -6
- package/src/lib/execution-recovery.js +4 -1
- package/src/lib/execution-resume.js +4 -1
- package/src/lib/execution-store.js +19 -1
- package/src/lib/exit-evidence.js +139 -7
- package/src/lib/output-blocks.js +11 -0
- package/src/lib/resume-resources.js +2 -2
- package/src/lib/status-formatter.js +34 -130
- package/src/lib/status-probe.js +149 -0
- package/src/lib/store-decode-cache.js +141 -0
- package/src/lib/usage.js +3 -0
- package/test/cargo-lock-version.mjs +121 -0
- package/test/check-rust-coverage.mjs +71 -0
- package/test/check-web-archive.mjs +154 -0
- package/test/ci-workflow-invariants.js +204 -0
- package/test/command-builder-docker-cleanup.js +24 -0
- package/test/command-exit-codes.mjs +91 -0
- package/test/docker-runtime-options.js +2 -0
- package/test/docker-snapshot-safety.js +39 -0
- package/test/execution-resume.js +2 -0
- package/test/fixtures/fake-npm.sh +64 -0
- package/test/fixtures/lychee-report.md +39 -0
- package/test/javascript-first-ci.js +69 -0
- package/test/links-workflow.mjs +127 -0
- package/test/lint-gate.mjs +56 -0
- package/test/npm-publish.mjs +181 -0
- package/test/recheck-broken-links.mjs +452 -0
- package/test/regression-176.js +2 -0
- package/test/regression-187.js +2 -0
- package/test/regression-191.js +4 -0
- package/test/regression-193.js +2 -0
- package/test/regression-198.js +2 -0
- package/test/regression-207.js +8 -2
- package/test/regression-224.js +2 -0
- package/test/regression-236.js +250 -0
- package/test/regression-238.js +129 -0
- package/test/regression-241.js +254 -0
- package/test/regression-242.js +241 -0
- package/test/regression-243.js +493 -0
- package/test/regression-244.js +471 -0
- package/test/release-push-retry.mjs +172 -0
- package/test/run-js-tests.mjs +15 -3
- package/test/setup/private-snapshot-lock.js +17 -0
- package/test/shared-golden.js +29 -2
- package/test/transient-links.mjs +139 -0
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,25 @@
|
|
|
1
1
|
# start-command
|
|
2
2
|
|
|
3
|
+
## 0.39.0
|
|
4
|
+
|
|
5
|
+
### Minor Changes
|
|
6
|
+
|
|
7
|
+
- CI/CD no longer reports false results (#234). A rejected `npm publish` or any failing release command now fails the release instead of printing success, and the published version is confirmed on the registry: npm gets up to 20 minutes to show it, and a "cannot publish over" conflict is read as "already landed" rather than retried. A release push that loses the race for `main` is rebased and retried. Releases wait for every check their stage requires. A failing Rust test is reported as a failed test rather than as low coverage. `bun test -t <pattern>` keeps the 30s default timeout. Snapshot-resume tests in both languages use a private snapshot lock. The link checker fails on every broken live link, re-checks transient 429/5xx answers, persists its cache and uploads its report. ESLint warnings fail CI and the pre-commit hook. The changeset PR is opened with `RELEASE_PR_TOKEN` so its checks run. Step outputs reach `run:` blocks through `env`, tool versions are pinned, path-filtered workflows re-run when their scripts change, and the npm-publish tests no longer reach the real npm on macOS.
|
|
8
|
+
|
|
9
|
+
Support Docker `--memory-reservation` with sizes, daemon percentages and percentage ranges. Validate the soft cap against the effective hard memory limit, record it in status, and preserve or override it across resumes and recovery.
|
|
10
|
+
|
|
11
|
+
Reduce Docker cgroup sampler polling with a configurable five-second default, one namespace lookup per sampler, and local cgroup liveness checks. Set `START_COMMAND_CGROUP_SAMPLE_INTERVAL_SECONDS` to a positive whole number of seconds to override the default for attached and detached runs.
|
|
12
|
+
|
|
13
|
+
Main pushes publish again (#240). Since #234 the Release job was skipped on every push to `main`: at job level the wildcard `needs.*.result` list also saw the deliberately skipped changeset check behind `test`/`coverage`. Release now compares each needed job by name, and the JavaScript Stage prints every term of the release condition and fails a main push whose release was skipped although all of its needs succeeded.
|
|
14
|
+
|
|
15
|
+
Attribute a Docker daemon restart without the host journal (#241). With a mounted `docker.sock` (Docker-outside-of-Docker) or a remote daemon, an exit-137 task is now reported as `killed (docker daemon restart)` when another container finished within the 15 s shutdown window with exit 137/143, or was started again after it, and was not OOM-killed. `exitEvidence.daemonRestartSource` records whether the host journal or the Docker API supported the attribution, and `--status` collects the evidence itself when the detached watcher died with the daemon before it could log any.
|
|
16
|
+
|
|
17
|
+
Sample Docker cgroup counters through one long-lived `docker exec -i` per run when the cgroup is only readable through the daemon (a mounted `docker.sock`). The host paces it with one tick per interval on stdin, so the container runs only shell builtins and the stream ends with the sampler; the per-round exec stays as a fallback. Docker is asked for `{{.State.Running}}` only after a round without a sample, and `START_COMMAND_CGROUP_SAMPLE_INTERVAL_SECONDS=off` disables sampling.
|
|
18
|
+
|
|
19
|
+
`$ --status` and `$ --list` cost far less with many records (#243). The store is decoded once per content within a process, and a JSON copy of the decoded records next to `executions.lino` (tagged with the store's SHA-256) lets the next process skip the decode entirely. One `docker inspect --type container` now answers every docker probe of a query instead of one or two processes per record, and records whose container is already marked removed are no longer probed. Output is unchanged; probes the batch cannot answer (an old or stubbed `docker`) run one by one as before.
|
|
20
|
+
|
|
21
|
+
`--init` / `--no-init` run Docker's bundled tini (`docker run --init`) as PID 1 for docker isolation, so orphaned processes of a long-running task are reaped instead of piling up as zombies (#244). The default stays off; `START_COMMAND_DOCKER_INIT=1` turns it on for every docker launch on a host, and `--no-init` overrides that. The choice is stored with the record, shown as an `init` row in the start block and in `--status`, and carried into `--resume` snapshots and stacked launches; records from before keep running without it. Exit codes and OOM evidence are unchanged, but `--stop` (`docker stop`) now ends a command without a SIGTERM handler with 143 at once instead of 137 after the grace period.
|
|
22
|
+
|
|
3
23
|
## 0.38.0
|
|
4
24
|
|
|
5
25
|
### Minor Changes
|
package/README.md
CHANGED
|
@@ -73,9 +73,11 @@ Released into the public domain under the [Unlicense](../LICENSE).
|
|
|
73
73
|
|
|
74
74
|
### Docker resource and recovery controls
|
|
75
75
|
|
|
76
|
-
Launch with `--memory 64m`, `--memory-swap 64m` and `--cpus 1`, or use daemon
|
|
76
|
+
Launch with `--memory 64m`, `--memory-swap 64m`, `--memory-reservation 32m` and `--cpus 1`, or use daemon
|
|
77
77
|
percentages and uniform ranges such as `--memory '70%-80%'`. Memory-swap defaults
|
|
78
|
-
to memory.
|
|
78
|
+
to memory. Reservation is a soft cap and must be below the hard memory cap when
|
|
79
|
+
one is set. It accepts the same sizes and percentages, is shown in `--status`,
|
|
80
|
+
and survives resumes. `--cpu-penalty` enables delayed caps and reports state through status.
|
|
79
81
|
Manual `--resume <id> --memory 128m` updates before starting;
|
|
80
82
|
`--on-kill-resume-memory` changes memory only after a fresh qualifying OOM.
|
|
81
83
|
See the [full resource and recovery guide](../README.md#docker-launch-limits-recovery-limits-and-cpu-penalty)
|
package/bunfig.toml
CHANGED
|
@@ -1,3 +1,7 @@
|
|
|
1
1
|
# Bun configuration
|
|
2
2
|
# Tests use repository naming without conventional suffixes, so package scripts pass
|
|
3
3
|
# the test files explicitly instead of relying on Bun's default discovery.
|
|
4
|
+
|
|
5
|
+
[test]
|
|
6
|
+
# Every test process gets a private Docker snapshot lock (issue #234).
|
|
7
|
+
preload = ["./test/setup/private-snapshot-lock.js"]
|
package/eslint.config.mjs
CHANGED
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "start-command",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.39.0",
|
|
4
4
|
"description": "Gamification of coding, execute any command with ability to auto-report issues on GitHub",
|
|
5
5
|
"main": "src/bin/cli.js",
|
|
6
6
|
"exports": {
|
|
@@ -12,12 +12,12 @@
|
|
|
12
12
|
},
|
|
13
13
|
"scripts": {
|
|
14
14
|
"test": "bun ../scripts/run-js-tests.mjs",
|
|
15
|
-
"lint": "eslint .",
|
|
15
|
+
"lint": "eslint . --max-warnings 0",
|
|
16
16
|
"lint:fix": "eslint . --fix",
|
|
17
17
|
"format": "prettier --write .",
|
|
18
18
|
"format:check": "prettier --check .",
|
|
19
19
|
"check:file-size": "bun ../scripts/check-file-size.mjs",
|
|
20
|
-
"lint:scripts": "cd .. && eslint .",
|
|
20
|
+
"lint:scripts": "cd .. && eslint . --max-warnings 0",
|
|
21
21
|
"format:check:scripts": "cd .. && prettier --check \"scripts/**/*.{js,mjs,cjs}\"",
|
|
22
22
|
"check": "bun run lint && bun run lint:scripts && bun run format:check && bun run format:check:scripts && bun run check:file-size",
|
|
23
23
|
"prepare": "cd .. && husky .husky || true",
|
|
@@ -58,7 +58,7 @@
|
|
|
58
58
|
},
|
|
59
59
|
"lint-staged": {
|
|
60
60
|
"*.{js,mjs,cjs}": [
|
|
61
|
-
"eslint --fix --max-warnings
|
|
61
|
+
"eslint --fix --max-warnings 0 --no-warn-ignored",
|
|
62
62
|
"prettier --write",
|
|
63
63
|
"prettier --check"
|
|
64
64
|
],
|
package/src/lib/args-parser.js
CHANGED
|
@@ -14,6 +14,7 @@
|
|
|
14
14
|
* --mount <mount-spec> Docker --mount spec (repeatable, docker only)
|
|
15
15
|
* --env, -e <KEY=VALUE> Environment variable for docker container (repeatable, docker only)
|
|
16
16
|
* --privileged Run docker container in privileged mode (docker only)
|
|
17
|
+
* --init, --no-init Run an init process (tini) as PID 1 to reap orphans (docker only; default START_COMMAND_DOCKER_INIT)
|
|
17
18
|
* --network <name> Connect docker container to a named network (repeatable, docker only)
|
|
18
19
|
* --network-alias <alias> Add an alias to the first network (repeatable, docker only)
|
|
19
20
|
* --endpoint <endpoint> SSH endpoint (required for ssh isolation, e.g., user@host)
|
|
@@ -47,6 +48,7 @@ const dockerNetworkOptions = require('./docker-network-options');
|
|
|
47
48
|
const dockerRecoveryOptions = require('./docker-recovery-options');
|
|
48
49
|
const cpuPenalty = require('./cpu-penalty');
|
|
49
50
|
const dockerResourceOptions = require('./docker-resource-options');
|
|
51
|
+
const dockerInitOption = require('./docker-init-option');
|
|
50
52
|
const snapshotResumeOptions = require('./snapshot-resume-options');
|
|
51
53
|
const launchIdentityOptions = require('./launch-identity-options');
|
|
52
54
|
const { parseSequence, isSequence } = require('./sequence-parser');
|
|
@@ -199,6 +201,7 @@ function parseArgs(args) {
|
|
|
199
201
|
labels: [], // Caller attribution, applied to every Docker level
|
|
200
202
|
help: false,
|
|
201
203
|
privileged: false, // Run docker container in privileged mode
|
|
204
|
+
init: null, // docker run --init; null until resolved from START_COMMAND_DOCKER_INIT (issue #244)
|
|
202
205
|
network: null, // First Docker network name (compatibility accessor)
|
|
203
206
|
networks: [], // Ordered Docker network names
|
|
204
207
|
networkAliases: [], // Docker network-scoped aliases
|
|
@@ -430,6 +433,11 @@ function parseOption(args, index, options) {
|
|
|
430
433
|
return 1;
|
|
431
434
|
}
|
|
432
435
|
|
|
436
|
+
// --init / --no-init (for docker, issue #244)
|
|
437
|
+
if (dockerInitOption.parse(args, index, options)) {
|
|
438
|
+
return 1;
|
|
439
|
+
}
|
|
440
|
+
|
|
433
441
|
const networkOption = dockerNetworkOptions.parse(args, index, options);
|
|
434
442
|
if (networkOption) {
|
|
435
443
|
return networkOption;
|
|
@@ -803,6 +811,7 @@ function validateOptions(options) {
|
|
|
803
811
|
|
|
804
812
|
dockerRecoveryOptions.validate(options);
|
|
805
813
|
dockerResourceOptions.validate(options);
|
|
814
|
+
dockerInitOption.validate(options);
|
|
806
815
|
cpuPenalty.validate(options);
|
|
807
816
|
|
|
808
817
|
// Session name is only valid with isolation
|
package/src/lib/cgroup-memory.js
CHANGED
|
@@ -27,8 +27,8 @@
|
|
|
27
27
|
|
|
28
28
|
const { shellQuote } = require('./isolation-log-utils');
|
|
29
29
|
|
|
30
|
-
/**
|
|
31
|
-
const CGROUP_SAMPLE_INTERVAL_SECONDS =
|
|
30
|
+
/** Default seconds between samples; the watcher environment may override it. */
|
|
31
|
+
const CGROUP_SAMPLE_INTERVAL_SECONDS = 5;
|
|
32
32
|
|
|
33
33
|
/** Shell variables used by the sampler snippets. */
|
|
34
34
|
const CGROUP_SHELL_VARS = {
|
|
@@ -74,10 +74,50 @@ function buildCgroupFunctionsSnippet() {
|
|
|
74
74
|
].join('; ');
|
|
75
75
|
}
|
|
76
76
|
|
|
77
|
+
/**
|
|
78
|
+
* In-container loop of the long-lived sampler exec (issue #242): `$1` is the
|
|
79
|
+
* cgroup directory. The host paces it with one newline per interval on stdin
|
|
80
|
+
* (`docker exec -i`), so it runs only shell builtins: the task's own CPU
|
|
81
|
+
* limit pays for no `cat`, `sed` or `sleep` processes. Stdin reaches EOF when
|
|
82
|
+
* the exec's client is gone, so the loop never outlives the sampler; dockerd
|
|
83
|
+
* would accept its output even then. Exit 3: unreadable counters.
|
|
84
|
+
* @returns {string} Shell script for `sh -c`
|
|
85
|
+
*/
|
|
86
|
+
function buildCgroupStreamScript() {
|
|
87
|
+
return [
|
|
88
|
+
'while read -r __scx_tick; do __scx_oom=; __scx_kill=; __scx_max=-; __scx_peak=-',
|
|
89
|
+
'[ -r "$1/memory.events" ] || exit 3',
|
|
90
|
+
'while read -r __scx_key __scx_value; do case "$__scx_key" in ' +
|
|
91
|
+
'oom) __scx_oom=$__scx_value;; oom_kill) __scx_kill=$__scx_value;; esac; ' +
|
|
92
|
+
'done < "$1/memory.events"',
|
|
93
|
+
'case "$__scx_oom:$__scx_kill" in *[!0-9:]*|:*|*:) exit 3;; esac',
|
|
94
|
+
'[ ! -r "$1/memory.max" ] || read -r __scx_max < "$1/memory.max"',
|
|
95
|
+
'[ ! -r "$1/memory.peak" ] || read -r __scx_peak < "$1/memory.peak"',
|
|
96
|
+
'printf \'%s %s %s %s %s\\n\' "${__scx_max:--}" "${__scx_peak:--}" ' +
|
|
97
|
+
'"$__scx_oom" "$__scx_kill" "$1" || exit 0; done',
|
|
98
|
+
].join('; ');
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/** `$0` of the long-lived exec, visible in `ps` inside the task container. */
|
|
102
|
+
const CGROUP_STREAM_NAME = 'start-command-cgroup-stream';
|
|
103
|
+
|
|
104
|
+
/** Interval value that turns the sampler off (issue #242). */
|
|
105
|
+
const CGROUP_SAMPLING_OFF_REASON =
|
|
106
|
+
'sampling disabled by START_COMMAND_CGROUP_SAMPLE_INTERVAL_SECONDS=off';
|
|
107
|
+
|
|
77
108
|
/**
|
|
78
109
|
* Shell fragment starting the background sampler. It writes each sample to a
|
|
79
110
|
* temp file with `mv`, so the watcher never reads a half-written line, and
|
|
80
|
-
* ends by itself once the
|
|
111
|
+
* ends by itself once the container is gone.
|
|
112
|
+
*
|
|
113
|
+
* Where the cgroup is readable locally it reads it directly. Otherwise, with
|
|
114
|
+
* a private cgroup namespace (a mounted docker.sock), one long-lived
|
|
115
|
+
* `docker exec -i` answers one host tick per interval (issue #242); a
|
|
116
|
+
* successful read proves the container runs, so Docker is asked for liveness
|
|
117
|
+
* only after a failed round or when the stream ends. Without a usable stream it falls back
|
|
118
|
+
* to one `docker exec` per round; without `sh` in the image (exit 126/127) or
|
|
119
|
+
* without any readable cgroup it records the reason and stops.
|
|
120
|
+
* `START_COMMAND_CGROUP_SAMPLE_INTERVAL_SECONDS=off` disables sampling.
|
|
81
121
|
* @param {string} containerName - Container to sample
|
|
82
122
|
* @returns {string} Shell command
|
|
83
123
|
*/
|
|
@@ -85,32 +125,61 @@ function buildCgroupSamplerStartSnippet(containerName) {
|
|
|
85
125
|
const v = CGROUP_SHELL_VARS;
|
|
86
126
|
const name = shellQuote(containerName);
|
|
87
127
|
const remoteRead = '__start_command_cgroup_read /sys/fs/cgroup';
|
|
128
|
+
const unavailable = (detail) =>
|
|
129
|
+
`printf '%s\\n' "local cgroup unavailable; ${detail}" > "$${v.file}.reason"`;
|
|
130
|
+
const execReason = unavailable(
|
|
131
|
+
'Docker exec exit=$__scs_exit: ${__scs_detail:-private cgroup v2 counters unreadable}'
|
|
132
|
+
);
|
|
88
133
|
return [
|
|
89
134
|
buildCgroupFunctionsSnippet(),
|
|
90
135
|
`${v.file}="\${TMPDIR:-/tmp}/start-command-cgroup.$$"`,
|
|
91
136
|
`rm -f "$${v.file}" "$${v.file}.tmp" "$${v.file}.reason"`,
|
|
92
137
|
`__start_command_memory_limit=$(docker inspect -f '{{.HostConfig.Memory}}' ${name} 2>/dev/null)`,
|
|
93
138
|
`__start_command_sample_name=${name}`,
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
`
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
`
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
`
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
'if [ -n "$__scs_previous" ] && [ "$
|
|
109
|
-
'
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
`
|
|
113
|
-
`
|
|
139
|
+
`( export START_COMMAND_CGROUP_SAMPLING=1; ` +
|
|
140
|
+
// Validate in the watcher, so attached and detached launches inherit the
|
|
141
|
+
// same setting. Never allow a malformed/zero sleep to create a busy loop.
|
|
142
|
+
`__scs_interval=\${START_COMMAND_CGROUP_SAMPLE_INTERVAL_SECONDS:-${CGROUP_SAMPLE_INTERVAL_SECONDS}}; ` +
|
|
143
|
+
`case "$__scs_interval" in [Oo][Ff][Ff]) printf '%s\\n' '${CGROUP_SAMPLING_OFF_REASON}' > "$${v.file}.reason"; exit 0;; ` +
|
|
144
|
+
`*[!0-9]*|'') __scs_interval=${CGROUP_SAMPLE_INTERVAL_SECONDS};; esac; ` +
|
|
145
|
+
`[ "$__scs_interval" -gt 0 ] 2>/dev/null || __scs_interval=${CGROUP_SAMPLE_INTERVAL_SECONDS}; ` +
|
|
146
|
+
// Store one `<max> <peak> <oom> <oom_kill>` line. The previous sample is
|
|
147
|
+
// read back from the file, so the stream's reader subshell can record
|
|
148
|
+
// too. The first counter is a baseline, including after watcher restart;
|
|
149
|
+
// only changes between adjacent samples supply exit-time evidence.
|
|
150
|
+
`__scs_record() { set -f; set -- $1; set +f; case "$3:$4" in *[!0-9:]*|:*|*:) return 1;; esac; ` +
|
|
151
|
+
`__scs_now=$(date +%s); __scs_previous=''; __scs_changed=0; __scs_previous_at=0; ` +
|
|
152
|
+
`[ ! -r "$${v.file}" ] || read -r __scs_p __scs_p __scs_p __scs_previous __scs_changed __scs_previous_at __scs_p < "$${v.file}"; ` +
|
|
153
|
+
'if [ -n "$__scs_previous" ] && [ "$4" -gt "$__scs_previous" ] 2>/dev/null && [ "$((__scs_now-__scs_previous_at))" -le 3 ]; then __scs_changed=$__scs_now; fi; ' +
|
|
154
|
+
`printf '%s %s %s %s %s %s %s\\n' "$1" "$2" "$3" "$4" "$__scs_changed" "$__scs_now" "$__scs_dir" > "$${v.file}.tmp" && mv -f "$${v.file}.tmp" "$${v.file}"; }; ` +
|
|
155
|
+
`__scs_dir=$(__start_command_cgroup_dir ${name}); ` +
|
|
156
|
+
// The namespace is fixed at container creation, including when reads fail.
|
|
157
|
+
`__scs_ns=$(docker inspect -f '{{.HostConfig.CgroupnsMode}}' ${name} 2>/dev/null); __scs_stream=1; __scs_reader=''; ` +
|
|
158
|
+
`while :; do __scs_ok=''; if [ -n "$__scs_dir" ]; then ` +
|
|
159
|
+
'__scs_line=$(__start_command_cgroup_read "$__scs_dir") && __scs_record "$__scs_line" && __scs_ok=1; ' +
|
|
160
|
+
`elif [ "$__scs_ns" != private ]; then ${unavailable('container cgroup namespace=${__scs_ns:-unavailable}, private required')}; break; ` +
|
|
161
|
+
`elif [ "$__scs_stream" = 1 ]; then ` +
|
|
162
|
+
// The reader ends with the stream; stopping the sampler stops it too.
|
|
163
|
+
`trap 'kill "$__scs_reader" 2>/dev/null; wait "$__scs_reader" 2>/dev/null; exit 0' TERM; ` +
|
|
164
|
+
// The ticker runs in the background, so a failed exec is reported at
|
|
165
|
+
// once; it ends on its first write after the exec is gone.
|
|
166
|
+
`{ { while :; do echo || exit 0; sleep "$__scs_interval" || exit 0; done 2>/dev/null & } | ` +
|
|
167
|
+
`docker exec -i ${name} sh -c ${shellQuote(buildCgroupStreamScript())} ${CGROUP_STREAM_NAME} /sys/fs/cgroup 2>&1; printf '__scs_exit %s\\n' "$?"; } | ` +
|
|
168
|
+
`{ __scs_got=''; __scs_exit=''; __scs_detail=''; while IFS= read -r __scs_line; do case "$__scs_line" in ` +
|
|
169
|
+
`'__scs_exit '*) __scs_exit=\${__scs_line#__scs_exit };; ` +
|
|
170
|
+
`*) if __scs_record "$__scs_line"; then __scs_got=1; elif [ -z "$__scs_detail" ]; then __scs_detail=$(printf '%s' "$__scs_line" | cut -c 1-240); fi;; esac; done; ` +
|
|
171
|
+
`[ -z "$__scs_got" ] || exit 0; ${execReason}; ` +
|
|
172
|
+
`case "$__scs_exit" in ''|0|*[!0-9]*) exit 1;; esac; exit "$__scs_exit"; } & ` +
|
|
173
|
+
`__scs_reader=$!; wait "$__scs_reader"; __scs_exit=$?; trap - TERM; __scs_reader=''; ` +
|
|
174
|
+
// No shell in the image: a per-round exec cannot succeed either.
|
|
175
|
+
`case "$__scs_exit" in 0) ;; 126|127) break;; *) __scs_stream=0;; esac; ` +
|
|
176
|
+
`else __scs_remote=$(docker exec ${name} sh -c ${shellQuote(`${buildCgroupFunctionsSnippet()}; ${remoteRead}`)} 2>&1); __scs_exit=$?; ` +
|
|
177
|
+
'if [ "$__scs_exit" = 0 ] && __scs_record "$__scs_remote"; then __scs_ok=1; ' +
|
|
178
|
+
`else __scs_detail=$(printf "%s" "$__scs_remote" | sed -n "1p" | cut -c 1-240); ${execReason}; fi; fi; ` +
|
|
179
|
+
// A successful read needs no Docker liveness probe, and the completion
|
|
180
|
+
// watcher still stops us after docker wait.
|
|
181
|
+
`if [ -z "$__scs_ok" ] && [ "$(docker inspect -f '{{.State.Running}}' ${name} 2>/dev/null)" = false ]; then break; fi; ` +
|
|
182
|
+
`sleep "$__scs_interval"; done ) >/dev/null 2>&1 & ${v.sampler}=$!`,
|
|
114
183
|
].join('; ');
|
|
115
184
|
}
|
|
116
185
|
|
|
@@ -305,13 +374,16 @@ function formatCgroupMemory(memory) {
|
|
|
305
374
|
|
|
306
375
|
module.exports = {
|
|
307
376
|
CGROUP_SAMPLE_INTERVAL_SECONDS,
|
|
377
|
+
CGROUP_SAMPLING_OFF_REASON,
|
|
308
378
|
CGROUP_SHELL_VARS,
|
|
379
|
+
CGROUP_STREAM_NAME,
|
|
309
380
|
OOM_SCOPE,
|
|
310
381
|
OOM_SCOPE_NOTES,
|
|
311
382
|
buildCgroupFunctionsSnippet,
|
|
312
383
|
buildCgroupMemoryLogSnippet,
|
|
313
384
|
buildCgroupSamplerStartSnippet,
|
|
314
385
|
buildCgroupSamplerStopSnippet,
|
|
386
|
+
buildCgroupStreamScript,
|
|
315
387
|
describeCgroupOomScope,
|
|
316
388
|
formatCgroupMemory,
|
|
317
389
|
formatCgroupMemoryLogLine,
|
|
@@ -60,6 +60,7 @@ function buildNextLevelCommand(options, command) {
|
|
|
60
60
|
for (const [key, flag] of [
|
|
61
61
|
['memory', '--memory'],
|
|
62
62
|
['memorySwap', '--memory-swap'],
|
|
63
|
+
['memoryReservation', '--memory-reservation'],
|
|
63
64
|
['cpus', '--cpus'],
|
|
64
65
|
['onKillResumeMemory', '--on-kill-resume-memory'],
|
|
65
66
|
]) {
|
|
@@ -112,6 +113,9 @@ function buildNextLevelCommand(options, command) {
|
|
|
112
113
|
if (options.privileged) {
|
|
113
114
|
parts.push('--privileged');
|
|
114
115
|
}
|
|
116
|
+
if (options.init) {
|
|
117
|
+
parts.push('--init');
|
|
118
|
+
}
|
|
115
119
|
}
|
|
116
120
|
|
|
117
121
|
// Pass through global flags
|
|
@@ -13,6 +13,7 @@
|
|
|
13
13
|
*/
|
|
14
14
|
|
|
15
15
|
const { spawnSync } = require('child_process');
|
|
16
|
+
const { prefetchedInspect } = require('./docker-inspect-batch');
|
|
16
17
|
|
|
17
18
|
/** `containerState` of a record whose container no longer exists. */
|
|
18
19
|
const CONTAINER_STATE_REMOVED = 'removed';
|
|
@@ -50,11 +51,9 @@ function isDockerNotFound(result) {
|
|
|
50
51
|
* the inspect failed; `notFound` says the failure was "no such container"
|
|
51
52
|
*/
|
|
52
53
|
function dockerInspect(dockerCommand, spawnOptions, format, target) {
|
|
53
|
-
const result =
|
|
54
|
-
|
|
55
|
-
['inspect', '-f', format, target],
|
|
56
|
-
spawnOptions
|
|
57
|
-
);
|
|
54
|
+
const result =
|
|
55
|
+
prefetchedInspect(format, target) ||
|
|
56
|
+
spawnSync(dockerCommand, ['inspect', '-f', format, target], spawnOptions);
|
|
58
57
|
if (result.error || result.status !== 0 || !result.stdout) {
|
|
59
58
|
return { stdout: null, notFound: isDockerNotFound(result) };
|
|
60
59
|
}
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Parse and validate the opt-in init process for docker isolation
|
|
3
|
+
* (issue #244):
|
|
4
|
+
* --init run `docker run --init`, so Docker's bundled tini is PID 1
|
|
5
|
+
* and reaps orphaned processes
|
|
6
|
+
* --no-init keep the command as PID 1, overriding the environment default
|
|
7
|
+
*
|
|
8
|
+
* Without either flag, START_COMMAND_DOCKER_INIT=1 turns it on for every
|
|
9
|
+
* docker launch on the host. The default stays off: a command that is its own
|
|
10
|
+
* supervisor keeps today's behaviour, and recovery works the same for
|
|
11
|
+
* containers started with or without it.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
const DOCKER_INIT_ENV = 'START_COMMAND_DOCKER_INIT';
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Whether the environment asks for an init process by default.
|
|
18
|
+
* @param {object} [env] - Environment
|
|
19
|
+
* @returns {boolean}
|
|
20
|
+
*/
|
|
21
|
+
function dockerInitFromEnv(env = process.env) {
|
|
22
|
+
const value = String(env[DOCKER_INIT_ENV] || '')
|
|
23
|
+
.trim()
|
|
24
|
+
.toLowerCase();
|
|
25
|
+
return ['1', 'true', 'yes', 'on'].includes(value);
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
function parseDockerInitOption(args, index, options) {
|
|
29
|
+
const arg = args[index];
|
|
30
|
+
if (arg === '--init' || arg === '--no-init') {
|
|
31
|
+
options.init = arg === '--init';
|
|
32
|
+
return 1;
|
|
33
|
+
}
|
|
34
|
+
return 0;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Resolve `options.init` to a boolean: the flag if one was given (`null`
|
|
39
|
+
* means none), otherwise the environment default for launches whose stack
|
|
40
|
+
* includes docker. Idempotent, because `validateOptions` may run again on
|
|
41
|
+
* options it already resolved: `--no-init` (or a resolved `false`) without
|
|
42
|
+
* docker is the default anyway and is accepted.
|
|
43
|
+
* @param {object} options - Parsed options
|
|
44
|
+
* @param {object} [env] - Environment
|
|
45
|
+
* @throws {Error} If `--init` is given without docker isolation
|
|
46
|
+
*/
|
|
47
|
+
function validateDockerInitOption(options, env = process.env) {
|
|
48
|
+
const stack = options.isolatedStack || [options.isolated];
|
|
49
|
+
const hasDocker = options.isolated !== null && stack.includes('docker');
|
|
50
|
+
if (options.init === true && !hasDocker) {
|
|
51
|
+
throw new Error(
|
|
52
|
+
'--init option is only valid when isolation stack includes docker'
|
|
53
|
+
);
|
|
54
|
+
}
|
|
55
|
+
if (typeof options.init !== 'boolean') {
|
|
56
|
+
options.init = hasDocker && dockerInitFromEnv(env);
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
module.exports = {
|
|
61
|
+
DOCKER_INIT_ENV,
|
|
62
|
+
dockerInitFromEnv,
|
|
63
|
+
parse: parseDockerInitOption,
|
|
64
|
+
validate: validateDockerInitOption,
|
|
65
|
+
};
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* One `docker inspect` for every container a status query probes (issue #243).
|
|
3
|
+
*
|
|
4
|
+
* `--list` used to start one or two `docker inspect` processes per record
|
|
5
|
+
* (the live state, the process IDs, the removal check), one after another,
|
|
6
|
+
* and `--status` up to three for its one record. While a query runs, this
|
|
7
|
+
* module asks Docker once for every field those probes read, and answers the
|
|
8
|
+
* per-record `docker inspect -f <format> <target>` calls from that answer.
|
|
9
|
+
*
|
|
10
|
+
* Docker prints one line per found container, in argument order, and one
|
|
11
|
+
* `No such container: <target>` line on stderr per missing one. Each line
|
|
12
|
+
* is matched back to its target by name or ID prefix (Docker's own lookup
|
|
13
|
+
* order), so the answer does not depend on the line order. A target the
|
|
14
|
+
* batch cannot account for (an old or stubbed docker, a daemon error) is
|
|
15
|
+
* not answered, and its per-record probe runs as before.
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
const { spawnSync } = require('child_process');
|
|
19
|
+
|
|
20
|
+
/** Fields of the batched inspect, in output order. */
|
|
21
|
+
const BATCH_FIELDS = [
|
|
22
|
+
'.Name',
|
|
23
|
+
'.Id',
|
|
24
|
+
'.State.Pid',
|
|
25
|
+
'.State.Running',
|
|
26
|
+
'.State.ExitCode',
|
|
27
|
+
'.State.OOMKilled',
|
|
28
|
+
'.State.StartedAt',
|
|
29
|
+
'.State.FinishedAt',
|
|
30
|
+
];
|
|
31
|
+
const BATCH_FORMAT = BATCH_FIELDS.map((field) => `{{${field}}}`).join(' ');
|
|
32
|
+
const NOT_FOUND_LINE = /no such (?:object|container):\s*(\S+)/gi;
|
|
33
|
+
const PLACEHOLDER = /\{\{\s*(\.[A-Za-z.]+)\s*\}\}/g;
|
|
34
|
+
|
|
35
|
+
/** Answers of the query in progress: target -> fields, or null (missing). */
|
|
36
|
+
let prefetched = null;
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* Containers the status probes of these records would inspect. Records
|
|
40
|
+
* whose container is already marked removed are skipped: they are never
|
|
41
|
+
* probed (issue #243).
|
|
42
|
+
* @param {Object[]} records - Execution records
|
|
43
|
+
* @returns {string[]} Unique names and IDs
|
|
44
|
+
*/
|
|
45
|
+
function dockerProbeTargets(records) {
|
|
46
|
+
const targets = new Set();
|
|
47
|
+
for (const record of records) {
|
|
48
|
+
const opts = (record && record.options) || {};
|
|
49
|
+
if (opts.isolated !== 'docker' || record.containerState === 'removed') {
|
|
50
|
+
continue;
|
|
51
|
+
}
|
|
52
|
+
for (const target of [opts.sessionName, opts.containerId]) {
|
|
53
|
+
if (typeof target === 'string' && target) {
|
|
54
|
+
targets.add(target);
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
return [...targets];
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* Match a batched inspect answer to its targets.
|
|
63
|
+
* @param {string[]} targets - Names and IDs passed to docker
|
|
64
|
+
* @param {string} stdout - One line of `BATCH_FORMAT` per found container
|
|
65
|
+
* @param {string} stderr - Docker's errors
|
|
66
|
+
* @returns {Map<string, Object|null>} Fields per answered target, null for
|
|
67
|
+
* a target Docker reported missing; unanswered targets are absent
|
|
68
|
+
*/
|
|
69
|
+
function parseBatchInspect(targets, stdout, stderr) {
|
|
70
|
+
const rows = String(stdout || '')
|
|
71
|
+
.split('\n')
|
|
72
|
+
.map((line) => line.trim().split(/\s+/))
|
|
73
|
+
.filter((parts) => parts.length === BATCH_FIELDS.length)
|
|
74
|
+
.map((parts) =>
|
|
75
|
+
Object.fromEntries(BATCH_FIELDS.map((field, i) => [field, parts[i]]))
|
|
76
|
+
);
|
|
77
|
+
const missing = new Set(
|
|
78
|
+
[...String(stderr || '').matchAll(NOT_FOUND_LINE)].map((m) => m[1])
|
|
79
|
+
);
|
|
80
|
+
const answers = new Map();
|
|
81
|
+
for (const target of targets) {
|
|
82
|
+
// Docker's own "no such container" wins over a prefix of another
|
|
83
|
+
// target's ID.
|
|
84
|
+
const row = missing.has(target)
|
|
85
|
+
? null
|
|
86
|
+
: rows.find((r) => r['.Name'] === `/${target}`) ||
|
|
87
|
+
rows.find((r) => r['.Id'].startsWith(target));
|
|
88
|
+
if (row !== undefined) {
|
|
89
|
+
answers.set(target, row);
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
return answers;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* Run the batched inspect.
|
|
97
|
+
* @returns {Map<string, Object|null>} See `parseBatchInspect`
|
|
98
|
+
*/
|
|
99
|
+
function runBatchInspect(targets, dockerCommand, spawnOptions) {
|
|
100
|
+
if (targets.length === 0) {
|
|
101
|
+
return new Map();
|
|
102
|
+
}
|
|
103
|
+
const result = spawnSync(
|
|
104
|
+
dockerCommand,
|
|
105
|
+
['inspect', '--type', 'container', '-f', BATCH_FORMAT, ...targets],
|
|
106
|
+
spawnOptions
|
|
107
|
+
);
|
|
108
|
+
if (result.error) {
|
|
109
|
+
return new Map();
|
|
110
|
+
}
|
|
111
|
+
return parseBatchInspect(targets, result.stdout, result.stderr);
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/**
|
|
115
|
+
* Run `fn` with the docker probes of `records` answered by one inspect.
|
|
116
|
+
* Nested calls reuse the outer answer.
|
|
117
|
+
* @param {Object[]} records - Records the query will probe
|
|
118
|
+
* @param {{dockerCommand: string, spawnOptions: Object}} docker - How to run docker
|
|
119
|
+
* @param {function(): *} fn - The query
|
|
120
|
+
* @returns {*} What `fn` returns
|
|
121
|
+
*/
|
|
122
|
+
function withDockerInspectBatch(records, docker, fn) {
|
|
123
|
+
if (prefetched) {
|
|
124
|
+
return fn();
|
|
125
|
+
}
|
|
126
|
+
prefetched = runBatchInspect(
|
|
127
|
+
dockerProbeTargets(records),
|
|
128
|
+
docker.dockerCommand,
|
|
129
|
+
docker.spawnOptions
|
|
130
|
+
);
|
|
131
|
+
try {
|
|
132
|
+
return fn();
|
|
133
|
+
} finally {
|
|
134
|
+
prefetched = null;
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
/**
|
|
139
|
+
* Answer `docker inspect -f <format> <target>` from the batch.
|
|
140
|
+
* @param {string} format - Go template of the per-record probe
|
|
141
|
+
* @param {string} target - Container name or ID
|
|
142
|
+
* @returns {{success: boolean, status: number, stdout: string,
|
|
143
|
+
* stderr: string}|null} A spawn-like result, or null when the batch has
|
|
144
|
+
* no answer and the probe must run
|
|
145
|
+
*/
|
|
146
|
+
function prefetchedInspect(format, target) {
|
|
147
|
+
if (!prefetched || !prefetched.has(target)) {
|
|
148
|
+
return null;
|
|
149
|
+
}
|
|
150
|
+
const row = prefetched.get(target);
|
|
151
|
+
if (row === null) {
|
|
152
|
+
return {
|
|
153
|
+
success: false,
|
|
154
|
+
status: 1,
|
|
155
|
+
stdout: '',
|
|
156
|
+
stderr: `Error response from daemon: No such container: ${target}\n`,
|
|
157
|
+
};
|
|
158
|
+
}
|
|
159
|
+
let known = true;
|
|
160
|
+
const stdout = format.replace(PLACEHOLDER, (_, field) => {
|
|
161
|
+
known = known && field in row;
|
|
162
|
+
return row[field];
|
|
163
|
+
});
|
|
164
|
+
if (!known || stdout.includes('{{')) {
|
|
165
|
+
return null;
|
|
166
|
+
}
|
|
167
|
+
return { success: true, status: 0, stdout: `${stdout}\n`, stderr: '' };
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
module.exports = {
|
|
171
|
+
BATCH_FORMAT,
|
|
172
|
+
dockerProbeTargets,
|
|
173
|
+
parseBatchInspect,
|
|
174
|
+
prefetchedInspect,
|
|
175
|
+
withDockerInspectBatch,
|
|
176
|
+
};
|
|
@@ -4,6 +4,7 @@ const { normalizeResourceLimits } = require('./docker-resource-limits');
|
|
|
4
4
|
const FIELDS = {
|
|
5
5
|
'--memory': 'memory',
|
|
6
6
|
'--memory-swap': 'memorySwap',
|
|
7
|
+
'--memory-reservation': 'memoryReservation',
|
|
7
8
|
'--cpus': 'cpus',
|
|
8
9
|
'--on-kill-resume-memory': 'onKillResumeMemory',
|
|
9
10
|
};
|
|
@@ -95,7 +96,7 @@ function readDaemonCapacity(runner) {
|
|
|
95
96
|
|
|
96
97
|
function resolveResourceOptions(options, runner, random = Math.random) {
|
|
97
98
|
const specs = {};
|
|
98
|
-
for (const field of ['memory', 'memorySwap', 'cpus']) {
|
|
99
|
+
for (const field of ['memory', 'memorySwap', 'memoryReservation', 'cpus']) {
|
|
99
100
|
if (options[field] !== null && options[field] !== undefined) {
|
|
100
101
|
specs[field] = parseResourceSpec(options[field], field === 'cpus');
|
|
101
102
|
}
|
|
@@ -137,8 +138,26 @@ function resolveResourceOptions(options, runner, random = Math.random) {
|
|
|
137
138
|
) {
|
|
138
139
|
throw new Error('memory-swap must be at least memory');
|
|
139
140
|
}
|
|
141
|
+
// Compare effective values so lowering a cap also checks a saved reservation.
|
|
142
|
+
const existingReservation = limits
|
|
143
|
+
.find((flag) => flag.startsWith('--memory-reservation='))
|
|
144
|
+
?.slice('--memory-reservation='.length);
|
|
145
|
+
const reservation =
|
|
146
|
+
resolved.memoryReservation ??
|
|
147
|
+
(existingReservation
|
|
148
|
+
? parseResourceSpec(existingReservation).amount
|
|
149
|
+
: null);
|
|
150
|
+
if (
|
|
151
|
+
memory !== null &&
|
|
152
|
+
memory !== undefined &&
|
|
153
|
+
reservation !== null &&
|
|
154
|
+
reservation !== undefined &&
|
|
155
|
+
reservation >= memory
|
|
156
|
+
) {
|
|
157
|
+
throw new Error('memory-reservation must be less than memory');
|
|
158
|
+
}
|
|
140
159
|
for (const [field, amount] of Object.entries(resolved)) {
|
|
141
|
-
const flag =
|
|
160
|
+
const flag = Object.keys(FIELDS).find((key) => FIELDS[key] === field);
|
|
142
161
|
for (let i = limits.length - 1; i >= 0; i--) {
|
|
143
162
|
if (
|
|
144
163
|
limits[i].startsWith(`${flag}=`) ||
|