start-command 0.38.0 → 0.39.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +20 -0
- package/README.md +4 -2
- package/bunfig.toml +4 -0
- package/eslint.config.mjs +1 -0
- package/package.json +4 -4
- package/src/lib/args-parser.js +9 -0
- package/src/lib/cgroup-memory.js +95 -23
- package/src/lib/command-builder.js +4 -0
- package/src/lib/container-removal.js +4 -5
- package/src/lib/detached-finalize.js +1 -0
- package/src/lib/docker-init-option.js +65 -0
- package/src/lib/docker-inspect-batch.js +176 -0
- package/src/lib/docker-resource-options.js +21 -2
- package/src/lib/docker-runtime-args.js +12 -3
- package/src/lib/execution-control.js +8 -6
- package/src/lib/execution-recovery.js +4 -1
- package/src/lib/execution-resume.js +4 -1
- package/src/lib/execution-store.js +19 -1
- package/src/lib/exit-evidence.js +139 -7
- package/src/lib/output-blocks.js +11 -0
- package/src/lib/resume-resources.js +2 -2
- package/src/lib/status-formatter.js +34 -130
- package/src/lib/status-probe.js +149 -0
- package/src/lib/store-decode-cache.js +141 -0
- package/src/lib/usage.js +3 -0
- package/test/cargo-lock-version.mjs +121 -0
- package/test/check-rust-coverage.mjs +71 -0
- package/test/check-web-archive.mjs +154 -0
- package/test/ci-workflow-invariants.js +204 -0
- package/test/command-builder-docker-cleanup.js +24 -0
- package/test/command-exit-codes.mjs +91 -0
- package/test/docker-runtime-options.js +2 -0
- package/test/docker-snapshot-safety.js +39 -0
- package/test/execution-resume.js +2 -0
- package/test/fixtures/fake-npm.sh +64 -0
- package/test/fixtures/lychee-report.md +39 -0
- package/test/javascript-first-ci.js +69 -0
- package/test/links-workflow.mjs +127 -0
- package/test/lint-gate.mjs +56 -0
- package/test/npm-publish.mjs +181 -0
- package/test/recheck-broken-links.mjs +452 -0
- package/test/regression-176.js +2 -0
- package/test/regression-187.js +2 -0
- package/test/regression-191.js +4 -0
- package/test/regression-193.js +2 -0
- package/test/regression-198.js +2 -0
- package/test/regression-207.js +8 -2
- package/test/regression-224.js +2 -0
- package/test/regression-236.js +250 -0
- package/test/regression-238.js +129 -0
- package/test/regression-241.js +254 -0
- package/test/regression-242.js +241 -0
- package/test/regression-243.js +493 -0
- package/test/regression-244.js +471 -0
- package/test/release-push-retry.mjs +172 -0
- package/test/run-js-tests.mjs +15 -3
- package/test/setup/private-snapshot-lock.js +17 -0
- package/test/shared-golden.js +29 -2
- package/test/transient-links.mjs +139 -0
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Docker runtime options shared by `docker run`, `[Isolation]` status lines
|
|
3
|
-
* and execution-record metadata: privileged mode,
|
|
4
|
-
*
|
|
3
|
+
* and execution-record metadata: privileged mode, the init process
|
|
4
|
+
* (issue #244), environment, volumes, mounts, networks (issue #150) and
|
|
5
|
+
* resource limits (issue #176).
|
|
5
6
|
*/
|
|
6
7
|
|
|
7
8
|
const { getDockerNetworks } = require('./docker-network-lifecycle');
|
|
@@ -31,7 +32,7 @@ function getOnKillResume(options = {}) {
|
|
|
31
32
|
* mounts, --mount specs, network configuration and resource limits. Returned
|
|
32
33
|
* in a stable order so they can be spliced into the `docker run` argv before
|
|
33
34
|
* the image name.
|
|
34
|
-
* @param {object} options - Options (privileged, env, volumes, mounts, network, networkAliases, resourceLimits)
|
|
35
|
+
* @param {object} options - Options (privileged, init, env, volumes, mounts, network, networkAliases, resourceLimits)
|
|
35
36
|
* @returns {string[]} Docker CLI arguments
|
|
36
37
|
*/
|
|
37
38
|
function buildDockerRuntimeArgs(options = {}) {
|
|
@@ -39,6 +40,10 @@ function buildDockerRuntimeArgs(options = {}) {
|
|
|
39
40
|
if (options.privileged) {
|
|
40
41
|
args.push('--privileged');
|
|
41
42
|
}
|
|
43
|
+
// tini as PID 1 reaps orphaned processes (issue #244).
|
|
44
|
+
if (options.init) {
|
|
45
|
+
args.push('--init');
|
|
46
|
+
}
|
|
42
47
|
for (const envVar of options.env || []) {
|
|
43
48
|
args.push('-e', envVar);
|
|
44
49
|
}
|
|
@@ -94,6 +99,9 @@ function buildDockerRuntimeStatusLines(options = {}) {
|
|
|
94
99
|
if (options.privileged) {
|
|
95
100
|
lines.push(`[Isolation] Privileged: true`);
|
|
96
101
|
}
|
|
102
|
+
if (options.init) {
|
|
103
|
+
lines.push(`[Isolation] Init: true`);
|
|
104
|
+
}
|
|
97
105
|
const networks = getDockerNetworks(options);
|
|
98
106
|
if (networks.length > 0) {
|
|
99
107
|
lines.push(`[Isolation] Network: ${networks[0]}`);
|
|
@@ -145,6 +153,7 @@ function buildDockerRuntimeMetadata(options = {}) {
|
|
|
145
153
|
labels: options.labels && options.labels.length > 0 ? options.labels : null,
|
|
146
154
|
commandHandoff: options.detached ? true : null,
|
|
147
155
|
privileged: options.privileged || null,
|
|
156
|
+
init: options.init || null,
|
|
148
157
|
network: networks[0] || null,
|
|
149
158
|
networks: networks.length > 0 ? networks : null,
|
|
150
159
|
networkAliases:
|
|
@@ -11,6 +11,7 @@ const {
|
|
|
11
11
|
formatAsNestedLinksNotation,
|
|
12
12
|
} = require('./output-blocks');
|
|
13
13
|
const { getDockerCommand, getDockerSpawnOptions } = require('./docker-cleanup');
|
|
14
|
+
const { prefetchedInspect } = require('./docker-inspect-batch');
|
|
14
15
|
|
|
15
16
|
const ControlAction = {
|
|
16
17
|
STOP: 'stop',
|
|
@@ -148,14 +149,15 @@ function collectProcessIds(record, runner = runCommand) {
|
|
|
148
149
|
),
|
|
149
150
|
];
|
|
150
151
|
addIfPresent(processIds, 'commandPids', commandPids);
|
|
152
|
+
} else if (isolated === 'docker' && record.containerState === 'removed') {
|
|
153
|
+
// The container is gone, so there is nothing to ask Docker (#243).
|
|
154
|
+
addIfPresent(processIds, 'containerId', opts.containerId);
|
|
151
155
|
} else if (isolated === 'docker') {
|
|
152
156
|
addIfPresent(processIds, 'containerId', opts.containerId);
|
|
153
|
-
const
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
'
|
|
157
|
-
sessionName,
|
|
158
|
-
]);
|
|
157
|
+
const format = '{{.Id}} {{.State.Pid}}';
|
|
158
|
+
const result =
|
|
159
|
+
prefetchedInspect(format, sessionName) ||
|
|
160
|
+
runner(getDockerCommand(), ['inspect', '-f', format, sessionName]);
|
|
159
161
|
if (result.success && result.stdout.trim()) {
|
|
160
162
|
const [containerId, pidValue] = result.stdout.trim().split(/\s+/);
|
|
161
163
|
addIfPresent(processIds, 'containerId', containerId);
|
|
@@ -336,6 +336,7 @@ function recoverKilledExecution(params = {}) {
|
|
|
336
336
|
record.options.exitEvidence = {
|
|
337
337
|
mainOom: terminalOom,
|
|
338
338
|
daemonRestart: logged.daemonRestart,
|
|
339
|
+
...evidence.sourceOf(logged),
|
|
339
340
|
};
|
|
340
341
|
let recoveryLimits = null;
|
|
341
342
|
if (opts.onKillResumeMemory && terminalOom) {
|
|
@@ -490,7 +491,9 @@ function recoverKilledExecution(params = {}) {
|
|
|
490
491
|
const updated = runner(getDockerCommand(), [
|
|
491
492
|
'update',
|
|
492
493
|
...resourceLimits.filter((s) =>
|
|
493
|
-
/^--(?:memory|memory-swap|cpus|cpu-quota|cpu-period)=/.test(
|
|
494
|
+
/^--(?:memory|memory-swap|memory-reservation|cpus|cpu-quota|cpu-period)=/.test(
|
|
495
|
+
s
|
|
496
|
+
)
|
|
494
497
|
),
|
|
495
498
|
containerName,
|
|
496
499
|
]);
|
|
@@ -127,6 +127,7 @@ function buildLaunchOptions(record) {
|
|
|
127
127
|
useCommandStream: opts.useCommandStream || false,
|
|
128
128
|
shell: opts.shell || 'auto',
|
|
129
129
|
privileged: opts.privileged || false,
|
|
130
|
+
init: opts.init || false,
|
|
130
131
|
env: opts.env || [],
|
|
131
132
|
volumes: opts.volumes || [],
|
|
132
133
|
mounts: opts.mounts || [],
|
|
@@ -535,7 +536,9 @@ async function resumeExecution(store, identifier, deps = {}) {
|
|
|
535
536
|
args: [
|
|
536
537
|
'update',
|
|
537
538
|
...resources.resourceLimits.filter((s) =>
|
|
538
|
-
/^--(?:memory|memory-swap|cpus|cpu-quota|cpu-period)=/.test(
|
|
539
|
+
/^--(?:memory|memory-swap|memory-reservation|cpus|cpu-quota|cpu-period)=/.test(
|
|
540
|
+
s
|
|
541
|
+
)
|
|
539
542
|
),
|
|
540
543
|
plan.sessionName,
|
|
541
544
|
],
|
|
@@ -25,6 +25,7 @@ const os = require('os');
|
|
|
25
25
|
const { execFileSync, spawnSync } = require('child_process');
|
|
26
26
|
const crypto = require('crypto');
|
|
27
27
|
const { atomicWrite } = require('./atomic-write');
|
|
28
|
+
const { decodeStoreContent } = require('./store-decode-cache');
|
|
28
29
|
|
|
29
30
|
// Synchronous wrapper using Bun's native ESM support
|
|
30
31
|
// This works because Bun handles ESM/CJS interop
|
|
@@ -38,6 +39,12 @@ function encodeSync(data) {
|
|
|
38
39
|
}
|
|
39
40
|
}
|
|
40
41
|
|
|
42
|
+
const DECODE_LOG = {
|
|
43
|
+
memory: () => 'Reused the decoded store: the file is unchanged',
|
|
44
|
+
copy: () => 'Read the decoded copy of the store: the file is unchanged',
|
|
45
|
+
decode: (content) => `Decoded the store (${content.length} characters)`,
|
|
46
|
+
};
|
|
47
|
+
|
|
41
48
|
function decodeSync(notation) {
|
|
42
49
|
try {
|
|
43
50
|
const codec = require('lino-objects-codec');
|
|
@@ -52,6 +59,8 @@ const DEFAULT_APP_FOLDER = path.join(os.homedir(), '.start-command');
|
|
|
52
59
|
const LINO_DB_FILE = 'executions.lino';
|
|
53
60
|
const LINKS_DB_FILE = 'executions.links';
|
|
54
61
|
const LOCK_FILE = 'executions.lock';
|
|
62
|
+
// Decoded records as JSON, for the next process (issue #243)
|
|
63
|
+
const DECODED_COPY_FILE = 'executions.decoded.js.json';
|
|
55
64
|
const { LockManager } = require('./store-lock');
|
|
56
65
|
|
|
57
66
|
/**
|
|
@@ -262,6 +271,7 @@ class ExecutionStore {
|
|
|
262
271
|
this.linoDbPath = path.join(this.appFolder, LINO_DB_FILE);
|
|
263
272
|
this.linksDbPath = path.join(this.appFolder, LINKS_DB_FILE);
|
|
264
273
|
this.lockFilePath = path.join(this.appFolder, LOCK_FILE);
|
|
274
|
+
this.decodedCopyPath = path.join(this.appFolder, DECODED_COPY_FILE);
|
|
265
275
|
this.useLinks = options.useLinks !== false && isClinkInstalled();
|
|
266
276
|
this.verbose = options.verbose || false;
|
|
267
277
|
|
|
@@ -302,7 +312,14 @@ class ExecutionStore {
|
|
|
302
312
|
return [];
|
|
303
313
|
}
|
|
304
314
|
|
|
305
|
-
|
|
315
|
+
// Decoded once per content, not once per call (issue #243).
|
|
316
|
+
const { data, source } = decodeStoreContent(
|
|
317
|
+
this.linoDbPath,
|
|
318
|
+
content,
|
|
319
|
+
decodeSync,
|
|
320
|
+
{ copyPath: this.decodedCopyPath }
|
|
321
|
+
);
|
|
322
|
+
this.log(DECODE_LOG[source](content));
|
|
306
323
|
if (!Array.isArray(data)) {
|
|
307
324
|
return [];
|
|
308
325
|
}
|
|
@@ -915,4 +932,5 @@ module.exports = {
|
|
|
915
932
|
LINO_DB_FILE,
|
|
916
933
|
LINKS_DB_FILE,
|
|
917
934
|
LOCK_FILE,
|
|
935
|
+
DECODED_COPY_FILE,
|
|
918
936
|
};
|
package/src/lib/exit-evidence.js
CHANGED
|
@@ -5,6 +5,16 @@ const MAIN_OOM = 'Exit evidence: main-oom (recent cgroup oom_kill delta)';
|
|
|
5
5
|
const DOCKER_OOM = 'Exit evidence: main-oom (recent Docker oom event)';
|
|
6
6
|
const DAEMON_RESTART =
|
|
7
7
|
'Exit evidence: docker-daemon-restart (Docker service journal)';
|
|
8
|
+
const DAEMON_RESTART_API =
|
|
9
|
+
'Exit evidence: docker-daemon-restart (Docker API: containers finished together)';
|
|
10
|
+
const DAEMON_RESTART_PREFIX = 'Exit evidence: docker-daemon-restart';
|
|
11
|
+
// dockerd gives every container its stop timeout and kills the rest within its
|
|
12
|
+
// 15 s shutdown timeout, so the containers of one restart finish this close.
|
|
13
|
+
const DAEMON_RESTART_WINDOW_MS = 15000;
|
|
14
|
+
const CONTAINER_STATE_FORMAT =
|
|
15
|
+
'{{.Id}} {{.State.FinishedAt}} {{.State.StartedAt}} {{.State.ExitCode}} {{.State.OOMKilled}}';
|
|
16
|
+
// Bounds one `docker inspect` call on hosts with many stopped containers.
|
|
17
|
+
const MAX_PEER_CONTAINERS = 256;
|
|
8
18
|
|
|
9
19
|
function recentOomDelta(sample, finishedAt) {
|
|
10
20
|
const fields = String(sample || '')
|
|
@@ -39,6 +49,42 @@ function daemonRestartEvidence(journal, containerId) {
|
|
|
39
49
|
return Boolean(exited && forced);
|
|
40
50
|
}
|
|
41
51
|
|
|
52
|
+
/**
|
|
53
|
+
* Socket-only daemon-restart evidence (issue #241): another container that
|
|
54
|
+
* finished within the same shutdown window, was not OOM-killed, and was either
|
|
55
|
+
* stopped by a signal (137/143) or started again afterwards (restart policy;
|
|
56
|
+
* moby resets its ExitCode to 0 but keeps FinishedAt). `docker events` cannot
|
|
57
|
+
* be used: its buffer does not survive the restart, while State does.
|
|
58
|
+
* @param {string} states - `CONTAINER_STATE_FORMAT` lines, one per container
|
|
59
|
+
*/
|
|
60
|
+
function daemonRestartApiEvidence(states, containerId, finishedAt) {
|
|
61
|
+
const finished = Date.parse(finishedAt);
|
|
62
|
+
return (
|
|
63
|
+
Number.isFinite(finished) &&
|
|
64
|
+
String(states || '')
|
|
65
|
+
.split('\n')
|
|
66
|
+
.some((line) => {
|
|
67
|
+
const [id, peerFinished, peerStarted, exit, oom] = line
|
|
68
|
+
.trim()
|
|
69
|
+
.split(/\s+/);
|
|
70
|
+
const end = Date.parse(peerFinished);
|
|
71
|
+
if (
|
|
72
|
+
!/^[a-f0-9]{64}$/.test(id || '') ||
|
|
73
|
+
id === containerId ||
|
|
74
|
+
oom !== 'false' ||
|
|
75
|
+
!Number.isFinite(end) ||
|
|
76
|
+
end <= 0 ||
|
|
77
|
+
Math.abs(end - finished) > DAEMON_RESTART_WINDOW_MS
|
|
78
|
+
) {
|
|
79
|
+
return false;
|
|
80
|
+
}
|
|
81
|
+
return (
|
|
82
|
+
exit === '137' || exit === '143' || Date.parse(peerStarted) > end
|
|
83
|
+
);
|
|
84
|
+
})
|
|
85
|
+
);
|
|
86
|
+
}
|
|
87
|
+
|
|
42
88
|
function recentDockerOom(events, finishedAt) {
|
|
43
89
|
const finished = Date.parse(finishedAt) / 1000;
|
|
44
90
|
return (
|
|
@@ -68,6 +114,10 @@ function collectExitEvidence(
|
|
|
68
114
|
// A local journal cannot diagnose a remote daemon. Require a local socket.
|
|
69
115
|
const local =
|
|
70
116
|
!process.env.DOCKER_HOST || process.env.DOCKER_HOST.startsWith('unix://');
|
|
117
|
+
const sampledOom =
|
|
118
|
+
(oomKilled === true || oomKilled === 'true') &&
|
|
119
|
+
(Number(exitCode) === 137 || Number(exitCode) < 0) &&
|
|
120
|
+
recentOomDelta(sample, finishedAt);
|
|
71
121
|
if (
|
|
72
122
|
Number(exitCode) === 137 &&
|
|
73
123
|
containerName &&
|
|
@@ -114,18 +164,35 @@ function collectExitEvidence(
|
|
|
114
164
|
if (events.success && recentDockerOom(events.stdout, finishedAt)) {
|
|
115
165
|
return DOCKER_OOM;
|
|
116
166
|
}
|
|
167
|
+
// Without a host journal (a mounted docker.sock, a remote daemon) the
|
|
168
|
+
// only record of the restart is the state of the other containers.
|
|
169
|
+
const peers = sampledOom
|
|
170
|
+
? { success: false, stdout: '' }
|
|
171
|
+
: run(docker, ['ps', '-a', '-q', '--no-trunc']);
|
|
172
|
+
const ids = peers.stdout
|
|
173
|
+
.split(/\s+/)
|
|
174
|
+
.filter(
|
|
175
|
+
(peer) => /^[a-f0-9]{64}$/.test(peer) && peer !== id.stdout.trim()
|
|
176
|
+
)
|
|
177
|
+
.slice(0, MAX_PEER_CONTAINERS);
|
|
178
|
+
// A container removed after `docker ps` fails inspect with status 1 but
|
|
179
|
+
// the others are still printed, so the output is used either way.
|
|
180
|
+
const states = ids.length
|
|
181
|
+
? run(docker, ['inspect', '-f', CONTAINER_STATE_FORMAT, ...ids])
|
|
182
|
+
: { success: false, stdout: '' };
|
|
183
|
+
if (
|
|
184
|
+
daemonRestartApiEvidence(states.stdout, id.stdout.trim(), finishedAt)
|
|
185
|
+
) {
|
|
186
|
+
return DAEMON_RESTART_API;
|
|
187
|
+
}
|
|
117
188
|
if (process.env.START_DEBUG === '1') {
|
|
118
189
|
console.error(
|
|
119
|
-
`[DEBUG] Exit evidence: container=${id.stdout.trim()} finishedAt=${finishedAt} eventsAvailable=${events.success} oomEvents=${JSON.stringify(events.stdout)} sample=${JSON.stringify(sample || '')}`
|
|
190
|
+
`[DEBUG] Exit evidence: container=${id.stdout.trim()} finishedAt=${finishedAt} eventsAvailable=${events.success} oomEvents=${JSON.stringify(events.stdout)} peerContainers=${ids.length} peerStates=${JSON.stringify(states.stdout)} sample=${JSON.stringify(sample || '')}`
|
|
120
191
|
);
|
|
121
192
|
}
|
|
122
193
|
}
|
|
123
194
|
}
|
|
124
|
-
if (
|
|
125
|
-
(oomKilled === true || oomKilled === 'true') &&
|
|
126
|
-
(Number(exitCode) === 137 || Number(exitCode) < 0) &&
|
|
127
|
-
recentOomDelta(sample, finishedAt)
|
|
128
|
-
) {
|
|
195
|
+
if (sampledOom) {
|
|
129
196
|
return MAIN_OOM;
|
|
130
197
|
}
|
|
131
198
|
return 'Exit evidence: unavailable (no attributed exit-time OOM or daemon restart evidence)';
|
|
@@ -134,12 +201,72 @@ function collectExitEvidence(
|
|
|
134
201
|
function fromLog(log) {
|
|
135
202
|
const lines = String(log || '').split('\n');
|
|
136
203
|
const last = lines.filter((line) => line.startsWith('Exit evidence:')).at(-1);
|
|
204
|
+
const daemonRestart = Boolean(last?.startsWith(DAEMON_RESTART_PREFIX));
|
|
137
205
|
return {
|
|
138
206
|
mainOom: last === MAIN_OOM || last === DOCKER_OOM,
|
|
139
|
-
daemonRestart
|
|
207
|
+
daemonRestart,
|
|
208
|
+
// Which source attributed the restart: the host journal or the API.
|
|
209
|
+
...(daemonRestart
|
|
210
|
+
? { daemonRestartSource: last === DAEMON_RESTART_API ? 'api' : 'journal' }
|
|
211
|
+
: {}),
|
|
140
212
|
};
|
|
141
213
|
}
|
|
142
214
|
|
|
215
|
+
/** The stored `daemonRestartSource`, kept only when a restart was attributed. */
|
|
216
|
+
function sourceOf(logged) {
|
|
217
|
+
return logged?.daemonRestart && logged.daemonRestartSource
|
|
218
|
+
? { daemonRestartSource: logged.daemonRestartSource }
|
|
219
|
+
: {};
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
// A watcher that shares the daemon with its task (a mounted docker.sock) dies
|
|
223
|
+
// in the same restart and never logs evidence, so `--status` collects it from
|
|
224
|
+
// the container state that survived, once per process (issue #241).
|
|
225
|
+
const lostWatcherEvidence = new Map();
|
|
226
|
+
function collectLostWatcherEvidence(record, state, runner) {
|
|
227
|
+
const name = record?.options?.sessionName;
|
|
228
|
+
if (
|
|
229
|
+
!name ||
|
|
230
|
+
record.options.exitEvidence ||
|
|
231
|
+
state?.running !== false ||
|
|
232
|
+
state.exitCode !== 137 ||
|
|
233
|
+
!state.finishedAt
|
|
234
|
+
) {
|
|
235
|
+
return null;
|
|
236
|
+
}
|
|
237
|
+
const key = `${name}\0${state.finishedAt}`;
|
|
238
|
+
if (!lostWatcherEvidence.has(key)) {
|
|
239
|
+
const logged = fromLog(
|
|
240
|
+
collectExitEvidence(
|
|
241
|
+
{
|
|
242
|
+
containerName: name,
|
|
243
|
+
finishedAt: state.finishedAt,
|
|
244
|
+
exitCode: state.exitCode,
|
|
245
|
+
oomKilled: state.oomKilled,
|
|
246
|
+
},
|
|
247
|
+
runner
|
|
248
|
+
)
|
|
249
|
+
);
|
|
250
|
+
lostWatcherEvidence.set(key, {
|
|
251
|
+
mainOom: logged.mainOom,
|
|
252
|
+
daemonRestart: logged.daemonRestart,
|
|
253
|
+
...sourceOf(logged),
|
|
254
|
+
});
|
|
255
|
+
}
|
|
256
|
+
return { ...lostWatcherEvidence.get(key) };
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
/** Stores the lost watcher's evidence when the log tail carries none. */
|
|
260
|
+
function addLostWatcherEvidence(enriched, record, state, logTail) {
|
|
261
|
+
const lost =
|
|
262
|
+
state && !/^Exit evidence:/m.test(logTail || '')
|
|
263
|
+
? collectLostWatcherEvidence(record, state)
|
|
264
|
+
: null;
|
|
265
|
+
if (lost) {
|
|
266
|
+
enriched.options = { ...enriched.options, exitEvidence: lost };
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
|
|
143
270
|
function buildExitEvidenceSnippet(containerName, quotedLogPath) {
|
|
144
271
|
const stderr = process.env.START_DEBUG === '1' ? '2>&1' : '2>/dev/null';
|
|
145
272
|
return `${shellQuote(process.execPath)} ${shellQuote(__filename)} ${shellQuote(containerName)} "$__start_command_cgroup" "$__start_command_finished" "$__start_command_exit" "$__start_command_oom" >> ${quotedLogPath} ${stderr}`;
|
|
@@ -149,11 +276,16 @@ module.exports = {
|
|
|
149
276
|
MAIN_OOM,
|
|
150
277
|
DOCKER_OOM,
|
|
151
278
|
DAEMON_RESTART,
|
|
279
|
+
DAEMON_RESTART_API,
|
|
152
280
|
recentOomDelta,
|
|
281
|
+
daemonRestartApiEvidence,
|
|
153
282
|
recentDockerOom,
|
|
154
283
|
daemonRestartEvidence,
|
|
155
284
|
collectExitEvidence,
|
|
156
285
|
fromLog,
|
|
286
|
+
sourceOf,
|
|
287
|
+
collectLostWatcherEvidence,
|
|
288
|
+
addLostWatcherEvidence,
|
|
157
289
|
buildExitEvidenceSnippet,
|
|
158
290
|
};
|
|
159
291
|
if (require.main === module) {
|
package/src/lib/output-blocks.js
CHANGED
|
@@ -119,6 +119,7 @@ function parseIsolationMetadata(extraLines) {
|
|
|
119
119
|
session: null,
|
|
120
120
|
endpoint: null,
|
|
121
121
|
user: null,
|
|
122
|
+
init: false,
|
|
122
123
|
};
|
|
123
124
|
|
|
124
125
|
for (const line of extraLines) {
|
|
@@ -146,6 +147,12 @@ function parseIsolationMetadata(extraLines) {
|
|
|
146
147
|
continue;
|
|
147
148
|
}
|
|
148
149
|
|
|
150
|
+
// Parse [Isolation] Init: true (docker run --init, issue #244)
|
|
151
|
+
if (line === '[Isolation] Init: true') {
|
|
152
|
+
metadata.init = true;
|
|
153
|
+
continue;
|
|
154
|
+
}
|
|
155
|
+
|
|
149
156
|
// Parse [Isolation] Endpoint: user@host
|
|
150
157
|
const endpointMatch = line.match(/\[Isolation\] Endpoint: (.+)/);
|
|
151
158
|
if (endpointMatch) {
|
|
@@ -207,6 +214,10 @@ function generateIsolationLines(metadata, containerOrScreenName = null) {
|
|
|
207
214
|
}
|
|
208
215
|
}
|
|
209
216
|
|
|
217
|
+
if (metadata.init) {
|
|
218
|
+
lines.push(createTimelineLine('init', 'true'));
|
|
219
|
+
}
|
|
220
|
+
|
|
210
221
|
if (metadata.user) {
|
|
211
222
|
lines.push(createTimelineLine('user', metadata.user));
|
|
212
223
|
}
|
|
@@ -34,7 +34,7 @@ function prepareResumeResources(record, probe, overrides, runner, random) {
|
|
|
34
34
|
const options = record.options || {};
|
|
35
35
|
if (options.isolated !== 'docker') {
|
|
36
36
|
if (
|
|
37
|
-
['memory', 'memorySwap', 'cpus'].some(
|
|
37
|
+
['memory', 'memorySwap', 'memoryReservation', 'cpus'].some(
|
|
38
38
|
(k) => overrides[k] !== null && overrides[k] !== undefined
|
|
39
39
|
)
|
|
40
40
|
) {
|
|
@@ -81,7 +81,7 @@ function prepareResumeResources(record, probe, overrides, runner, random) {
|
|
|
81
81
|
};
|
|
82
82
|
result.update =
|
|
83
83
|
probe.state === 'stopped' &&
|
|
84
|
-
(['memory', 'memorySwap', 'cpus'].some(
|
|
84
|
+
(['memory', 'memorySwap', 'memoryReservation', 'cpus'].some(
|
|
85
85
|
(k) => overrides[k] !== null && overrides[k] !== undefined
|
|
86
86
|
) ||
|
|
87
87
|
Boolean(options.cpuPenaltyConfig));
|
|
@@ -14,7 +14,7 @@ const {
|
|
|
14
14
|
formatTextOptionValue,
|
|
15
15
|
} = require('./status-links-notation');
|
|
16
16
|
const { collectProcessIds } = require('./execution-control');
|
|
17
|
-
const { getDockerCommand
|
|
17
|
+
const { getDockerCommand } = require('./docker-cleanup');
|
|
18
18
|
const {
|
|
19
19
|
describeExitCode,
|
|
20
20
|
CGROUP_OOM_EXIT_REASON,
|
|
@@ -25,136 +25,31 @@ const {
|
|
|
25
25
|
FATAL_MARKER_TAIL_BYTES,
|
|
26
26
|
readLogTail,
|
|
27
27
|
} = require('./isolation-log-utils');
|
|
28
|
-
const { normalizeDockerTimestamp } = require('./docker-post-mortem');
|
|
29
28
|
const { END_TIME_SOURCE } = require('./detached-finalize');
|
|
30
29
|
const { formatCgroupMemory } = require('./cgroup-memory');
|
|
30
|
+
const { addLostWatcherEvidence } = require('./exit-evidence');
|
|
31
31
|
const {
|
|
32
32
|
attachContainerRemoval,
|
|
33
|
-
dockerInspect,
|
|
34
33
|
persistContainerRemoval,
|
|
35
34
|
} = require('./container-removal');
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
* as "stopped"; it means "unknown", so callers can keep the session running
|
|
49
|
-
* instead of fabricating a terminal `-1` result.
|
|
50
|
-
*
|
|
51
|
-
* @param {string} sessionName - Container name
|
|
52
|
-
* @param {Object|null} [probe] - Receives `sessionNotFound` (issue #225)
|
|
53
|
-
* @returns {{running: boolean, exitCode: number|null, oomKilled: boolean|null}|null} State,
|
|
54
|
-
* or null when the container cannot be inspected (not found yet, removed, or docker error)
|
|
55
|
-
*/
|
|
56
|
-
function inspectDockerState(sessionName, probe = null) {
|
|
57
|
-
const { stdout, notFound } = dockerInspect(
|
|
58
|
-
getDockerCommand(),
|
|
59
|
-
dockerProbeSpawnOptions(),
|
|
60
|
-
'{{.State.Running}} {{.State.ExitCode}} {{.State.OOMKilled}} ' +
|
|
61
|
-
'{{.State.StartedAt}} {{.State.FinishedAt}}',
|
|
62
|
-
sessionName
|
|
63
|
-
);
|
|
64
|
-
if (probe) {
|
|
65
|
-
// "No such container" is a fact, unlike a docker that cannot be reached
|
|
66
|
-
// (issue #225); it still reads as unknown liveness here (issue #136).
|
|
67
|
-
probe.sessionNotFound = notFound;
|
|
68
|
-
}
|
|
69
|
-
if (stdout === null) {
|
|
70
|
-
return null;
|
|
71
|
-
}
|
|
72
|
-
// Parsed positionally and defensively: an older docker (or a stubbed one)
|
|
73
|
-
// may answer with fewer fields than the template asks for.
|
|
74
|
-
const [runningRaw, exitRaw, oomKilledRaw, startedRaw, finishedRaw] = stdout
|
|
75
|
-
.trim()
|
|
76
|
-
.split(/\s+/);
|
|
77
|
-
const exitCode = Number.parseInt(exitRaw, 10);
|
|
35
|
+
const { withDockerInspectBatch } = require('./docker-inspect-batch');
|
|
36
|
+
const {
|
|
37
|
+
backendExitCode,
|
|
38
|
+
dockerProbeSpawnOptions,
|
|
39
|
+
inspectDockerState,
|
|
40
|
+
isDetachedDockerRecord,
|
|
41
|
+
readDockerState,
|
|
42
|
+
resolveOomObservation,
|
|
43
|
+
} = require('./status-probe');
|
|
44
|
+
|
|
45
|
+
/** How the status probes run docker. */
|
|
46
|
+
function dockerProbe() {
|
|
78
47
|
return {
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
oomKilled:
|
|
82
|
-
oomKilledRaw === 'true' ? true : oomKilledRaw === 'false' ? false : null,
|
|
83
|
-
startedAt: normalizeDockerTimestamp(startedRaw),
|
|
84
|
-
finishedAt: normalizeDockerTimestamp(finishedRaw),
|
|
48
|
+
dockerCommand: getDockerCommand(),
|
|
49
|
+
spawnOptions: dockerProbeSpawnOptions(),
|
|
85
50
|
};
|
|
86
51
|
}
|
|
87
52
|
|
|
88
|
-
function isDetachedDockerRecord(record) {
|
|
89
|
-
const opts = record.options || {};
|
|
90
|
-
return (
|
|
91
|
-
opts.isolated === 'docker' &&
|
|
92
|
-
opts.isolationMode === 'detached' &&
|
|
93
|
-
Boolean(opts.sessionName)
|
|
94
|
-
);
|
|
95
|
-
}
|
|
96
|
-
|
|
97
|
-
function dockerProbeSpawnOptions() {
|
|
98
|
-
return getDockerSpawnOptions({
|
|
99
|
-
encoding: 'utf8',
|
|
100
|
-
env: process.env,
|
|
101
|
-
stdio: ['pipe', 'pipe', 'pipe'],
|
|
102
|
-
});
|
|
103
|
-
}
|
|
104
|
-
|
|
105
|
-
function readDockerState(record, probe = null) {
|
|
106
|
-
const opts = record.options || {};
|
|
107
|
-
if (opts.isolated !== 'docker' || !opts.sessionName) {
|
|
108
|
-
return null;
|
|
109
|
-
}
|
|
110
|
-
const state = inspectDockerState(opts.sessionName, probe);
|
|
111
|
-
if (
|
|
112
|
-
record.attempt &&
|
|
113
|
-
state?.startedAt &&
|
|
114
|
-
Date.parse(state.startedAt) < Date.parse(record.attempt.startedAt)
|
|
115
|
-
) {
|
|
116
|
-
return null;
|
|
117
|
-
}
|
|
118
|
-
return state;
|
|
119
|
-
}
|
|
120
|
-
|
|
121
|
-
/**
|
|
122
|
-
* Best-effort terminal exit code reported by the isolation backend itself
|
|
123
|
-
* (currently docker via `docker inspect .State.ExitCode`). Returns null when
|
|
124
|
-
* the backend cannot provide a real code, so callers never surface the `-1`
|
|
125
|
-
* sentinel for a session whose real exit code is simply not available yet.
|
|
126
|
-
* A running container has no terminal exit code (docker reports `0` for it),
|
|
127
|
-
* so only a stopped container contributes one.
|
|
128
|
-
* @param {{running: boolean, exitCode: number|null}|null} dockerState - Inspected state
|
|
129
|
-
* @returns {number|null}
|
|
130
|
-
*/
|
|
131
|
-
function backendExitCode(dockerState) {
|
|
132
|
-
return dockerState && !dockerState.running ? dockerState.exitCode : null;
|
|
133
|
-
}
|
|
134
|
-
|
|
135
|
-
/**
|
|
136
|
-
* Reconcile the OOM observation from the stored record and from `docker inspect`.
|
|
137
|
-
*
|
|
138
|
-
* `State.OOMKilled` is a container-cgroup flag: the kernel sets it when ANY
|
|
139
|
-
* process in the cgroup is OOM-killed and it is never cleared for the life of
|
|
140
|
-
* the container (moby/moby#47618). It is therefore an *observation*, never a
|
|
141
|
-
* verdict about the session (issue #151) — a container that lost one child
|
|
142
|
-
* process keeps running and can still exit `0`. Once observed, the flag stays
|
|
143
|
-
* `true` for the record.
|
|
144
|
-
* @param {Object} record - Execution record
|
|
145
|
-
* @param {{oomKilled: boolean|null}|null} dockerState - Inspected state
|
|
146
|
-
* @returns {boolean|undefined} Observation, or undefined when nothing is known
|
|
147
|
-
*/
|
|
148
|
-
function resolveOomObservation(record, dockerState) {
|
|
149
|
-
if (record.oomKilled === true || dockerState?.oomKilled === true) {
|
|
150
|
-
return true;
|
|
151
|
-
}
|
|
152
|
-
if (record.oomKilled === false || dockerState?.oomKilled === false) {
|
|
153
|
-
return false;
|
|
154
|
-
}
|
|
155
|
-
return undefined;
|
|
156
|
-
}
|
|
157
|
-
|
|
158
53
|
/**
|
|
159
54
|
* Check if a detached isolation session is still running
|
|
160
55
|
* @param {Object} record - Execution record
|
|
@@ -433,6 +328,7 @@ function resolveDetachedStatus(record, logTail, probe = null) {
|
|
|
433
328
|
(oomKilled === true ? 137 : -1);
|
|
434
329
|
}
|
|
435
330
|
applyEndTime(enriched, dockerState, footer.finishedAt);
|
|
331
|
+
addLostWatcherEvidence(enriched, record, dockerState, logTail);
|
|
436
332
|
}
|
|
437
333
|
|
|
438
334
|
return enriched;
|
|
@@ -563,10 +459,7 @@ function enrichDetachedStatus(record) {
|
|
|
563
459
|
const logTail = record
|
|
564
460
|
? readAttemptLogTail(record, FATAL_MARKER_TAIL_BYTES)
|
|
565
461
|
: null;
|
|
566
|
-
const probe =
|
|
567
|
-
dockerCommand: getDockerCommand(),
|
|
568
|
-
spawnOptions: dockerProbeSpawnOptions(),
|
|
569
|
-
};
|
|
462
|
+
const probe = dockerProbe();
|
|
570
463
|
const resolved = attachContainerRemoval(
|
|
571
464
|
resolveDetachedStatus(record, logTail, probe),
|
|
572
465
|
probe
|
|
@@ -920,10 +813,17 @@ function queryStatus(store, identifier, outputFormat) {
|
|
|
920
813
|
};
|
|
921
814
|
}
|
|
922
815
|
try {
|
|
923
|
-
// Enrich detached execution status with live session check
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
816
|
+
// Enrich detached execution status with live session check; its docker
|
|
817
|
+
// probes are answered by one inspect (issue #243).
|
|
818
|
+
const withProcessIds = withDockerInspectBatch(
|
|
819
|
+
[record],
|
|
820
|
+
dockerProbe(),
|
|
821
|
+
() => {
|
|
822
|
+
const enrichedRecord = enrichDetachedStatus(record);
|
|
823
|
+
persistContainerRemoval(store, record, enrichedRecord);
|
|
824
|
+
return attachProcessIds(enrichedRecord);
|
|
825
|
+
}
|
|
826
|
+
);
|
|
927
827
|
// Attach currentTime so callers can see how long an executing command has been running
|
|
928
828
|
const withCurrentTime = require('./execution-chain').attachIdentity(
|
|
929
829
|
attachCurrentTime(withProcessIds),
|
|
@@ -952,8 +852,12 @@ function listExecutions(store, outputFormat, options = {}) {
|
|
|
952
852
|
}
|
|
953
853
|
|
|
954
854
|
try {
|
|
855
|
+
const all = store.getAll();
|
|
856
|
+
// One docker inspect for every listed container (issue #243).
|
|
955
857
|
const enriched = sortRecordsByStartTimeDesc(
|
|
956
|
-
|
|
858
|
+
withDockerInspectBatch(all, dockerProbe(), () =>
|
|
859
|
+
all.map(prepareRecordForList)
|
|
860
|
+
)
|
|
957
861
|
);
|
|
958
862
|
// --running reports the reconciled status, not the stored one: a record
|
|
959
863
|
// whose session already died is listed as executed even if the store still
|