start-command 0.38.0 → 0.39.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. package/CHANGELOG.md +20 -0
  2. package/README.md +4 -2
  3. package/bunfig.toml +4 -0
  4. package/eslint.config.mjs +1 -0
  5. package/package.json +4 -4
  6. package/src/lib/args-parser.js +9 -0
  7. package/src/lib/cgroup-memory.js +95 -23
  8. package/src/lib/command-builder.js +4 -0
  9. package/src/lib/container-removal.js +4 -5
  10. package/src/lib/detached-finalize.js +1 -0
  11. package/src/lib/docker-init-option.js +65 -0
  12. package/src/lib/docker-inspect-batch.js +176 -0
  13. package/src/lib/docker-resource-options.js +21 -2
  14. package/src/lib/docker-runtime-args.js +12 -3
  15. package/src/lib/execution-control.js +8 -6
  16. package/src/lib/execution-recovery.js +4 -1
  17. package/src/lib/execution-resume.js +4 -1
  18. package/src/lib/execution-store.js +19 -1
  19. package/src/lib/exit-evidence.js +139 -7
  20. package/src/lib/output-blocks.js +11 -0
  21. package/src/lib/resume-resources.js +2 -2
  22. package/src/lib/status-formatter.js +34 -130
  23. package/src/lib/status-probe.js +149 -0
  24. package/src/lib/store-decode-cache.js +141 -0
  25. package/src/lib/usage.js +3 -0
  26. package/test/cargo-lock-version.mjs +121 -0
  27. package/test/check-rust-coverage.mjs +71 -0
  28. package/test/check-web-archive.mjs +154 -0
  29. package/test/ci-workflow-invariants.js +204 -0
  30. package/test/command-builder-docker-cleanup.js +24 -0
  31. package/test/command-exit-codes.mjs +91 -0
  32. package/test/docker-runtime-options.js +2 -0
  33. package/test/docker-snapshot-safety.js +39 -0
  34. package/test/execution-resume.js +2 -0
  35. package/test/fixtures/fake-npm.sh +64 -0
  36. package/test/fixtures/lychee-report.md +39 -0
  37. package/test/javascript-first-ci.js +69 -0
  38. package/test/links-workflow.mjs +127 -0
  39. package/test/lint-gate.mjs +56 -0
  40. package/test/npm-publish.mjs +181 -0
  41. package/test/recheck-broken-links.mjs +452 -0
  42. package/test/regression-176.js +2 -0
  43. package/test/regression-187.js +2 -0
  44. package/test/regression-191.js +4 -0
  45. package/test/regression-193.js +2 -0
  46. package/test/regression-198.js +2 -0
  47. package/test/regression-207.js +8 -2
  48. package/test/regression-224.js +2 -0
  49. package/test/regression-236.js +250 -0
  50. package/test/regression-238.js +129 -0
  51. package/test/regression-241.js +254 -0
  52. package/test/regression-242.js +241 -0
  53. package/test/regression-243.js +493 -0
  54. package/test/regression-244.js +471 -0
  55. package/test/release-push-retry.mjs +172 -0
  56. package/test/run-js-tests.mjs +15 -3
  57. package/test/setup/private-snapshot-lock.js +17 -0
  58. package/test/shared-golden.js +29 -2
  59. package/test/transient-links.mjs +139 -0
@@ -1,7 +1,8 @@
1
1
  /**
2
2
  * Docker runtime options shared by `docker run`, `[Isolation]` status lines
3
- * and execution-record metadata: privileged mode, environment, volumes,
4
- * mounts, networks (issue #150) and resource limits (issue #176).
3
+ * and execution-record metadata: privileged mode, the init process
4
+ * (issue #244), environment, volumes, mounts, networks (issue #150) and
5
+ * resource limits (issue #176).
5
6
  */
6
7
 
7
8
  const { getDockerNetworks } = require('./docker-network-lifecycle');
@@ -31,7 +32,7 @@ function getOnKillResume(options = {}) {
31
32
  * mounts, --mount specs, network configuration and resource limits. Returned
32
33
  * in a stable order so they can be spliced into the `docker run` argv before
33
34
  * the image name.
34
- * @param {object} options - Options (privileged, env, volumes, mounts, network, networkAliases, resourceLimits)
35
+ * @param {object} options - Options (privileged, init, env, volumes, mounts, network, networkAliases, resourceLimits)
35
36
  * @returns {string[]} Docker CLI arguments
36
37
  */
37
38
  function buildDockerRuntimeArgs(options = {}) {
@@ -39,6 +40,10 @@ function buildDockerRuntimeArgs(options = {}) {
39
40
  if (options.privileged) {
40
41
  args.push('--privileged');
41
42
  }
43
+ // tini as PID 1 reaps orphaned processes (issue #244).
44
+ if (options.init) {
45
+ args.push('--init');
46
+ }
42
47
  for (const envVar of options.env || []) {
43
48
  args.push('-e', envVar);
44
49
  }
@@ -94,6 +99,9 @@ function buildDockerRuntimeStatusLines(options = {}) {
94
99
  if (options.privileged) {
95
100
  lines.push(`[Isolation] Privileged: true`);
96
101
  }
102
+ if (options.init) {
103
+ lines.push(`[Isolation] Init: true`);
104
+ }
97
105
  const networks = getDockerNetworks(options);
98
106
  if (networks.length > 0) {
99
107
  lines.push(`[Isolation] Network: ${networks[0]}`);
@@ -145,6 +153,7 @@ function buildDockerRuntimeMetadata(options = {}) {
145
153
  labels: options.labels && options.labels.length > 0 ? options.labels : null,
146
154
  commandHandoff: options.detached ? true : null,
147
155
  privileged: options.privileged || null,
156
+ init: options.init || null,
148
157
  network: networks[0] || null,
149
158
  networks: networks.length > 0 ? networks : null,
150
159
  networkAliases:
@@ -11,6 +11,7 @@ const {
11
11
  formatAsNestedLinksNotation,
12
12
  } = require('./output-blocks');
13
13
  const { getDockerCommand, getDockerSpawnOptions } = require('./docker-cleanup');
14
+ const { prefetchedInspect } = require('./docker-inspect-batch');
14
15
 
15
16
  const ControlAction = {
16
17
  STOP: 'stop',
@@ -148,14 +149,15 @@ function collectProcessIds(record, runner = runCommand) {
148
149
  ),
149
150
  ];
150
151
  addIfPresent(processIds, 'commandPids', commandPids);
152
+ } else if (isolated === 'docker' && record.containerState === 'removed') {
153
+ // The container is gone, so there is nothing to ask Docker (#243).
154
+ addIfPresent(processIds, 'containerId', opts.containerId);
151
155
  } else if (isolated === 'docker') {
152
156
  addIfPresent(processIds, 'containerId', opts.containerId);
153
- const result = runner(getDockerCommand(), [
154
- 'inspect',
155
- '-f',
156
- '{{.Id}} {{.State.Pid}}',
157
- sessionName,
158
- ]);
157
+ const format = '{{.Id}} {{.State.Pid}}';
158
+ const result =
159
+ prefetchedInspect(format, sessionName) ||
160
+ runner(getDockerCommand(), ['inspect', '-f', format, sessionName]);
159
161
  if (result.success && result.stdout.trim()) {
160
162
  const [containerId, pidValue] = result.stdout.trim().split(/\s+/);
161
163
  addIfPresent(processIds, 'containerId', containerId);
@@ -336,6 +336,7 @@ function recoverKilledExecution(params = {}) {
336
336
  record.options.exitEvidence = {
337
337
  mainOom: terminalOom,
338
338
  daemonRestart: logged.daemonRestart,
339
+ ...evidence.sourceOf(logged),
339
340
  };
340
341
  let recoveryLimits = null;
341
342
  if (opts.onKillResumeMemory && terminalOom) {
@@ -490,7 +491,9 @@ function recoverKilledExecution(params = {}) {
490
491
  const updated = runner(getDockerCommand(), [
491
492
  'update',
492
493
  ...resourceLimits.filter((s) =>
493
- /^--(?:memory|memory-swap|cpus|cpu-quota|cpu-period)=/.test(s)
494
+ /^--(?:memory|memory-swap|memory-reservation|cpus|cpu-quota|cpu-period)=/.test(
495
+ s
496
+ )
494
497
  ),
495
498
  containerName,
496
499
  ]);
@@ -127,6 +127,7 @@ function buildLaunchOptions(record) {
127
127
  useCommandStream: opts.useCommandStream || false,
128
128
  shell: opts.shell || 'auto',
129
129
  privileged: opts.privileged || false,
130
+ init: opts.init || false,
130
131
  env: opts.env || [],
131
132
  volumes: opts.volumes || [],
132
133
  mounts: opts.mounts || [],
@@ -535,7 +536,9 @@ async function resumeExecution(store, identifier, deps = {}) {
535
536
  args: [
536
537
  'update',
537
538
  ...resources.resourceLimits.filter((s) =>
538
- /^--(?:memory|memory-swap|cpus|cpu-quota|cpu-period)=/.test(s)
539
+ /^--(?:memory|memory-swap|memory-reservation|cpus|cpu-quota|cpu-period)=/.test(
540
+ s
541
+ )
539
542
  ),
540
543
  plan.sessionName,
541
544
  ],
@@ -25,6 +25,7 @@ const os = require('os');
25
25
  const { execFileSync, spawnSync } = require('child_process');
26
26
  const crypto = require('crypto');
27
27
  const { atomicWrite } = require('./atomic-write');
28
+ const { decodeStoreContent } = require('./store-decode-cache');
28
29
 
29
30
  // Synchronous wrapper using Bun's native ESM support
30
31
  // This works because Bun handles ESM/CJS interop
@@ -38,6 +39,12 @@ function encodeSync(data) {
38
39
  }
39
40
  }
40
41
 
42
+ const DECODE_LOG = {
43
+ memory: () => 'Reused the decoded store: the file is unchanged',
44
+ copy: () => 'Read the decoded copy of the store: the file is unchanged',
45
+ decode: (content) => `Decoded the store (${content.length} characters)`,
46
+ };
47
+
41
48
  function decodeSync(notation) {
42
49
  try {
43
50
  const codec = require('lino-objects-codec');
@@ -52,6 +59,8 @@ const DEFAULT_APP_FOLDER = path.join(os.homedir(), '.start-command');
52
59
  const LINO_DB_FILE = 'executions.lino';
53
60
  const LINKS_DB_FILE = 'executions.links';
54
61
  const LOCK_FILE = 'executions.lock';
62
+ // Decoded records as JSON, for the next process (issue #243)
63
+ const DECODED_COPY_FILE = 'executions.decoded.js.json';
55
64
  const { LockManager } = require('./store-lock');
56
65
 
57
66
  /**
@@ -262,6 +271,7 @@ class ExecutionStore {
262
271
  this.linoDbPath = path.join(this.appFolder, LINO_DB_FILE);
263
272
  this.linksDbPath = path.join(this.appFolder, LINKS_DB_FILE);
264
273
  this.lockFilePath = path.join(this.appFolder, LOCK_FILE);
274
+ this.decodedCopyPath = path.join(this.appFolder, DECODED_COPY_FILE);
265
275
  this.useLinks = options.useLinks !== false && isClinkInstalled();
266
276
  this.verbose = options.verbose || false;
267
277
 
@@ -302,7 +312,14 @@ class ExecutionStore {
302
312
  return [];
303
313
  }
304
314
 
305
- const data = decodeSync(content);
315
+ // Decoded once per content, not once per call (issue #243).
316
+ const { data, source } = decodeStoreContent(
317
+ this.linoDbPath,
318
+ content,
319
+ decodeSync,
320
+ { copyPath: this.decodedCopyPath }
321
+ );
322
+ this.log(DECODE_LOG[source](content));
306
323
  if (!Array.isArray(data)) {
307
324
  return [];
308
325
  }
@@ -915,4 +932,5 @@ module.exports = {
915
932
  LINO_DB_FILE,
916
933
  LINKS_DB_FILE,
917
934
  LOCK_FILE,
935
+ DECODED_COPY_FILE,
918
936
  };
@@ -5,6 +5,16 @@ const MAIN_OOM = 'Exit evidence: main-oom (recent cgroup oom_kill delta)';
5
5
  const DOCKER_OOM = 'Exit evidence: main-oom (recent Docker oom event)';
6
6
  const DAEMON_RESTART =
7
7
  'Exit evidence: docker-daemon-restart (Docker service journal)';
8
+ const DAEMON_RESTART_API =
9
+ 'Exit evidence: docker-daemon-restart (Docker API: containers finished together)';
10
+ const DAEMON_RESTART_PREFIX = 'Exit evidence: docker-daemon-restart';
11
+ // dockerd gives every container its stop timeout and kills the rest within its
12
+ // 15 s shutdown timeout, so the containers of one restart finish this close.
13
+ const DAEMON_RESTART_WINDOW_MS = 15000;
14
+ const CONTAINER_STATE_FORMAT =
15
+ '{{.Id}} {{.State.FinishedAt}} {{.State.StartedAt}} {{.State.ExitCode}} {{.State.OOMKilled}}';
16
+ // Bounds one `docker inspect` call on hosts with many stopped containers.
17
+ const MAX_PEER_CONTAINERS = 256;
8
18
 
9
19
  function recentOomDelta(sample, finishedAt) {
10
20
  const fields = String(sample || '')
@@ -39,6 +49,42 @@ function daemonRestartEvidence(journal, containerId) {
39
49
  return Boolean(exited && forced);
40
50
  }
41
51
 
52
+ /**
53
+ * Socket-only daemon-restart evidence (issue #241): another container that
54
+ * finished within the same shutdown window, was not OOM-killed, and was either
55
+ * stopped by a signal (137/143) or started again afterwards (restart policy;
56
+ * moby resets its ExitCode to 0 but keeps FinishedAt). `docker events` cannot
57
+ * be used: its buffer does not survive the restart, while State does.
58
+ * @param {string} states - `CONTAINER_STATE_FORMAT` lines, one per container
59
+ */
60
+ function daemonRestartApiEvidence(states, containerId, finishedAt) {
61
+ const finished = Date.parse(finishedAt);
62
+ return (
63
+ Number.isFinite(finished) &&
64
+ String(states || '')
65
+ .split('\n')
66
+ .some((line) => {
67
+ const [id, peerFinished, peerStarted, exit, oom] = line
68
+ .trim()
69
+ .split(/\s+/);
70
+ const end = Date.parse(peerFinished);
71
+ if (
72
+ !/^[a-f0-9]{64}$/.test(id || '') ||
73
+ id === containerId ||
74
+ oom !== 'false' ||
75
+ !Number.isFinite(end) ||
76
+ end <= 0 ||
77
+ Math.abs(end - finished) > DAEMON_RESTART_WINDOW_MS
78
+ ) {
79
+ return false;
80
+ }
81
+ return (
82
+ exit === '137' || exit === '143' || Date.parse(peerStarted) > end
83
+ );
84
+ })
85
+ );
86
+ }
87
+
42
88
  function recentDockerOom(events, finishedAt) {
43
89
  const finished = Date.parse(finishedAt) / 1000;
44
90
  return (
@@ -68,6 +114,10 @@ function collectExitEvidence(
68
114
  // A local journal cannot diagnose a remote daemon. Require a local socket.
69
115
  const local =
70
116
  !process.env.DOCKER_HOST || process.env.DOCKER_HOST.startsWith('unix://');
117
+ const sampledOom =
118
+ (oomKilled === true || oomKilled === 'true') &&
119
+ (Number(exitCode) === 137 || Number(exitCode) < 0) &&
120
+ recentOomDelta(sample, finishedAt);
71
121
  if (
72
122
  Number(exitCode) === 137 &&
73
123
  containerName &&
@@ -114,18 +164,35 @@ function collectExitEvidence(
114
164
  if (events.success && recentDockerOom(events.stdout, finishedAt)) {
115
165
  return DOCKER_OOM;
116
166
  }
167
+ // Without a host journal (a mounted docker.sock, a remote daemon) the
168
+ // only record of the restart is the state of the other containers.
169
+ const peers = sampledOom
170
+ ? { success: false, stdout: '' }
171
+ : run(docker, ['ps', '-a', '-q', '--no-trunc']);
172
+ const ids = peers.stdout
173
+ .split(/\s+/)
174
+ .filter(
175
+ (peer) => /^[a-f0-9]{64}$/.test(peer) && peer !== id.stdout.trim()
176
+ )
177
+ .slice(0, MAX_PEER_CONTAINERS);
178
+ // A container removed after `docker ps` fails inspect with status 1 but
179
+ // the others are still printed, so the output is used either way.
180
+ const states = ids.length
181
+ ? run(docker, ['inspect', '-f', CONTAINER_STATE_FORMAT, ...ids])
182
+ : { success: false, stdout: '' };
183
+ if (
184
+ daemonRestartApiEvidence(states.stdout, id.stdout.trim(), finishedAt)
185
+ ) {
186
+ return DAEMON_RESTART_API;
187
+ }
117
188
  if (process.env.START_DEBUG === '1') {
118
189
  console.error(
119
- `[DEBUG] Exit evidence: container=${id.stdout.trim()} finishedAt=${finishedAt} eventsAvailable=${events.success} oomEvents=${JSON.stringify(events.stdout)} sample=${JSON.stringify(sample || '')}`
190
+ `[DEBUG] Exit evidence: container=${id.stdout.trim()} finishedAt=${finishedAt} eventsAvailable=${events.success} oomEvents=${JSON.stringify(events.stdout)} peerContainers=${ids.length} peerStates=${JSON.stringify(states.stdout)} sample=${JSON.stringify(sample || '')}`
120
191
  );
121
192
  }
122
193
  }
123
194
  }
124
- if (
125
- (oomKilled === true || oomKilled === 'true') &&
126
- (Number(exitCode) === 137 || Number(exitCode) < 0) &&
127
- recentOomDelta(sample, finishedAt)
128
- ) {
195
+ if (sampledOom) {
129
196
  return MAIN_OOM;
130
197
  }
131
198
  return 'Exit evidence: unavailable (no attributed exit-time OOM or daemon restart evidence)';
@@ -134,12 +201,72 @@ function collectExitEvidence(
134
201
  function fromLog(log) {
135
202
  const lines = String(log || '').split('\n');
136
203
  const last = lines.filter((line) => line.startsWith('Exit evidence:')).at(-1);
204
+ const daemonRestart = Boolean(last?.startsWith(DAEMON_RESTART_PREFIX));
137
205
  return {
138
206
  mainOom: last === MAIN_OOM || last === DOCKER_OOM,
139
- daemonRestart: last === DAEMON_RESTART,
207
+ daemonRestart,
208
+ // Which source attributed the restart: the host journal or the API.
209
+ ...(daemonRestart
210
+ ? { daemonRestartSource: last === DAEMON_RESTART_API ? 'api' : 'journal' }
211
+ : {}),
140
212
  };
141
213
  }
142
214
 
215
+ /** The stored `daemonRestartSource`, kept only when a restart was attributed. */
216
+ function sourceOf(logged) {
217
+ return logged?.daemonRestart && logged.daemonRestartSource
218
+ ? { daemonRestartSource: logged.daemonRestartSource }
219
+ : {};
220
+ }
221
+
222
+ // A watcher that shares the daemon with its task (a mounted docker.sock) dies
223
+ // in the same restart and never logs evidence, so `--status` collects it from
224
+ // the container state that survived, once per process (issue #241).
225
+ const lostWatcherEvidence = new Map();
226
+ function collectLostWatcherEvidence(record, state, runner) {
227
+ const name = record?.options?.sessionName;
228
+ if (
229
+ !name ||
230
+ record.options.exitEvidence ||
231
+ state?.running !== false ||
232
+ state.exitCode !== 137 ||
233
+ !state.finishedAt
234
+ ) {
235
+ return null;
236
+ }
237
+ const key = `${name}\0${state.finishedAt}`;
238
+ if (!lostWatcherEvidence.has(key)) {
239
+ const logged = fromLog(
240
+ collectExitEvidence(
241
+ {
242
+ containerName: name,
243
+ finishedAt: state.finishedAt,
244
+ exitCode: state.exitCode,
245
+ oomKilled: state.oomKilled,
246
+ },
247
+ runner
248
+ )
249
+ );
250
+ lostWatcherEvidence.set(key, {
251
+ mainOom: logged.mainOom,
252
+ daemonRestart: logged.daemonRestart,
253
+ ...sourceOf(logged),
254
+ });
255
+ }
256
+ return { ...lostWatcherEvidence.get(key) };
257
+ }
258
+
259
+ /** Stores the lost watcher's evidence when the log tail carries none. */
260
+ function addLostWatcherEvidence(enriched, record, state, logTail) {
261
+ const lost =
262
+ state && !/^Exit evidence:/m.test(logTail || '')
263
+ ? collectLostWatcherEvidence(record, state)
264
+ : null;
265
+ if (lost) {
266
+ enriched.options = { ...enriched.options, exitEvidence: lost };
267
+ }
268
+ }
269
+
143
270
  function buildExitEvidenceSnippet(containerName, quotedLogPath) {
144
271
  const stderr = process.env.START_DEBUG === '1' ? '2>&1' : '2>/dev/null';
145
272
  return `${shellQuote(process.execPath)} ${shellQuote(__filename)} ${shellQuote(containerName)} "$__start_command_cgroup" "$__start_command_finished" "$__start_command_exit" "$__start_command_oom" >> ${quotedLogPath} ${stderr}`;
@@ -149,11 +276,16 @@ module.exports = {
149
276
  MAIN_OOM,
150
277
  DOCKER_OOM,
151
278
  DAEMON_RESTART,
279
+ DAEMON_RESTART_API,
152
280
  recentOomDelta,
281
+ daemonRestartApiEvidence,
153
282
  recentDockerOom,
154
283
  daemonRestartEvidence,
155
284
  collectExitEvidence,
156
285
  fromLog,
286
+ sourceOf,
287
+ collectLostWatcherEvidence,
288
+ addLostWatcherEvidence,
157
289
  buildExitEvidenceSnippet,
158
290
  };
159
291
  if (require.main === module) {
@@ -119,6 +119,7 @@ function parseIsolationMetadata(extraLines) {
119
119
  session: null,
120
120
  endpoint: null,
121
121
  user: null,
122
+ init: false,
122
123
  };
123
124
 
124
125
  for (const line of extraLines) {
@@ -146,6 +147,12 @@ function parseIsolationMetadata(extraLines) {
146
147
  continue;
147
148
  }
148
149
 
150
+ // Parse [Isolation] Init: true (docker run --init, issue #244)
151
+ if (line === '[Isolation] Init: true') {
152
+ metadata.init = true;
153
+ continue;
154
+ }
155
+
149
156
  // Parse [Isolation] Endpoint: user@host
150
157
  const endpointMatch = line.match(/\[Isolation\] Endpoint: (.+)/);
151
158
  if (endpointMatch) {
@@ -207,6 +214,10 @@ function generateIsolationLines(metadata, containerOrScreenName = null) {
207
214
  }
208
215
  }
209
216
 
217
+ if (metadata.init) {
218
+ lines.push(createTimelineLine('init', 'true'));
219
+ }
220
+
210
221
  if (metadata.user) {
211
222
  lines.push(createTimelineLine('user', metadata.user));
212
223
  }
@@ -34,7 +34,7 @@ function prepareResumeResources(record, probe, overrides, runner, random) {
34
34
  const options = record.options || {};
35
35
  if (options.isolated !== 'docker') {
36
36
  if (
37
- ['memory', 'memorySwap', 'cpus'].some(
37
+ ['memory', 'memorySwap', 'memoryReservation', 'cpus'].some(
38
38
  (k) => overrides[k] !== null && overrides[k] !== undefined
39
39
  )
40
40
  ) {
@@ -81,7 +81,7 @@ function prepareResumeResources(record, probe, overrides, runner, random) {
81
81
  };
82
82
  result.update =
83
83
  probe.state === 'stopped' &&
84
- (['memory', 'memorySwap', 'cpus'].some(
84
+ (['memory', 'memorySwap', 'memoryReservation', 'cpus'].some(
85
85
  (k) => overrides[k] !== null && overrides[k] !== undefined
86
86
  ) ||
87
87
  Boolean(options.cpuPenaltyConfig));
@@ -14,7 +14,7 @@ const {
14
14
  formatTextOptionValue,
15
15
  } = require('./status-links-notation');
16
16
  const { collectProcessIds } = require('./execution-control');
17
- const { getDockerCommand, getDockerSpawnOptions } = require('./docker-cleanup');
17
+ const { getDockerCommand } = require('./docker-cleanup');
18
18
  const {
19
19
  describeExitCode,
20
20
  CGROUP_OOM_EXIT_REASON,
@@ -25,136 +25,31 @@ const {
25
25
  FATAL_MARKER_TAIL_BYTES,
26
26
  readLogTail,
27
27
  } = require('./isolation-log-utils');
28
- const { normalizeDockerTimestamp } = require('./docker-post-mortem');
29
28
  const { END_TIME_SOURCE } = require('./detached-finalize');
30
29
  const { formatCgroupMemory } = require('./cgroup-memory');
30
+ const { addLostWatcherEvidence } = require('./exit-evidence');
31
31
  const {
32
32
  attachContainerRemoval,
33
- dockerInspect,
34
33
  persistContainerRemoval,
35
34
  } = require('./container-removal');
36
-
37
- /**
38
- * Inspect the live state of a detached docker container by name.
39
- *
40
- * Distinguishes three outcomes that matter for status reporting:
41
- * - the container exists and is running,
42
- * - the container exists but has stopped (with a real exit code), and
43
- * - the container cannot be inspected at all.
44
- *
45
- * The last case is crucial on slow Docker-in-Docker hosts (issue #136): right
46
- * after `docker run -d` returns, `docker inspect <name>` can transiently fail
47
- * because the container is not visible yet. A failed inspect must NOT be read
48
- * as "stopped"; it means "unknown", so callers can keep the session running
49
- * instead of fabricating a terminal `-1` result.
50
- *
51
- * @param {string} sessionName - Container name
52
- * @param {Object|null} [probe] - Receives `sessionNotFound` (issue #225)
53
- * @returns {{running: boolean, exitCode: number|null, oomKilled: boolean|null}|null} State,
54
- * or null when the container cannot be inspected (not found yet, removed, or docker error)
55
- */
56
- function inspectDockerState(sessionName, probe = null) {
57
- const { stdout, notFound } = dockerInspect(
58
- getDockerCommand(),
59
- dockerProbeSpawnOptions(),
60
- '{{.State.Running}} {{.State.ExitCode}} {{.State.OOMKilled}} ' +
61
- '{{.State.StartedAt}} {{.State.FinishedAt}}',
62
- sessionName
63
- );
64
- if (probe) {
65
- // "No such container" is a fact, unlike a docker that cannot be reached
66
- // (issue #225); it still reads as unknown liveness here (issue #136).
67
- probe.sessionNotFound = notFound;
68
- }
69
- if (stdout === null) {
70
- return null;
71
- }
72
- // Parsed positionally and defensively: an older docker (or a stubbed one)
73
- // may answer with fewer fields than the template asks for.
74
- const [runningRaw, exitRaw, oomKilledRaw, startedRaw, finishedRaw] = stdout
75
- .trim()
76
- .split(/\s+/);
77
- const exitCode = Number.parseInt(exitRaw, 10);
35
+ const { withDockerInspectBatch } = require('./docker-inspect-batch');
36
+ const {
37
+ backendExitCode,
38
+ dockerProbeSpawnOptions,
39
+ inspectDockerState,
40
+ isDetachedDockerRecord,
41
+ readDockerState,
42
+ resolveOomObservation,
43
+ } = require('./status-probe');
44
+
45
+ /** How the status probes run docker. */
46
+ function dockerProbe() {
78
47
  return {
79
- running: runningRaw === 'true',
80
- exitCode: Number.isFinite(exitCode) ? exitCode : null,
81
- oomKilled:
82
- oomKilledRaw === 'true' ? true : oomKilledRaw === 'false' ? false : null,
83
- startedAt: normalizeDockerTimestamp(startedRaw),
84
- finishedAt: normalizeDockerTimestamp(finishedRaw),
48
+ dockerCommand: getDockerCommand(),
49
+ spawnOptions: dockerProbeSpawnOptions(),
85
50
  };
86
51
  }
87
52
 
88
- function isDetachedDockerRecord(record) {
89
- const opts = record.options || {};
90
- return (
91
- opts.isolated === 'docker' &&
92
- opts.isolationMode === 'detached' &&
93
- Boolean(opts.sessionName)
94
- );
95
- }
96
-
97
- function dockerProbeSpawnOptions() {
98
- return getDockerSpawnOptions({
99
- encoding: 'utf8',
100
- env: process.env,
101
- stdio: ['pipe', 'pipe', 'pipe'],
102
- });
103
- }
104
-
105
- function readDockerState(record, probe = null) {
106
- const opts = record.options || {};
107
- if (opts.isolated !== 'docker' || !opts.sessionName) {
108
- return null;
109
- }
110
- const state = inspectDockerState(opts.sessionName, probe);
111
- if (
112
- record.attempt &&
113
- state?.startedAt &&
114
- Date.parse(state.startedAt) < Date.parse(record.attempt.startedAt)
115
- ) {
116
- return null;
117
- }
118
- return state;
119
- }
120
-
121
- /**
122
- * Best-effort terminal exit code reported by the isolation backend itself
123
- * (currently docker via `docker inspect .State.ExitCode`). Returns null when
124
- * the backend cannot provide a real code, so callers never surface the `-1`
125
- * sentinel for a session whose real exit code is simply not available yet.
126
- * A running container has no terminal exit code (docker reports `0` for it),
127
- * so only a stopped container contributes one.
128
- * @param {{running: boolean, exitCode: number|null}|null} dockerState - Inspected state
129
- * @returns {number|null}
130
- */
131
- function backendExitCode(dockerState) {
132
- return dockerState && !dockerState.running ? dockerState.exitCode : null;
133
- }
134
-
135
- /**
136
- * Reconcile the OOM observation from the stored record and from `docker inspect`.
137
- *
138
- * `State.OOMKilled` is a container-cgroup flag: the kernel sets it when ANY
139
- * process in the cgroup is OOM-killed and it is never cleared for the life of
140
- * the container (moby/moby#47618). It is therefore an *observation*, never a
141
- * verdict about the session (issue #151) — a container that lost one child
142
- * process keeps running and can still exit `0`. Once observed, the flag stays
143
- * `true` for the record.
144
- * @param {Object} record - Execution record
145
- * @param {{oomKilled: boolean|null}|null} dockerState - Inspected state
146
- * @returns {boolean|undefined} Observation, or undefined when nothing is known
147
- */
148
- function resolveOomObservation(record, dockerState) {
149
- if (record.oomKilled === true || dockerState?.oomKilled === true) {
150
- return true;
151
- }
152
- if (record.oomKilled === false || dockerState?.oomKilled === false) {
153
- return false;
154
- }
155
- return undefined;
156
- }
157
-
158
53
  /**
159
54
  * Check if a detached isolation session is still running
160
55
  * @param {Object} record - Execution record
@@ -433,6 +328,7 @@ function resolveDetachedStatus(record, logTail, probe = null) {
433
328
  (oomKilled === true ? 137 : -1);
434
329
  }
435
330
  applyEndTime(enriched, dockerState, footer.finishedAt);
331
+ addLostWatcherEvidence(enriched, record, dockerState, logTail);
436
332
  }
437
333
 
438
334
  return enriched;
@@ -563,10 +459,7 @@ function enrichDetachedStatus(record) {
563
459
  const logTail = record
564
460
  ? readAttemptLogTail(record, FATAL_MARKER_TAIL_BYTES)
565
461
  : null;
566
- const probe = {
567
- dockerCommand: getDockerCommand(),
568
- spawnOptions: dockerProbeSpawnOptions(),
569
- };
462
+ const probe = dockerProbe();
570
463
  const resolved = attachContainerRemoval(
571
464
  resolveDetachedStatus(record, logTail, probe),
572
465
  probe
@@ -920,10 +813,17 @@ function queryStatus(store, identifier, outputFormat) {
920
813
  };
921
814
  }
922
815
  try {
923
- // Enrich detached execution status with live session check
924
- const enrichedRecord = enrichDetachedStatus(record);
925
- persistContainerRemoval(store, record, enrichedRecord);
926
- const withProcessIds = attachProcessIds(enrichedRecord);
816
+ // Enrich detached execution status with live session check; its docker
817
+ // probes are answered by one inspect (issue #243).
818
+ const withProcessIds = withDockerInspectBatch(
819
+ [record],
820
+ dockerProbe(),
821
+ () => {
822
+ const enrichedRecord = enrichDetachedStatus(record);
823
+ persistContainerRemoval(store, record, enrichedRecord);
824
+ return attachProcessIds(enrichedRecord);
825
+ }
826
+ );
927
827
  // Attach currentTime so callers can see how long an executing command has been running
928
828
  const withCurrentTime = require('./execution-chain').attachIdentity(
929
829
  attachCurrentTime(withProcessIds),
@@ -952,8 +852,12 @@ function listExecutions(store, outputFormat, options = {}) {
952
852
  }
953
853
 
954
854
  try {
855
+ const all = store.getAll();
856
+ // One docker inspect for every listed container (issue #243).
955
857
  const enriched = sortRecordsByStartTimeDesc(
956
- store.getAll().map(prepareRecordForList)
858
+ withDockerInspectBatch(all, dockerProbe(), () =>
859
+ all.map(prepareRecordForList)
860
+ )
957
861
  );
958
862
  // --running reports the reconciled status, not the stored one: a record
959
863
  // whose session already died is listed as executed even if the store still