start-command 0.37.9 → 0.39.0

package/src/lib/command-builder.js

@@ -60,6 +60,7 @@ function buildNextLevelCommand(options, command) {
     for (const [key, flag] of [
       ['memory', '--memory'],
       ['memorySwap', '--memory-swap'],
+      ['memoryReservation', '--memory-reservation'],
       ['cpus', '--cpus'],
       ['onKillResumeMemory', '--on-kill-resume-memory'],
     ]) {
@@ -112,6 +113,9 @@ function buildNextLevelCommand(options, command) {
     if (options.privileged) {
       parts.push('--privileged');
     }
+    if (options.init) {
+      parts.push('--init');
+    }
   }
 
   // Pass through global flags

package/src/lib/container-removal.js

@@ -13,6 +13,7 @@
  */
 
 const { spawnSync } = require('child_process');
+const { prefetchedInspect } = require('./docker-inspect-batch');
 
 /** `containerState` of a record whose container no longer exists. */
 const CONTAINER_STATE_REMOVED = 'removed';
@@ -50,11 +51,9 @@ function isDockerNotFound(result) {
  *   the inspect failed; `notFound` says the failure was "no such container"
  */
 function dockerInspect(dockerCommand, spawnOptions, format, target) {
-  const result = spawnSync(
-    dockerCommand,
-    ['inspect', '-f', format, target],
-    spawnOptions
-  );
+  const result =
+    prefetchedInspect(format, target) ||
+    spawnSync(dockerCommand, ['inspect', '-f', format, target], spawnOptions);
   if (result.error || result.status !== 0 || !result.stdout) {
     return { stdout: null, notFound: isDockerNotFound(result) };
   }

package/src/lib/detached-finalize.js

@@ -170,6 +170,7 @@ function finalizeDetachedExecution(options = {}) {
   );
   record.options.exitEvidence = {
     daemonRestart: logged.daemonRestart,
+    ...evidence.sourceOf(logged),
     mainOom:
       !logged.daemonRestart &&
       (logged.mainOom ||

package/src/lib/docker-command-handoff.js

@@ -12,7 +12,9 @@ function buildCommandHandoffArgs(mainArgs, markerPath) {
   return [
     'sh',
     '-c',
-    'p=$1; shift; if [ -f "$p" ]; then exec sh "$p"; fi; exec "$@"',
+    // `$p.env` refreshes the START_* variables a `docker start` cannot
+    // change (issue #230) before either command runs.
+    'p=$1; shift; if [ -f "$p.env" ]; then . "$p.env"; fi; if [ -f "$p" ]; then exec sh "$p"; fi; exec "$@"',
     'start-command-handoff',
     markerPath,
     ...mainArgs,
@@ -49,27 +51,53 @@ function buildCommandHandoffScript(command, options = {}) {
   return `exec ${shellQuote(shell)}${interactive} -c ${shellQuote(effective)}\n`;
 }
 
-function writeCommandHandoff(containerName, command, runner, options = {}) {
+function copyIntoContainer(containerName, target, content, runner) {
   const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'start-handoff-'));
   const file = path.join(dir, 'command');
   try {
-    fs.writeFileSync(file, buildCommandHandoffScript(command, options), {
-      mode: 0o644,
-    });
+    fs.writeFileSync(file, content, { mode: 0o644 });
     fs.chmodSync(file, 0o644);
     return runner(require('./docker-cleanup').getDockerCommand(), [
       'cp',
       file,
-      `${containerName}:${commandHandoffPath(containerName)}`,
+      `${containerName}:${target}`,
     ]);
   } finally {
     fs.rmSync(dir, { recursive: true, force: true });
   }
 }
 
+function writeCommandHandoff(containerName, command, runner, options = {}) {
+  return copyIntoContainer(
+    containerName,
+    commandHandoffPath(containerName),
+    buildCommandHandoffScript(command, options),
+    runner
+  );
+}
+
+/**
+ * Copy the START_* variables of the next attempt into a stopped container
+ * that was created with the handoff selector (issue #230).
+ * @param {string} containerName - Container name
+ * @param {object} env - Variables from `buildExecutionEnv`
+ * @param {Function} runner - Command runner
+ * @returns {object} Runner result
+ */
+function writeExecutionEnvHandoff(containerName, env, runner) {
+  const statement = require('./execution-env').buildExecutionEnvExport(env);
+  return copyIntoContainer(
+    containerName,
+    `${commandHandoffPath(containerName)}.env`,
+    statement ? `${statement}\n` : '',
+    runner
+  );
+}
+
 module.exports = {
   commandHandoffPath,
   buildCommandHandoffArgs,
   buildCommandHandoffScript,
   writeCommandHandoff,
+  writeExecutionEnvHandoff,
 };

package/src/lib/docker-init-option.js

@@ -0,0 +1,65 @@
+/**
+ * Parse and validate the opt-in init process for docker isolation
+ * (issue #244):
+ *   --init      run `docker run --init`, so Docker's bundled tini is PID 1
+ *               and reaps orphaned processes
+ *   --no-init   keep the command as PID 1, overriding the environment default
+ *
+ * Without either flag, START_COMMAND_DOCKER_INIT=1 turns it on for every
+ * docker launch on the host. The default stays off: a command that is its own
+ * supervisor keeps today's behaviour, and recovery works the same for
+ * containers started with or without it.
+ */
+
+const DOCKER_INIT_ENV = 'START_COMMAND_DOCKER_INIT';
+
+/**
+ * Whether the environment asks for an init process by default.
+ * @param {object} [env] - Environment
+ * @returns {boolean}
+ */
+function dockerInitFromEnv(env = process.env) {
+  const value = String(env[DOCKER_INIT_ENV] || '')
+    .trim()
+    .toLowerCase();
+  return ['1', 'true', 'yes', 'on'].includes(value);
+}
+
+function parseDockerInitOption(args, index, options) {
+  const arg = args[index];
+  if (arg === '--init' || arg === '--no-init') {
+    options.init = arg === '--init';
+    return 1;
+  }
+  return 0;
+}
+
+/**
+ * Resolve `options.init` to a boolean: the flag if one was given (`null`
+ * means none), otherwise the environment default for launches whose stack
+ * includes docker. Idempotent, because `validateOptions` may run again on
+ * options it already resolved: `--no-init` (or a resolved `false`) without
+ * docker is the default anyway and is accepted.
+ * @param {object} options - Parsed options
+ * @param {object} [env] - Environment
+ * @throws {Error} If `--init` is given without docker isolation
+ */
+function validateDockerInitOption(options, env = process.env) {
+  const stack = options.isolatedStack || [options.isolated];
+  const hasDocker = options.isolated !== null && stack.includes('docker');
+  if (options.init === true && !hasDocker) {
+    throw new Error(
+      '--init option is only valid when isolation stack includes docker'
+    );
+  }
+  if (typeof options.init !== 'boolean') {
+    options.init = hasDocker && dockerInitFromEnv(env);
+  }
+}
+
+module.exports = {
+  DOCKER_INIT_ENV,
+  dockerInitFromEnv,
+  parse: parseDockerInitOption,
+  validate: validateDockerInitOption,
+};

package/src/lib/docker-inspect-batch.js

@@ -0,0 +1,176 @@
+/**
+ * One `docker inspect` for every container a status query probes (issue #243).
+ *
+ * `--list` used to start one or two `docker inspect` processes per record
+ * (the live state, the process IDs, the removal check), one after another,
+ * and `--status` up to three for its one record. While a query runs, this
+ * module asks Docker once for every field those probes read, and answers the
+ * per-record `docker inspect -f <format> <target>` calls from that answer.
+ *
+ * Docker prints one line per found container, in argument order, and one
+ * `No such container: <target>` line on stderr per missing one. Each line
+ * is matched back to its target by name or ID prefix (Docker's own lookup
+ * order), so the answer does not depend on the line order. A target the
+ * batch cannot account for (an old or stubbed docker, a daemon error) is
+ * not answered, and its per-record probe runs as before.
+ */
+
+const { spawnSync } = require('child_process');
+
+/** Fields of the batched inspect, in output order. */
+const BATCH_FIELDS = [
+  '.Name',
+  '.Id',
+  '.State.Pid',
+  '.State.Running',
+  '.State.ExitCode',
+  '.State.OOMKilled',
+  '.State.StartedAt',
+  '.State.FinishedAt',
+];
+const BATCH_FORMAT = BATCH_FIELDS.map((field) => `{{${field}}}`).join(' ');
+const NOT_FOUND_LINE = /no such (?:object|container):\s*(\S+)/gi;
+const PLACEHOLDER = /\{\{\s*(\.[A-Za-z.]+)\s*\}\}/g;
+
+/** Answers of the query in progress: target -> fields, or null (missing). */
+let prefetched = null;
+
+/**
+ * Containers the status probes of these records would inspect. Records
+ * whose container is already marked removed are skipped: they are never
+ * probed (issue #243).
+ * @param {Object[]} records - Execution records
+ * @returns {string[]} Unique names and IDs
+ */
+function dockerProbeTargets(records) {
+  const targets = new Set();
+  for (const record of records) {
+    const opts = (record && record.options) || {};
+    if (opts.isolated !== 'docker' || record.containerState === 'removed') {
+      continue;
+    }
+    for (const target of [opts.sessionName, opts.containerId]) {
+      if (typeof target === 'string' && target) {
+        targets.add(target);
+      }
+    }
+  }
+  return [...targets];
+}
+
+/**
+ * Match a batched inspect answer to its targets.
+ * @param {string[]} targets - Names and IDs passed to docker
+ * @param {string} stdout - One line of `BATCH_FORMAT` per found container
+ * @param {string} stderr - Docker's errors
+ * @returns {Map<string, Object|null>} Fields per answered target, null for
+ *   a target Docker reported missing; unanswered targets are absent
+ */
+function parseBatchInspect(targets, stdout, stderr) {
+  const rows = String(stdout || '')
+    .split('\n')
+    .map((line) => line.trim().split(/\s+/))
+    .filter((parts) => parts.length === BATCH_FIELDS.length)
+    .map((parts) =>
+      Object.fromEntries(BATCH_FIELDS.map((field, i) => [field, parts[i]]))
+    );
+  const missing = new Set(
+    [...String(stderr || '').matchAll(NOT_FOUND_LINE)].map((m) => m[1])
+  );
+  const answers = new Map();
+  for (const target of targets) {
+    // Docker's own "no such container" wins over a prefix of another
+    // target's ID.
+    const row = missing.has(target)
+      ? null
+      : rows.find((r) => r['.Name'] === `/${target}`) ||
+        rows.find((r) => r['.Id'].startsWith(target));
+    if (row !== undefined) {
+      answers.set(target, row);
+    }
+  }
+  return answers;
+}
+
+/**
+ * Run the batched inspect.
+ * @returns {Map<string, Object|null>} See `parseBatchInspect`
+ */
+function runBatchInspect(targets, dockerCommand, spawnOptions) {
+  if (targets.length === 0) {
+    return new Map();
+  }
+  const result = spawnSync(
+    dockerCommand,
+    ['inspect', '--type', 'container', '-f', BATCH_FORMAT, ...targets],
+    spawnOptions
+  );
+  if (result.error) {
+    return new Map();
+  }
+  return parseBatchInspect(targets, result.stdout, result.stderr);
+}
+
+/**
+ * Run `fn` with the docker probes of `records` answered by one inspect.
+ * Nested calls reuse the outer answer.
+ * @param {Object[]} records - Records the query will probe
+ * @param {{dockerCommand: string, spawnOptions: Object}} docker - How to run docker
+ * @param {function(): *} fn - The query
+ * @returns {*} What `fn` returns
+ */
+function withDockerInspectBatch(records, docker, fn) {
+  if (prefetched) {
+    return fn();
+  }
+  prefetched = runBatchInspect(
+    dockerProbeTargets(records),
+    docker.dockerCommand,
+    docker.spawnOptions
+  );
+  try {
+    return fn();
+  } finally {
+    prefetched = null;
+  }
+}
+
+/**
+ * Answer `docker inspect -f <format> <target>` from the batch.
+ * @param {string} format - Go template of the per-record probe
+ * @param {string} target - Container name or ID
+ * @returns {{success: boolean, status: number, stdout: string,
+ *   stderr: string}|null} A spawn-like result, or null when the batch has
+ *   no answer and the probe must run
+ */
+function prefetchedInspect(format, target) {
+  if (!prefetched || !prefetched.has(target)) {
+    return null;
+  }
+  const row = prefetched.get(target);
+  if (row === null) {
+    return {
+      success: false,
+      status: 1,
+      stdout: '',
+      stderr: `Error response from daemon: No such container: ${target}\n`,
+    };
+  }
+  let known = true;
+  const stdout = format.replace(PLACEHOLDER, (_, field) => {
+    known = known && field in row;
+    return row[field];
+  });
+  if (!known || stdout.includes('{{')) {
+    return null;
+  }
+  return { success: true, status: 0, stdout: `${stdout}\n`, stderr: '' };
+}
+
+module.exports = {
+  BATCH_FORMAT,
+  dockerProbeTargets,
+  parseBatchInspect,
+  prefetchedInspect,
+  withDockerInspectBatch,
+};

package/src/lib/docker-resource-options.js

@@ -4,6 +4,7 @@ const { normalizeResourceLimits } = require('./docker-resource-limits');
 const FIELDS = {
   '--memory': 'memory',
   '--memory-swap': 'memorySwap',
+  '--memory-reservation': 'memoryReservation',
   '--cpus': 'cpus',
   '--on-kill-resume-memory': 'onKillResumeMemory',
 };
@@ -95,7 +96,7 @@ function readDaemonCapacity(runner) {
 
 function resolveResourceOptions(options, runner, random = Math.random) {
   const specs = {};
-  for (const field of ['memory', 'memorySwap', 'cpus']) {
+  for (const field of ['memory', 'memorySwap', 'memoryReservation', 'cpus']) {
     if (options[field] !== null && options[field] !== undefined) {
       specs[field] = parseResourceSpec(options[field], field === 'cpus');
     }
@@ -137,8 +138,26 @@ function resolveResourceOptions(options, runner, random = Math.random) {
   ) {
     throw new Error('memory-swap must be at least memory');
   }
+  // Compare effective values so lowering a cap also checks a saved reservation.
+  const existingReservation = limits
+    .find((flag) => flag.startsWith('--memory-reservation='))
+    ?.slice('--memory-reservation='.length);
+  const reservation =
+    resolved.memoryReservation ??
+    (existingReservation
+      ? parseResourceSpec(existingReservation).amount
+      : null);
+  if (
+    memory !== null &&
+    memory !== undefined &&
+    reservation !== null &&
+    reservation !== undefined &&
+    reservation >= memory
+  ) {
+    throw new Error('memory-reservation must be less than memory');
+  }
   for (const [field, amount] of Object.entries(resolved)) {
-    const flag = field === 'memorySwap' ? '--memory-swap' : `--${field}`;
+    const flag = Object.keys(FIELDS).find((key) => FIELDS[key] === field);
     for (let i = limits.length - 1; i >= 0; i--) {
       if (
         limits[i].startsWith(`${flag}=`) ||

package/src/lib/docker-runtime-args.js

@@ -1,7 +1,8 @@
 /**
  * Docker runtime options shared by `docker run`, `[Isolation]` status lines
- * and execution-record metadata: privileged mode, environment, volumes,
- * mounts, networks (issue #150) and resource limits (issue #176).
+ * and execution-record metadata: privileged mode, the init process
+ * (issue #244), environment, volumes, mounts, networks (issue #150) and
+ * resource limits (issue #176).
  */
 
 const { getDockerNetworks } = require('./docker-network-lifecycle');
@@ -31,7 +32,7 @@ function getOnKillResume(options = {}) {
  * mounts, --mount specs, network configuration and resource limits. Returned
  * in a stable order so they can be spliced into the `docker run` argv before
  * the image name.
- * @param {object} options - Options (privileged, env, volumes, mounts, network, networkAliases, resourceLimits)
+ * @param {object} options - Options (privileged, init, env, volumes, mounts, network, networkAliases, resourceLimits)
  * @returns {string[]} Docker CLI arguments
  */
 function buildDockerRuntimeArgs(options = {}) {
@@ -39,9 +40,20 @@ function buildDockerRuntimeArgs(options = {}) {
   if (options.privileged) {
     args.push('--privileged');
   }
+  // tini as PID 1 reaps orphaned processes (issue #244).
+  if (options.init) {
+    args.push('--init');
+  }
   for (const envVar of options.env || []) {
     args.push('-e', envVar);
   }
+  // Execution identity (issue #230); last, so a user `--env` cannot spoof it.
+  const executionEnv = require('./execution-env');
+  for (const envVar of executionEnv.executionEnvAssignments(
+    executionEnv.executionEnvFromOptions(options)
+  )) {
+    args.push('-e', envVar);
+  }
   for (const label of require('./docker-labels').dockerLabels(options)) {
     args.push('--label', label);
   }
@@ -87,6 +99,9 @@ function buildDockerRuntimeStatusLines(options = {}) {
   if (options.privileged) {
     lines.push(`[Isolation] Privileged: true`);
   }
+  if (options.init) {
+    lines.push(`[Isolation] Init: true`);
+  }
   const networks = getDockerNetworks(options);
   if (networks.length > 0) {
     lines.push(`[Isolation] Network: ${networks[0]}`);
@@ -138,6 +153,7 @@ function buildDockerRuntimeMetadata(options = {}) {
     labels: options.labels && options.labels.length > 0 ? options.labels : null,
     commandHandoff: options.detached ? true : null,
     privileged: options.privileged || null,
+    init: options.init || null,
     network: networks[0] || null,
     networks: networks.length > 0 ? networks : null,
     networkAliases:

package/src/lib/execution-attempt.js

@@ -46,6 +46,21 @@ function readAttemptActivity(record) {
   return record;
 }
 
+/** Number of the record's current attempt: 1 until it is resumed. */
+function currentAttemptNumber(record) {
+  return (
+    record?.attempt?.number ||
+    (Number(record?.options?.resumeCount) || 0) +
+      (Number(record?.options?.recoveryAttempts) || 0) +
+      1
+  );
+}
+
+/** Number the next attempt of `record` gets (`START_ATTEMPT`, issue #230). */
+function nextAttemptNumber(record) {
+  return currentAttemptNumber(record) + 1;
+}
+
 function createAttempt(record, plan) {
   let logOffset = null;
   if (record.logPath) {
@@ -58,11 +73,7 @@ function createAttempt(record, plan) {
     }
   }
   return {
-    number:
-      (record.attempt?.number ||
-        (Number(record.options?.resumeCount) || 0) +
-          (Number(record.options?.recoveryAttempts) || 0) +
-          1) + 1,
+    number: nextAttemptNumber(record),
     startedAt: new Date().toISOString(),
     logOffset,
     lastOutputAt: null,
@@ -78,10 +89,7 @@ function archiveAttempt(record) {
   readAttemptActivity(record);
   const snapshot = {
     ...(record.attempt || {
-      number:
-        (Number(record.options?.resumeCount) || 0) +
-        (Number(record.options?.recoveryAttempts) || 0) +
-        1,
+      number: currentAttemptNumber(record),
       startedAt: record.options?.resumedAt || record.startTime,
       logOffset: null,
     }),
@@ -176,6 +184,8 @@ module.exports = {
   appendLifecycle,
   archiveAttempt,
   createAttempt,
+  currentAttemptNumber,
+  nextAttemptNumber,
   patchAttempt,
   readAttemptActivity,
   readAttemptLogTail,