start-command 0.38.0 → 0.39.1

package/src/lib/docker-resource-options.js

@@ -4,6 +4,7 @@ const { normalizeResourceLimits } = require('./docker-resource-limits');
 const FIELDS = {
   '--memory': 'memory',
   '--memory-swap': 'memorySwap',
+  '--memory-reservation': 'memoryReservation',
   '--cpus': 'cpus',
   '--on-kill-resume-memory': 'onKillResumeMemory',
 };
@@ -95,7 +96,7 @@ function readDaemonCapacity(runner) {
 
 function resolveResourceOptions(options, runner, random = Math.random) {
   const specs = {};
-  for (const field of ['memory', 'memorySwap', 'cpus']) {
+  for (const field of ['memory', 'memorySwap', 'memoryReservation', 'cpus']) {
     if (options[field] !== null && options[field] !== undefined) {
       specs[field] = parseResourceSpec(options[field], field === 'cpus');
     }
@@ -137,8 +138,26 @@ function resolveResourceOptions(options, runner, random = Math.random) {
   ) {
     throw new Error('memory-swap must be at least memory');
   }
+  // Compare effective values so lowering a cap also checks a saved reservation.
+  const existingReservation = limits
+    .find((flag) => flag.startsWith('--memory-reservation='))
+    ?.slice('--memory-reservation='.length);
+  const reservation =
+    resolved.memoryReservation ??
+    (existingReservation
+      ? parseResourceSpec(existingReservation).amount
+      : null);
+  if (
+    memory !== null &&
+    memory !== undefined &&
+    reservation !== null &&
+    reservation !== undefined &&
+    reservation >= memory
+  ) {
+    throw new Error('memory-reservation must be less than memory');
+  }
   for (const [field, amount] of Object.entries(resolved)) {
-    const flag = field === 'memorySwap' ? '--memory-swap' : `--${field}`;
+    const flag = Object.keys(FIELDS).find((key) => FIELDS[key] === field);
     for (let i = limits.length - 1; i >= 0; i--) {
       if (
         limits[i].startsWith(`${flag}=`) ||

package/src/lib/docker-runtime-args.js

@@ -1,7 +1,8 @@
 /**
  * Docker runtime options shared by `docker run`, `[Isolation]` status lines
- * and execution-record metadata: privileged mode, environment, volumes,
- * mounts, networks (issue #150) and resource limits (issue #176).
+ * and execution-record metadata: privileged mode, the init process
+ * (issue #244), environment, volumes, mounts, networks (issue #150) and
+ * resource limits (issue #176).
  */
 
 const { getDockerNetworks } = require('./docker-network-lifecycle');
@@ -31,7 +32,7 @@ function getOnKillResume(options = {}) {
  * mounts, --mount specs, network configuration and resource limits. Returned
  * in a stable order so they can be spliced into the `docker run` argv before
  * the image name.
- * @param {object} options - Options (privileged, env, volumes, mounts, network, networkAliases, resourceLimits)
+ * @param {object} options - Options (privileged, init, env, volumes, mounts, network, networkAliases, resourceLimits)
  * @returns {string[]} Docker CLI arguments
  */
 function buildDockerRuntimeArgs(options = {}) {
@@ -39,6 +40,10 @@ function buildDockerRuntimeArgs(options = {}) {
   if (options.privileged) {
     args.push('--privileged');
   }
+  // tini as PID 1 reaps orphaned processes (issue #244).
+  if (options.init) {
+    args.push('--init');
+  }
   for (const envVar of options.env || []) {
     args.push('-e', envVar);
   }
@@ -94,6 +99,9 @@ function buildDockerRuntimeStatusLines(options = {}) {
   if (options.privileged) {
     lines.push(`[Isolation] Privileged: true`);
   }
+  if (options.init) {
+    lines.push(`[Isolation] Init: true`);
+  }
   const networks = getDockerNetworks(options);
   if (networks.length > 0) {
     lines.push(`[Isolation] Network: ${networks[0]}`);
@@ -145,6 +153,7 @@ function buildDockerRuntimeMetadata(options = {}) {
     labels: options.labels && options.labels.length > 0 ? options.labels : null,
     commandHandoff: options.detached ? true : null,
     privileged: options.privileged || null,
+    init: options.init || null,
     network: networks[0] || null,
     networks: networks.length > 0 ? networks : null,
     networkAliases:

package/src/lib/execution-control.js

@@ -11,6 +11,7 @@ const {
   formatAsNestedLinksNotation,
 } = require('./output-blocks');
 const { getDockerCommand, getDockerSpawnOptions } = require('./docker-cleanup');
+const { prefetchedInspect } = require('./docker-inspect-batch');
 
 const ControlAction = {
   STOP: 'stop',
@@ -148,14 +149,15 @@ function collectProcessIds(record, runner = runCommand) {
       ),
     ];
     addIfPresent(processIds, 'commandPids', commandPids);
+  } else if (isolated === 'docker' && record.containerState === 'removed') {
+    // The container is gone, so there is nothing to ask Docker (#243).
+    addIfPresent(processIds, 'containerId', opts.containerId);
   } else if (isolated === 'docker') {
     addIfPresent(processIds, 'containerId', opts.containerId);
-    const result = runner(getDockerCommand(), [
-      'inspect',
-      '-f',
-      '{{.Id}} {{.State.Pid}}',
-      sessionName,
-    ]);
+    const format = '{{.Id}} {{.State.Pid}}';
+    const result =
+      prefetchedInspect(format, sessionName) ||
+      runner(getDockerCommand(), ['inspect', '-f', format, sessionName]);
     if (result.success && result.stdout.trim()) {
       const [containerId, pidValue] = result.stdout.trim().split(/\s+/);
       addIfPresent(processIds, 'containerId', containerId);

package/src/lib/execution-recovery.js

@@ -336,6 +336,7 @@ function recoverKilledExecution(params = {}) {
   record.options.exitEvidence = {
     mainOom: terminalOom,
     daemonRestart: logged.daemonRestart,
+    ...evidence.sourceOf(logged),
   };
   let recoveryLimits = null;
   if (opts.onKillResumeMemory && terminalOom) {
@@ -490,7 +491,9 @@ function recoverKilledExecution(params = {}) {
     const updated = runner(getDockerCommand(), [
       'update',
       ...resourceLimits.filter((s) =>
-        /^--(?:memory|memory-swap|cpus|cpu-quota|cpu-period)=/.test(s)
+        /^--(?:memory|memory-swap|memory-reservation|cpus|cpu-quota|cpu-period)=/.test(
+          s
+        )
       ),
       containerName,
     ]);

package/src/lib/execution-resume.js

@@ -127,6 +127,7 @@ function buildLaunchOptions(record) {
     useCommandStream: opts.useCommandStream || false,
     shell: opts.shell || 'auto',
     privileged: opts.privileged || false,
+    init: opts.init || false,
     env: opts.env || [],
     volumes: opts.volumes || [],
     mounts: opts.mounts || [],
@@ -535,7 +536,9 @@ async function resumeExecution(store, identifier, deps = {}) {
         args: [
           'update',
           ...resources.resourceLimits.filter((s) =>
-            /^--(?:memory|memory-swap|cpus|cpu-quota|cpu-period)=/.test(s)
+            /^--(?:memory|memory-swap|memory-reservation|cpus|cpu-quota|cpu-period)=/.test(
+              s
+            )
           ),
           plan.sessionName,
         ],

package/src/lib/execution-store.js

@@ -25,6 +25,7 @@ const os = require('os');
 const { execFileSync, spawnSync } = require('child_process');
 const crypto = require('crypto');
 const { atomicWrite } = require('./atomic-write');
+const { decodeStoreContent } = require('./store-decode-cache');
 
 // Synchronous wrapper using Bun's native ESM support
 // This works because Bun handles ESM/CJS interop
@@ -38,6 +39,12 @@ function encodeSync(data) {
   }
 }
 
+const DECODE_LOG = {
+  memory: () => 'Reused the decoded store: the file is unchanged',
+  copy: () => 'Read the decoded copy of the store: the file is unchanged',
+  decode: (content) => `Decoded the store (${content.length} characters)`,
+};
+
 function decodeSync(notation) {
   try {
     const codec = require('lino-objects-codec');
@@ -52,6 +59,8 @@ const DEFAULT_APP_FOLDER = path.join(os.homedir(), '.start-command');
 const LINO_DB_FILE = 'executions.lino';
 const LINKS_DB_FILE = 'executions.links';
 const LOCK_FILE = 'executions.lock';
+// Decoded records as JSON, for the next process (issue #243)
+const DECODED_COPY_FILE = 'executions.decoded.js.json';
 const { LockManager } = require('./store-lock');
 
 /**
@@ -262,6 +271,7 @@ class ExecutionStore {
     this.linoDbPath = path.join(this.appFolder, LINO_DB_FILE);
     this.linksDbPath = path.join(this.appFolder, LINKS_DB_FILE);
     this.lockFilePath = path.join(this.appFolder, LOCK_FILE);
+    this.decodedCopyPath = path.join(this.appFolder, DECODED_COPY_FILE);
     this.useLinks = options.useLinks !== false && isClinkInstalled();
     this.verbose = options.verbose || false;
 
@@ -302,7 +312,14 @@ class ExecutionStore {
         return [];
       }
 
-      const data = decodeSync(content);
+      // Decoded once per content, not once per call (issue #243).
+      const { data, source } = decodeStoreContent(
+        this.linoDbPath,
+        content,
+        decodeSync,
+        { copyPath: this.decodedCopyPath }
+      );
+      this.log(DECODE_LOG[source](content));
       if (!Array.isArray(data)) {
         return [];
       }
@@ -915,4 +932,5 @@ module.exports = {
   LINO_DB_FILE,
   LINKS_DB_FILE,
   LOCK_FILE,
+  DECODED_COPY_FILE,
 };

package/src/lib/exit-evidence.js

@@ -5,6 +5,16 @@ const MAIN_OOM = 'Exit evidence: main-oom (recent cgroup oom_kill delta)';
 const DOCKER_OOM = 'Exit evidence: main-oom (recent Docker oom event)';
 const DAEMON_RESTART =
   'Exit evidence: docker-daemon-restart (Docker service journal)';
+const DAEMON_RESTART_API =
+  'Exit evidence: docker-daemon-restart (Docker API: containers finished together)';
+const DAEMON_RESTART_PREFIX = 'Exit evidence: docker-daemon-restart';
+// dockerd gives every container its stop timeout and kills the rest within its
+// 15 s shutdown timeout, so the containers of one restart finish this close.
+const DAEMON_RESTART_WINDOW_MS = 15000;
+const CONTAINER_STATE_FORMAT =
+  '{{.Id}} {{.State.FinishedAt}} {{.State.StartedAt}} {{.State.ExitCode}} {{.State.OOMKilled}}';
+// Bounds one `docker inspect` call on hosts with many stopped containers.
+const MAX_PEER_CONTAINERS = 256;
 
 function recentOomDelta(sample, finishedAt) {
   const fields = String(sample || '')
@@ -39,6 +49,42 @@ function daemonRestartEvidence(journal, containerId) {
   return Boolean(exited && forced);
 }
 
+/**
+ * Socket-only daemon-restart evidence (issue #241): another container that
+ * finished within the same shutdown window, was not OOM-killed, and was either
+ * stopped by a signal (137/143) or started again afterwards (restart policy;
+ * moby resets its ExitCode to 0 but keeps FinishedAt). `docker events` cannot
+ * be used: its buffer does not survive the restart, while State does.
+ * @param {string} states - `CONTAINER_STATE_FORMAT` lines, one per container
+ */
+function daemonRestartApiEvidence(states, containerId, finishedAt) {
+  const finished = Date.parse(finishedAt);
+  return (
+    Number.isFinite(finished) &&
+    String(states || '')
+      .split('\n')
+      .some((line) => {
+        const [id, peerFinished, peerStarted, exit, oom] = line
+          .trim()
+          .split(/\s+/);
+        const end = Date.parse(peerFinished);
+        if (
+          !/^[a-f0-9]{64}$/.test(id || '') ||
+          id === containerId ||
+          oom !== 'false' ||
+          !Number.isFinite(end) ||
+          end <= 0 ||
+          Math.abs(end - finished) > DAEMON_RESTART_WINDOW_MS
+        ) {
+          return false;
+        }
+        return (
+          exit === '137' || exit === '143' || Date.parse(peerStarted) > end
+        );
+      })
+  );
+}
+
 function recentDockerOom(events, finishedAt) {
   const finished = Date.parse(finishedAt) / 1000;
   return (
@@ -68,6 +114,10 @@ function collectExitEvidence(
   // A local journal cannot diagnose a remote daemon. Require a local socket.
   const local =
     !process.env.DOCKER_HOST || process.env.DOCKER_HOST.startsWith('unix://');
+  const sampledOom =
+    (oomKilled === true || oomKilled === 'true') &&
+    (Number(exitCode) === 137 || Number(exitCode) < 0) &&
+    recentOomDelta(sample, finishedAt);
   if (
     Number(exitCode) === 137 &&
     containerName &&
@@ -114,18 +164,35 @@ function collectExitEvidence(
       if (events.success && recentDockerOom(events.stdout, finishedAt)) {
         return DOCKER_OOM;
       }
+      // Without a host journal (a mounted docker.sock, a remote daemon) the
+      // only record of the restart is the state of the other containers.
+      const peers = sampledOom
+        ? { success: false, stdout: '' }
+        : run(docker, ['ps', '-a', '-q', '--no-trunc']);
+      const ids = peers.stdout
+        .split(/\s+/)
+        .filter(
+          (peer) => /^[a-f0-9]{64}$/.test(peer) && peer !== id.stdout.trim()
+        )
+        .slice(0, MAX_PEER_CONTAINERS);
+      // A container removed after `docker ps` fails inspect with status 1 but
+      // the others are still printed, so the output is used either way.
+      const states = ids.length
+        ? run(docker, ['inspect', '-f', CONTAINER_STATE_FORMAT, ...ids])
+        : { success: false, stdout: '' };
+      if (
+        daemonRestartApiEvidence(states.stdout, id.stdout.trim(), finishedAt)
+      ) {
+        return DAEMON_RESTART_API;
+      }
       if (process.env.START_DEBUG === '1') {
         console.error(
-          `[DEBUG] Exit evidence: container=${id.stdout.trim()} finishedAt=${finishedAt} eventsAvailable=${events.success} oomEvents=${JSON.stringify(events.stdout)} sample=${JSON.stringify(sample || '')}`
+          `[DEBUG] Exit evidence: container=${id.stdout.trim()} finishedAt=${finishedAt} eventsAvailable=${events.success} oomEvents=${JSON.stringify(events.stdout)} peerContainers=${ids.length} peerStates=${JSON.stringify(states.stdout)} sample=${JSON.stringify(sample || '')}`
         );
       }
     }
   }
-  if (
-    (oomKilled === true || oomKilled === 'true') &&
-    (Number(exitCode) === 137 || Number(exitCode) < 0) &&
-    recentOomDelta(sample, finishedAt)
-  ) {
+  if (sampledOom) {
     return MAIN_OOM;
   }
   return 'Exit evidence: unavailable (no attributed exit-time OOM or daemon restart evidence)';
@@ -134,12 +201,72 @@ function collectExitEvidence(
 function fromLog(log) {
   const lines = String(log || '').split('\n');
   const last = lines.filter((line) => line.startsWith('Exit evidence:')).at(-1);
+  const daemonRestart = Boolean(last?.startsWith(DAEMON_RESTART_PREFIX));
   return {
     mainOom: last === MAIN_OOM || last === DOCKER_OOM,
-    daemonRestart: last === DAEMON_RESTART,
+    daemonRestart,
+    // Which source attributed the restart: the host journal or the API.
+    ...(daemonRestart
+      ? { daemonRestartSource: last === DAEMON_RESTART_API ? 'api' : 'journal' }
+      : {}),
   };
 }
 
+/** The stored `daemonRestartSource`, kept only when a restart was attributed. */
+function sourceOf(logged) {
+  return logged?.daemonRestart && logged.daemonRestartSource
+    ? { daemonRestartSource: logged.daemonRestartSource }
+    : {};
+}
+
+// A watcher that shares the daemon with its task (a mounted docker.sock) dies
+// in the same restart and never logs evidence, so `--status` collects it from
+// the container state that survived, once per process (issue #241).
+const lostWatcherEvidence = new Map();
+function collectLostWatcherEvidence(record, state, runner) {
+  const name = record?.options?.sessionName;
+  if (
+    !name ||
+    record.options.exitEvidence ||
+    state?.running !== false ||
+    state.exitCode !== 137 ||
+    !state.finishedAt
+  ) {
+    return null;
+  }
+  const key = `${name}\0${state.finishedAt}`;
+  if (!lostWatcherEvidence.has(key)) {
+    const logged = fromLog(
+      collectExitEvidence(
+        {
+          containerName: name,
+          finishedAt: state.finishedAt,
+          exitCode: state.exitCode,
+          oomKilled: state.oomKilled,
+        },
+        runner
+      )
+    );
+    lostWatcherEvidence.set(key, {
+      mainOom: logged.mainOom,
+      daemonRestart: logged.daemonRestart,
+      ...sourceOf(logged),
+    });
+  }
+  return { ...lostWatcherEvidence.get(key) };
+}
+
+/** Stores the lost watcher's evidence when the log tail carries none. */
+function addLostWatcherEvidence(enriched, record, state, logTail) {
+  const lost =
+    state && !/^Exit evidence:/m.test(logTail || '')
+      ? collectLostWatcherEvidence(record, state)
+      : null;
+  if (lost) {
+    enriched.options = { ...enriched.options, exitEvidence: lost };
+  }
+}
+
 function buildExitEvidenceSnippet(containerName, quotedLogPath) {
   const stderr = process.env.START_DEBUG === '1' ? '2>&1' : '2>/dev/null';
   return `${shellQuote(process.execPath)} ${shellQuote(__filename)} ${shellQuote(containerName)} "$__start_command_cgroup" "$__start_command_finished" "$__start_command_exit" "$__start_command_oom" >> ${quotedLogPath} ${stderr}`;
@@ -149,11 +276,16 @@ module.exports = {
   MAIN_OOM,
   DOCKER_OOM,
   DAEMON_RESTART,
+  DAEMON_RESTART_API,
   recentOomDelta,
+  daemonRestartApiEvidence,
   recentDockerOom,
   daemonRestartEvidence,
   collectExitEvidence,
   fromLog,
+  sourceOf,
+  collectLostWatcherEvidence,
+  addLostWatcherEvidence,
   buildExitEvidenceSnippet,
 };
 if (require.main === module) {

package/src/lib/output-blocks.js

@@ -119,6 +119,7 @@ function parseIsolationMetadata(extraLines) {
     session: null,
     endpoint: null,
     user: null,
+    init: false,
   };
 
   for (const line of extraLines) {
@@ -146,6 +147,12 @@ function parseIsolationMetadata(extraLines) {
       continue;
     }
 
+    // Parse [Isolation] Init: true (docker run --init, issue #244)
+    if (line === '[Isolation] Init: true') {
+      metadata.init = true;
+      continue;
+    }
+
     // Parse [Isolation] Endpoint: user@host
     const endpointMatch = line.match(/\[Isolation\] Endpoint: (.+)/);
     if (endpointMatch) {
@@ -207,6 +214,10 @@ function generateIsolationLines(metadata, containerOrScreenName = null) {
     }
   }
 
+  if (metadata.init) {
+    lines.push(createTimelineLine('init', 'true'));
+  }
+
   if (metadata.user) {
     lines.push(createTimelineLine('user', metadata.user));
   }

package/src/lib/resume-resources.js

@@ -34,7 +34,7 @@ function prepareResumeResources(record, probe, overrides, runner, random) {
   const options = record.options || {};
   if (options.isolated !== 'docker') {
     if (
-      ['memory', 'memorySwap', 'cpus'].some(
+      ['memory', 'memorySwap', 'memoryReservation', 'cpus'].some(
         (k) => overrides[k] !== null && overrides[k] !== undefined
       )
     ) {
@@ -81,7 +81,7 @@ function prepareResumeResources(record, probe, overrides, runner, random) {
   };
   result.update =
     probe.state === 'stopped' &&
-    (['memory', 'memorySwap', 'cpus'].some(
+    (['memory', 'memorySwap', 'memoryReservation', 'cpus'].some(
       (k) => overrides[k] !== null && overrides[k] !== undefined
     ) ||
       Boolean(options.cpuPenaltyConfig));

package/src/lib/screen-isolation.js

@@ -100,6 +100,28 @@ function supportsLogfileOption() {
   return version.patch >= 1;
 }
 
+/**
+ * Polls without the session in `screen -ls` and without an exit code before
+ * the session counts as gone. `screen -dmS` can return before the session is
+ * listed, so a missing session alone is no proof that the command finished;
+ * its exit code is written just before the session ends.
+ */
+const SCREEN_UNLISTED_GRACE_POLLS = 20;
+
+/**
+ * Whether an attached screen run is over.
+ * @param {boolean} listed - The session is in `screen -ls`
+ * @param {boolean} exitCodeReady - The exit-code sidecar has content (or no
+ *   sidecar is expected, as for a bare shell)
+ * @param {number} unlistedPolls - Consecutive polls without the session
+ * @returns {boolean}
+ */
+function isScreenRunFinished(listed, exitCodeReady, unlistedPolls) {
+  return (
+    !listed && (exitCodeReady || unlistedPolls >= SCREEN_UNLISTED_GRACE_POLLS)
+  );
+}
+
 /**
  * Run command in GNU Screen using detached mode with log capture.
  *
@@ -346,58 +368,64 @@ function runScreenWithLogCapture(
       const checkInterval = 100; // ms
       const maxWait = 300000; // 5 minutes max
       let waited = 0;
+      let unlistedPolls = 0;
+
+      const exitCodeReady = () => {
+        if (isBareShell) {
+          return true;
+        }
+        try {
+          return fs.readFileSync(exitCodeFile, 'utf8').trim() !== '';
+        } catch {
+          return false;
+        }
+      };
+
+      const finish = () => {
+        readAndDisplayOutput().then((output) => {
+          const exitCode = readExitCode();
+          cleanupTempFiles();
+          resolve({
+            success: exitCode === 0,
+            sessionName,
+            message: `Screen session "${sessionName}" exited with code ${exitCode}`,
+            exitCode,
+            output,
+          });
+        });
+      };
 
       const checkCompletion = () => {
+        let listed;
         try {
           // Check if session still exists
-          const sessions = execSync('screen -ls', {
+          listed = execSync('screen -ls', {
             encoding: 'utf8',
             stdio: ['pipe', 'pipe', 'pipe'],
-          });
-
-          if (!sessions.includes(sessionName)) {
-            // Session ended, read output and exit code
-            readAndDisplayOutput().then((output) => {
-              const exitCode = readExitCode();
-              cleanupTempFiles();
-              resolve({
-                success: exitCode === 0,
-                sessionName,
-                message: `Screen session "${sessionName}" exited with code ${exitCode}`,
-                exitCode,
-                output,
-              });
-            });
-            return;
-          }
-
-          waited += checkInterval;
-          if (waited >= maxWait) {
-            cleanupTempFiles();
-            resolve({
-              success: false,
-              sessionName,
-              message: `Screen session "${sessionName}" timed out after ${maxWait / 1000} seconds`,
-              exitCode: 1,
-            });
-            return;
-          }
-
-          setTimeout(checkCompletion, checkInterval);
+          }).includes(sessionName);
         } catch {
           // screen -ls failed, session probably ended
-          readAndDisplayOutput().then((output) => {
-            const exitCode = readExitCode();
-            cleanupTempFiles();
-            resolve({
-              success: exitCode === 0,
-              sessionName,
-              message: `Screen session "${sessionName}" exited with code ${exitCode}`,
-              exitCode,
-              output,
-            });
+          listed = false;
+        }
+        unlistedPolls = listed ? 0 : unlistedPolls + 1;
+        if (isScreenRunFinished(listed, exitCodeReady(), unlistedPolls)) {
+          finish();
+          return;
+        }
+
+        waited += checkInterval;
+        if (waited >= maxWait) {
+          cleanupTempFiles();
+          resolve({
+            success: false,
+            sessionName,
+            message: `Screen session "${sessionName}" timed out after ${maxWait / 1000} seconds`,
+            exitCode: 1,
           });
+          return;
         }
+
+        setTimeout(checkCompletion, checkInterval);
       };
 
       // Start checking after a brief delay
@@ -522,7 +550,9 @@ function resetScreenVersionCache() {
 }
 
 module.exports = {
+  SCREEN_UNLISTED_GRACE_POLLS,
   getScreenVersion,
+  isScreenRunFinished,
   supportsLogfileOption,
   runScreenWithLogCapture,
   startDetachedScreenWithLogCapture,