start-command 0.34.0 → 0.35.0

package/src/lib/isolation.js

@@ -505,96 +505,12 @@ const {
   canRunLinuxDockerImages,
 } = require('./docker-utils');
 
-/**
- * Build the docker run runtime argument list contributed by configurable
- * container options: privileged mode, environment variables, volumes/bind
- * mounts, --mount specs, and network configuration. Returned in a stable order so they can be spliced
- * into the `docker run` argv before the image name.
- * @param {object} options - Options (privileged, env, volumes, mounts, network, networkAliases)
- * @returns {string[]} Docker CLI arguments
- */
-function buildDockerRuntimeArgs(options = {}) {
-  const args = [];
-  if (options.privileged) {
-    args.push('--privileged');
-  }
-  for (const envVar of options.env || []) {
-    args.push('-e', envVar);
-  }
-  for (const volume of options.volumes || []) {
-    args.push('-v', volume);
-  }
-  for (const mount of options.mounts || []) {
-    args.push('--mount', mount);
-  }
-  const [firstNetwork] = getDockerNetworks(options);
-  if (firstNetwork) {
-    args.push('--network', firstNetwork);
-  }
-  for (const alias of options.networkAliases || []) {
-    args.push('--network-alias', alias);
-  }
-  return args;
-}
-
-/**
- * Build the human-readable `[Isolation]` status lines for docker runtime
- * options (volumes, mounts, env, privileged). Empty collections and a falsy
- * privileged flag contribute no lines.
- * @param {object} options - Options (volumes, mounts, env, privileged, network, networks, networkAliases)
- * @returns {string[]} Status lines for the start block / log header
- */
-function buildDockerRuntimeStatusLines(options = {}) {
-  const lines = [];
-  if (options.volumes && options.volumes.length > 0) {
-    lines.push(`[Isolation] Volumes: ${options.volumes.join(', ')}`);
-  }
-  if (options.mounts && options.mounts.length > 0) {
-    lines.push(`[Isolation] Mounts: ${options.mounts.join(', ')}`);
-  }
-  if (options.env && options.env.length > 0) {
-    lines.push(`[Isolation] Env: ${options.env.join(', ')}`);
-  }
-  if (options.privileged) {
-    lines.push(`[Isolation] Privileged: true`);
-  }
-  const networks = getDockerNetworks(options);
-  if (networks.length > 0) {
-    lines.push(`[Isolation] Network: ${networks[0]}`);
-  }
-  if (networks.length > 1) {
-    lines.push(`[Isolation] Networks: ${networks.join(', ')}`);
-  }
-  if (options.networkAliases && options.networkAliases.length > 0) {
-    lines.push(
-      `[Isolation] Network aliases: ${options.networkAliases.join(', ')}`
-    );
-  }
-  return lines;
-}
-
-/**
- * Build the execution-record metadata for docker runtime options, normalizing
- * empty collections and a falsy privileged flag to `null`.
- * @param {object} options - Options (volumes, mounts, env, privileged, network, networks, networkAliases)
- * @returns {{volumes: ?string[], mounts: ?string[], env: ?string[], privileged: ?boolean, network: ?string, networks: ?string[], networkAliases: ?string[]}}
- */
-function buildDockerRuntimeMetadata(options = {}) {
-  const networks = getDockerNetworks(options);
-  return {
-    volumes:
-      options.volumes && options.volumes.length > 0 ? options.volumes : null,
-    mounts: options.mounts && options.mounts.length > 0 ? options.mounts : null,
-    env: options.env && options.env.length > 0 ? options.env : null,
-    privileged: options.privileged || null,
-    network: networks[0] || null,
-    networks: networks.length > 0 ? networks : null,
-    networkAliases:
-      options.networkAliases && options.networkAliases.length > 0
-        ? options.networkAliases
-        : null,
-  };
-}
+const {
+  buildDockerRuntimeArgs,
+  buildDockerRuntimeStatusLines,
+  buildDockerRuntimeMetadata,
+} = require('./docker-runtime-args');
+const { buildRecoverySelectorArgs } = require('./execution-recovery');
 
 /**
  * Run command in Docker container
@@ -679,11 +595,22 @@ function runInDocker(command, options = {}) {
       const shellArgs = shellInteractiveFlag
         ? [shellToUse, shellInteractiveFlag]
         : [shellToUse];
-      const cmdArgs = isBareShell
+      const mainCmdArgs = isBareShell
         ? toShellWords(command)
         : isShellInvocationWithArgs(command)
           ? buildShellWithArgsCmdArgs(effectiveCommand)
           : [...shellArgs, '-c', effectiveCommand];
+      // A recovery command is selected by a marker that `docker cp` drops
+      // into the container before it is started again (issue #176).
+      const cmdArgs = options.recoveryCommand
+        ? buildRecoverySelectorArgs(mainCmdArgs, {
+            shell: shellToUse,
+            shellFlag: shellInteractiveFlag,
+            recoveryCommand: options.keepAlive
+              ? `${options.recoveryCommand}; exec ${shellToUse}`
+              : options.recoveryCommand,
+          })
+        : mainCmdArgs;
       dockerArgs.push(options.image, ...cmdArgs);
 
       if (DEBUG) {
@@ -736,7 +663,8 @@ function runInDocker(command, options = {}) {
         containerName,
         cleanupPolicy,
         options.logPath,
-        options.executionId || null
+        options.executionId || null,
+        { recoverOnKill: Boolean(options.onKillResume) }
       );
 
       let message = `Command started in detached docker container: ${containerName}`;

package/src/lib/spawn-helpers.js

@@ -12,6 +12,7 @@
 const { spawn } = require('child_process');
 const fs = require('fs');
 const path = require('path');
+const { resolveChildExitCode } = require('./exit-reason');
 
 function ensureParentDirectory(filePath) {
   fs.mkdirSync(path.dirname(filePath), { recursive: true });
@@ -282,8 +283,9 @@ function runWithNodeSpawn(options) {
   // The 'close' event fires after all stdio streams have been closed, ensuring
   // all stdout/stderr data has been received. The 'exit' event can fire before
   // buffered data is received, causing output loss on macOS (Issue #57).
-  child.on('close', (code) => {
-    const exitCode = code || 0;
+  child.on('close', (code, signal) => {
+    // A signal death reports `code === null`: never record it as exit 0.
+    const exitCode = resolveChildExitCode(code, signal);
     const durationMs = Date.now() - startTimeMs;
     const endTime = new Date().toISOString().replace('T', ' ').substring(0, 23);
 

package/src/lib/usage.js

@@ -16,6 +16,8 @@ Options:
   --privileged          Run docker container in privileged mode (docker only)
   --network <name>      Connect to a named network (repeatable, docker only)
   --network-alias <alias> Add alias to the first network (repeatable, docker only)
+  --on-kill-resume <N>  Resume a killed (exit 137 / OOM) detached docker session up to N times
+  --recovery-command <cmd>  Command to run in the same container on such a resume
   --endpoint <endpoint> SSH endpoint (required for ssh isolation, e.g., user@host)
   --isolated-user, -u [name]  Create isolated user with same permissions
   --keep-user           Keep isolated user after command completes
@@ -66,6 +68,7 @@ Examples:
   $ --resume my-docker-session             # Restart the stored command in the same container
   $ --resume my-docker-session -- bash     # Run a different command in the same container
   $ --resume-all                           # Re-attach every still-running execution
+  $ -i docker -d --on-kill-resume 3 --recovery-command 'solve --resume' -- solve
   $ --use-command-stream echo "Hello"      # Use command-stream library`);
   console.log('');
   console.log('Piping with $:');

package/test/docker-runtime-options.js

@@ -304,6 +304,9 @@ describe('buildDockerRuntimeMetadata', () => {
       network: null,
       networks: null,
       networkAliases: null,
+      resourceLimits: null,
+      onKillResume: null,
+      recoveryCommand: null,
     });
   });
 
@@ -326,6 +329,9 @@ describe('buildDockerRuntimeMetadata', () => {
         network: 'hive-formal-ai',
         networks: ['hive-formal-ai', 'public-egress'],
         networkAliases: ['formal-ai', 'checker'],
+        resourceLimits: null,
+        onKillResume: null,
+        recoveryCommand: null,
       }
     );
   });

package/test/regression-170.js

@@ -301,7 +301,10 @@ describe('issue #170.1: the detached watcher persists the terminal state', () =>
 
     const stored = store.get(record.uuid);
     expect(stored.status).toBe(ExecutionStatus.EXECUTED);
-    expect(stored.exitCode).toBe(0);
+    // `ExitCode=0` next to a zero `FinishedAt` is Docker's zero value, not an
+    // observed success (issue #174).
+    expect(stored.exitCode).toBe(-1);
+    expect(stored.exitReason).toBe('watcher-lost-container');
     expect(stored.endTimeSource).toBe('observed-at');
     expect(stored.observedAt).toBe(stored.endTime);
     expect(new Date(stored.endTime).getUTCFullYear()).toBeGreaterThan(2000);

package/test/regression-174.js

@@ -0,0 +1,397 @@
+/**
+ * Regression tests for issue #174: the detached docker watcher `docker rm -f`ed
+ * a still-running container when `docker logs -f` failed (ENOSPC) and
+ * finalized it as executed / exit 0.
+ *
+ *   - `docker logs -f C >> LOG` also returns when its own write to LOG fails,
+ *     and on a dockerd restart; the watcher must wait for the real exit;
+ *   - a container that is still running must never be removed, get an
+ *     `Exit Code:` footer, or be finalized;
+ *   - `ExitCode=0` next to a zero `FinishedAt` is Docker's zero value, never an
+ *     observed exit 0;
+ *   - a child killed by a signal (`code === null`) must never become exit 0.
+ */
+
+const { describe, it, expect, afterEach } = require('bun:test');
+const { spawnSync } = require('child_process');
+const fs = require('fs');
+const path = require('path');
+const os = require('os');
+
+const {
+  ExecutionStore,
+  ExecutionRecord,
+  ExecutionStatus,
+} = require('../src/lib/execution-store');
+const {
+  buildDetachedDockerCompletionScript,
+  DOCKER_CONTAINER_CLEANUP_POLICY,
+} = require('../src/lib/docker-cleanup');
+const {
+  LOG_CAPTURE_STOPPED_NOTE,
+  STILL_RUNNING_NOTE,
+} = require('../src/lib/docker-post-mortem');
+const {
+  WATCHER_LOST_CONTAINER,
+  finalizeDetachedExecution,
+} = require('../src/lib/detached-finalize');
+const { resolveChildExitCode } = require('../src/lib/exit-reason');
+const { runWithNodeSpawn } = require('../src/lib/spawn-helpers');
+
+const tempDirs = [];
+
+function makeTempDir(prefix) {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), prefix));
+  tempDirs.push(dir);
+  return dir;
+}
+
+afterEach(() => {
+  while (tempDirs.length > 0) {
+    const dir = tempDirs.pop();
+    fs.rmSync(dir, { recursive: true, force: true });
+  }
+});
+
+function detachedDockerRecord(logPath) {
+  return new ExecutionRecord({
+    status: ExecutionStatus.EXECUTING,
+    command: 'work',
+    logPath,
+    options: {
+      isolated: 'docker',
+      isolationMode: 'detached',
+      sessionName: 'start-command-174',
+    },
+  });
+}
+
+// The watcher body is a POSIX `sh` script; Windows has no `/bin/sh`.
+const needsPosixShell = () => {
+  if (process.platform !== 'win32') {
+    return false;
+  }
+  console.log('  Skipping: the watcher needs a POSIX /bin/sh');
+  return true;
+};
+
+/**
+ * Run the real watcher script against a scripted fake `docker`.
+ *
+ * `running` lists the successive answers to `docker inspect -f
+ * '{{.State.Running}}'` (`true`, `false`, or `fail` for an inspect that
+ * errors, as during a dockerd restart); the last answer sticks. `docker logs`
+ * writes one line and fails, exactly like `docker logs -f C >> LOG` whose write
+ * hit ENOSPC. Every call is recorded, and `rm` records whether the container was
+ * running at that moment.
+ */
+function runWatcher({ policy, running, state, withLog = true }) {
+  const dir = makeTempDir('watcher-174-');
+  const binDir = path.join(dir, 'bin');
+  const appFolder = path.join(dir, 'app');
+  fs.mkdirSync(binDir);
+  const logPath = path.join(dir, 'run.log');
+  fs.writeFileSync(path.join(dir, 'running'), `${running.join('\n')}\n`);
+  fs.writeFileSync(path.join(dir, 'state'), `${state}\n`);
+  const calls = path.join(dir, 'calls');
+  const dockerPath = path.join(binDir, 'docker');
+  fs.writeFileSync(
+    dockerPath,
+    [
+      '#!/bin/sh',
+      `dir='${dir}'`,
+      'case "$1" in',
+      '  inspect)',
+      '    case "$3" in',
+      '      *State.Running*)',
+      '        answer=$(head -n 1 "$dir/running")',
+      '        if [ "$(wc -l < "$dir/running")" -gt 1 ]; then',
+      '          tail -n +2 "$dir/running" > "$dir/running.next" && mv "$dir/running.next" "$dir/running"',
+      '        fi',
+      '        echo "inspect-running $answer" >> "$dir/calls"',
+      '        [ "$answer" = fail ] && exit 1',
+      '        echo "$answer" > "$dir/current"',
+      '        echo "$answer" ;;',
+      '      *State.Error*) echo "" ;;',
+      '      *) echo "inspect-state" >> "$dir/calls"; cat "$dir/state" ;;',
+      '    esac ;;',
+      '  logs) echo logs >> "$dir/calls"; echo work; exit 1 ;;',
+      '  wait) echo wait >> "$dir/calls"; echo 0 ;;',
+      '  rm) echo "rm running=$(cat "$dir/current" 2>/dev/null)" >> "$dir/calls" ;;',
+      'esac',
+      '',
+    ].join('\n'),
+    'utf8'
+  );
+  fs.chmodSync(dockerPath, 0o755);
+
+  const store = new ExecutionStore({ appFolder, useLinks: false });
+  const record = detachedDockerRecord(withLog ? logPath : null);
+  store.save(record);
+
+  const script = buildDetachedDockerCompletionScript(
+    'demo',
+    policy,
+    withLog ? logPath : null,
+    record.uuid
+  );
+  const result = spawnSync('/bin/sh', ['-c', script], {
+    encoding: 'utf8',
+    env: {
+      ...process.env,
+      PATH: `${binDir}${path.delimiter}${process.env.PATH}`,
+      START_APP_FOLDER: appFolder,
+      START_DISABLE_TRACKING: '',
+    },
+    timeout: 20000,
+  });
+  return {
+    status: result.status,
+    calls: fs.readFileSync(calls, 'utf8').trim().split('\n'),
+    log: fs.existsSync(logPath) ? fs.readFileSync(logPath, 'utf8') : '',
+    record: new ExecutionStore({ appFolder, useLinks: false }).get(record.uuid),
+  };
+}
+
+const FINISHED_7 =
+  '7 false 2026-09-27T10:00:00.000000000Z 2026-09-27T10:00:05.000000000Z';
+const FINISHED_0 =
+  '0 false 2026-09-27T10:00:00.000000000Z 2026-09-27T10:00:05.000000000Z';
+const NOT_FINISHED = '0 false 2026-09-27T10:00:00Z 0001-01-01T00:00:00Z';
+
+describe('issue #174: docker logs -f returning is not the container exiting', () => {
+  it('waits for the real exit after docker logs -f fails (ENOSPC)', () => {
+    if (needsPosixShell()) {
+      return;
+    }
+    const run = runWatcher({
+      policy: DOCKER_CONTAINER_CLEANUP_POLICY.DEFAULT,
+      running: ['true', 'true', 'false'],
+      state: FINISHED_7,
+    });
+
+    expect(run.status).toBe(0);
+    expect(run.calls).toContain('wait');
+    expect(run.calls.indexOf('wait')).toBeGreaterThan(
+      run.calls.indexOf('logs')
+    );
+    expect(run.calls.indexOf('inspect-state')).toBeGreaterThan(
+      run.calls.lastIndexOf('wait')
+    );
+    expect(run.log).toContain(LOG_CAPTURE_STOPPED_NOTE);
+    expect(run.log).toContain('Exit Code: 7');
+    expect(run.log).not.toContain('Exit Code: 0');
+    expect(run.record.status).toBe(ExecutionStatus.EXECUTED);
+    expect(run.record.exitCode).toBe(7);
+    expect(run.record.endTimeSource).toBe('docker-finished-at');
+  });
+
+  it('only removes the container once it has really exited', () => {
+    if (needsPosixShell()) {
+      return;
+    }
+    const run = runWatcher({
+      policy: DOCKER_CONTAINER_CLEANUP_POLICY.ALWAYS,
+      running: ['true', 'true', 'false'],
+      state: FINISHED_0,
+    });
+
+    expect(run.calls).toContain('rm running=false');
+    expect(run.calls).not.toContain('rm running=true');
+    expect(run.log).toContain('Exit Code: 0');
+    expect(run.record.exitCode).toBe(0);
+  });
+
+  it('keeps waiting across a failed docker inspect (dockerd restart)', () => {
+    if (needsPosixShell()) {
+      return;
+    }
+    const run = runWatcher({
+      policy: DOCKER_CONTAINER_CLEANUP_POLICY.DEFAULT,
+      running: ['true', 'true', 'true', 'false'],
+      state: FINISHED_7,
+    });
+
+    expect(run.calls.filter((call) => call === 'wait').length).toBe(2);
+    expect(run.record.exitCode).toBe(7);
+  });
+
+  it('applies the same wait to the no-log watcher', () => {
+    if (needsPosixShell()) {
+      return;
+    }
+    const run = runWatcher({
+      policy: DOCKER_CONTAINER_CLEANUP_POLICY.ALWAYS,
+      running: ['true', 'false'],
+      state: FINISHED_7,
+      withLog: false,
+    });
+
+    expect(run.calls.filter((call) => call === 'wait').length).toBe(2);
+    expect(run.calls).toContain('rm running=false');
+    expect(run.calls).not.toContain('rm running=true');
+    expect(run.record.exitCode).toBe(7);
+  });
+});
+
+describe('issue #174: a container that is still running is left alone', () => {
+  it('never removes, footers or finalizes a running container', () => {
+    if (needsPosixShell()) {
+      return;
+    }
+    // logs -f fails -> running; the wait loop's inspect fails (daemon down) ->
+    // the loop ends; once the daemon is back the container is still running.
+    const run = runWatcher({
+      policy: DOCKER_CONTAINER_CLEANUP_POLICY.ALWAYS,
+      running: ['true', 'fail', 'true'],
+      state: NOT_FINISHED,
+    });
+
+    expect(run.calls.some((call) => call.startsWith('rm'))).toBe(false);
+    expect(run.log).toContain('Container still running: demo');
+    expect(run.log).toContain(STILL_RUNNING_NOTE);
+    expect(run.log).not.toContain('Exit Code:');
+    expect(run.record.status).toBe(ExecutionStatus.EXECUTING);
+    expect(run.record.exitCode).toBeNull();
+  });
+
+  it('never removes a running container without a log either', () => {
+    if (needsPosixShell()) {
+      return;
+    }
+    const run = runWatcher({
+      policy: DOCKER_CONTAINER_CLEANUP_POLICY.ALWAYS,
+      running: ['fail', 'true'],
+      state: NOT_FINISHED,
+      withLog: false,
+    });
+
+    expect(run.calls.some((call) => call.startsWith('rm'))).toBe(false);
+    expect(run.record.status).toBe(ExecutionStatus.EXECUTING);
+  });
+});
+
+describe('issue #174: a zero FinishedAt is never an exit 0', () => {
+  it('keeps the container and records no success in the watcher', () => {
+    if (needsPosixShell()) {
+      return;
+    }
+    const run = runWatcher({
+      policy: DOCKER_CONTAINER_CLEANUP_POLICY.DEFAULT,
+      running: ['false'],
+      state: NOT_FINISHED,
+    });
+
+    expect(run.calls.some((call) => call.startsWith('rm'))).toBe(false);
+    expect(run.log).toContain('Exit Code: -1');
+    expect(run.log).not.toContain('Exit Code: 0');
+    expect(run.record.status).toBe(ExecutionStatus.EXECUTED);
+    expect(run.record.exitCode).toBe(-1);
+    expect(run.record.exitReason).toBe(WATCHER_LOST_CONTAINER);
+    expect(run.record.endTimeSource).toBe('observed-at');
+  });
+
+  it('refuses to finalize a record whose container is still running', () => {
+    const appFolder = makeTempDir('finalize-174-running-');
+    const store = new ExecutionStore({ appFolder, useLinks: false });
+    const record = detachedDockerRecord(null);
+    store.save(record);
+
+    const outcome = finalizeDetachedExecution({
+      store,
+      executionId: record.uuid,
+      exitCode: '0',
+      oomKilled: 'false',
+      startedAt: '2026-09-27T10:00:00Z',
+      finishedAt: '0001-01-01T00:00:00Z',
+      containerError: '',
+      running: 'true',
+    });
+
+    expect(outcome).toEqual({ updated: false, reason: 'still-running' });
+    expect(store.get(record.uuid).status).toBe(ExecutionStatus.EXECUTING);
+  });
+
+  it('does not trust exit 0 without a FinishedAt', () => {
+    const appFolder = makeTempDir('finalize-174-zero-');
+    const store = new ExecutionStore({ appFolder, useLinks: false });
+    const record = detachedDockerRecord(null);
+    store.save(record);
+
+    finalizeDetachedExecution({
+      store,
+      executionId: record.uuid,
+      exitCode: '0',
+      oomKilled: 'false',
+      startedAt: '2026-09-27T10:00:00Z',
+      finishedAt: '0001-01-01T00:00:00Z',
+      running: 'false',
+    });
+
+    const stored = store.get(record.uuid);
+    expect(stored.exitCode).toBe(-1);
+    expect(stored.exitReason).toBe(WATCHER_LOST_CONTAINER);
+  });
+
+  it('still records a real exit 0', () => {
+    const appFolder = makeTempDir('finalize-174-real-zero-');
+    const store = new ExecutionStore({ appFolder, useLinks: false });
+    const record = detachedDockerRecord(null);
+    store.save(record);
+
+    finalizeDetachedExecution({
+      store,
+      executionId: record.uuid,
+      exitCode: '0',
+      oomKilled: 'false',
+      startedAt: '2026-09-27T10:00:00Z',
+      finishedAt: '2026-09-27T10:00:05Z',
+      running: 'false',
+    });
+
+    const stored = store.get(record.uuid);
+    expect(stored.exitCode).toBe(0);
+    expect(stored.exitReason).toBeUndefined();
+    expect(stored.endTimeSource).toBe('docker-finished-at');
+  });
+});
+
+describe('issue #174: a signal death is never exit 0', () => {
+  it('maps a null code with a signal to 128+n', () => {
+    expect(resolveChildExitCode(null, 'SIGKILL')).toBe(137);
+    expect(resolveChildExitCode(null, 'SIGTERM')).toBe(143);
+    expect(resolveChildExitCode(null, 'SIGINT')).toBe(130);
+  });
+
+  it('maps a missing code without a signal to a failure', () => {
+    expect(resolveChildExitCode(null)).toBe(1);
+    expect(resolveChildExitCode(undefined)).toBe(1);
+    expect(resolveChildExitCode(null, 'SIGNOTREAL')).toBe(1);
+  });
+
+  it('keeps real exit codes, including 0', () => {
+    expect(resolveChildExitCode(0)).toBe(0);
+    expect(resolveChildExitCode(3, null)).toBe(3);
+  });
+
+  it('records a self-killed child as 137 in the node spawn path', async () => {
+    if (needsPosixShell()) {
+      return;
+    }
+    const dir = makeTempDir('spawn-174-');
+    const logFilePath = path.join(dir, 'run.log');
+    const exitCode = await new Promise((resolve) => {
+      runWithNodeSpawn({
+        shell: '/bin/sh',
+        shellArgs: ['-c', 'kill -KILL $$'],
+        logFilePath,
+        startTimeMs: Date.now(),
+        onComplete: (code) => resolve(code),
+        onError: () => resolve('error'),
+      });
+    });
+
+    expect(exitCode).toBe(137);
+    expect(fs.readFileSync(logFilePath, 'utf8')).toContain('Exit Code: 137');
+  });
+});