start-command 0.37.7 → 0.37.9

package/CHANGELOG.md

@@ -1,5 +1,21 @@
 # start-command
 
+## 0.37.9
+
+### Patch Changes
+
+- 55ed66d: A Docker snapshot resume can now be supervised safely. `--snapshot-wait <seconds>` (or `START_SNAPSHOT_LOCK_WAIT`) waits for another resume's snapshot on the host and says once which container it is waiting for. `--snapshot-disk-wait <seconds>` (or `START_SNAPSHOT_DISK_WAIT`) re-checks free disk instead of refusing at once. A busy lock and disk that is still short now exit 75 (`EX_TEMPFAIL`), so a caller can tell "retry later" from a real failure. `--remove-original-after <seconds>` removes the original only if the replacement is still running after the delay. `--snapshot-size-hint <bytes>` skips the `docker inspect --size` walk, and `--snapshot-prepare <command>` runs a cleanup command in the started original before the commit. A long `docker commit` writes its elapsed time to the log every minute. The `start-command.snapshot-image` label is documented as a stable interface, and `$ --cleanup <uuid-or-session-name>` removes an execution's containers and then their snapshot images, never forcing a removal (#224).
+
+  `$ --status` now reports `containerState removed` and `containerRemovedObservedAt` when the docker container of a finished detached execution no longer exists, instead of showing the old `containerId` as if the container still held its disk. The removal is recorded when start's own cleanup policy removes the container and when `--status` finds it gone; `containerId` is kept for history, and a docker daemon that cannot be reached is not mistaken for a removed container (#225).
+
+  `$ --status` no longer prints `[object Object]` for nested option values such as `launchOwner`, `exitEvidence`, `resolvedLimits` and `resourceLimitSpecs`: the links-notation output nests them like other objects and the text output prints them as compact JSON (#226).
+
+## 0.37.8
+
+### Patch Changes
+
+- 9ffcae7: A closed stdout/stderr pipe (`$ -- cmd | head -1`) no longer crashes start with an unhandled `EPIPE` and exit code 1: the command keeps running, its log and record are written, and start exits with the command's own exit code (#222).
+
 ## 0.37.7
 
 ### Patch Changes

package/README.md

@@ -40,6 +40,11 @@ checked examples.
 containers support copy-free command handoff; legacy snapshots require capacity
 preflight. Log uploads are private and sanitized; manual `--no-sanitize` opts out.
 
+Snapshot resumes take `--snapshot-wait`, `--snapshot-disk-wait`,
+`--remove-original-after`, `--snapshot-size-hint` and `--snapshot-prepare`, and
+exit 75 when they should be retried later; `--cleanup <id>` removes one
+execution's containers and snapshot images without forcing.
+
 ## Development
 
 ```bash

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "start-command",
-  "version": "0.37.7",
+  "version": "0.37.9",
   "description": "Gamification of coding, execute any command with ability to auto-report issues on GitHub",
   "main": "src/bin/cli.js",
   "exports": {

package/src/bin/cli.js

@@ -42,6 +42,17 @@ const { resolveChildExitCode } = require('../lib/exit-reason');
 const { printUsage } = require('../lib/usage');
 const { buildDisplayCommand, getCommandName } = require('../lib/shell-utils');
 
+// A closed stdout/stderr pipe (`$ --version | head -1`) must neither crash
+// start nor replace the command's exit code: drop the failed output and keep
+// running, logging and recording as usual (issue #222).
+for (const stream of [process.stdout, process.stderr]) {
+  stream.on('error', (err) => {
+    if (err.code !== 'EPIPE' && err.code !== 'ERR_STREAM_DESTROYED') {
+      throw err;
+    }
+  });
+}
+
 // Configuration from environment variables
 const config = {
   // Disable automatic issue creation (useful for testing)
@@ -538,6 +549,10 @@ async function runWithIsolation(
     if (mode !== 'detached') {
       executionRecord.complete(exitCode);
     }
+    if (result.containerRemoved) {
+      // The cleanup policy removed the container (issue #225).
+      require('../lib/container-removal').markContainerRemoved(executionRecord);
+    }
     try {
       store.save(
         mode === 'detached'

package/src/lib/args-parser-queries.js

@@ -7,7 +7,8 @@
  *
  * Query modes:   --status, --list [--running]
  * Control modes: --upload-log, --stop, --terminate, --attach [--read-only],
- *                --resume [-- <command>], --resume-all, --cleanup
+ *                --resume [-- <command>], --resume-all,
+ *                --cleanup [<uuid-or-session-name>]
  */
 
 /**
@@ -37,6 +38,7 @@ function createQueryOptionDefaults() {
     resumeAll: false, // Re-attach/resume every execution still marked running
     cleanup: false, // Clean up stale "executing" records
     cleanupDryRun: false, // Show what would be cleaned without cleaning
+    cleanupTarget: null, // UUID/session name whose containers and images --cleanup removes
   };
 }
 
@@ -75,15 +77,45 @@ const FLAG_OPTIONS = {
   '--resume-all': (options) => {
     options.resumeAll = true;
   },
-  '--cleanup': (options) => {
-    options.cleanup = true;
-  },
-  '--cleanup-dry-run': (options) => {
-    options.cleanup = true;
-    options.cleanupDryRun = true;
-  },
 };
 
+/**
+ * `--cleanup` and `--cleanup-dry-run` take an optional execution: without one
+ * they clean up stale records, with one they remove that execution's Docker
+ * containers and snapshot images (issue #224).
+ */
+const CLEANUP_OPTIONS = {
+  '--cleanup': false,
+  '--cleanup-dry-run': true,
+};
+
+function parseCleanupOption(args, index, options) {
+  const arg = args[index];
+  const equals = arg.indexOf('=');
+  const name = equals === -1 ? arg : arg.slice(0, equals);
+  if (!(name in CLEANUP_OPTIONS)) {
+    return 0;
+  }
+  options.cleanup = true;
+  options.cleanupDryRun = options.cleanupDryRun || CLEANUP_OPTIONS[name];
+  if (equals !== -1) {
+    const value = arg.slice(equals + 1);
+    if (!value) {
+      throw new Error(
+        `Option ${name} requires a UUID or session name argument`
+      );
+    }
+    options.cleanupTarget = value;
+    return 1;
+  }
+  const next = args[index + 1];
+  if (next !== undefined && !next.startsWith('-')) {
+    options.cleanupTarget = next;
+    return 2;
+  }
+  return 1;
+}
+
 function hasValue(value) {
   return value !== null && value !== undefined;
 }
@@ -98,6 +130,11 @@ function hasValue(value) {
 function parseQueryOption(args, index, options) {
   const arg = args[index];
 
+  const cleanupConsumed = parseCleanupOption(args, index, options);
+  if (cleanupConsumed > 0) {
+    return cleanupConsumed;
+  }
+
   const flag = FLAG_OPTIONS[arg];
   if (flag) {
     flag(options);

package/src/lib/args-parser.js

@@ -37,7 +37,7 @@
  * --read-only                      Follow --attach output without forwarding stdin
  * --resume <uuid-or-session> [-- cmd] Resume a stored execution (optionally with a new command)
  * --resume-all                     Re-attach or reconcile every execution still marked running
- * --cleanup                        Clean up stale "executing" records (processes that crashed or were killed)
+ * --cleanup [<uuid-or-session>]    Clean up stale "executing" records, or one execution's containers and snapshot images
  * --cleanup-dry-run                Show stale records that would be cleaned up (without actually cleaning)
  */
 
@@ -46,6 +46,7 @@ const dockerNetworkOptions = require('./docker-network-options');
 const dockerRecoveryOptions = require('./docker-recovery-options');
 const cpuPenalty = require('./cpu-penalty');
 const dockerResourceOptions = require('./docker-resource-options');
+const snapshotResumeOptions = require('./snapshot-resume-options');
 const { parseSequence, isSequence } = require('./sequence-parser');
 const { buildCommandString } = require('./shell-utils');
 const {
@@ -215,6 +216,7 @@ function parseArgs(args) {
     shell: 'auto', // Shell to use in isolation environments: auto, bash, zsh, sh
     useCommandStream: false, // Use command-stream library for command execution
     ...createQueryOptionDefaults(),
+    ...snapshotResumeOptions.createSnapshotOptionDefaults(),
   };
 
   let commandArgs = [];
@@ -566,6 +568,12 @@ function parseOption(args, index, options) {
     return 1;
   }
 
+  // Snapshot resume options (--snapshot-wait, --remove-original-after, ...)
+  const snapshotConsumed = snapshotResumeOptions.parse(args, index, options);
+  if (snapshotConsumed > 0) {
+    return snapshotConsumed;
+  }
+
   // Query/control options (--status, --list, --attach, --resume, ...)
   const queryConsumed = parseQueryOption(args, index, options);
   if (queryConsumed > 0) {
@@ -820,6 +828,7 @@ function validateOptions(options) {
     throw new Error('--keep-user option is only valid with --isolated-user');
   }
 
+  snapshotResumeOptions.validate(options);
   validateQueryOptions(options);
 
   // Validate shell option

package/src/lib/container-removal.js

@@ -0,0 +1,163 @@
+/**
+ * Removed-container bookkeeping for `--status` (issue #225).
+ *
+ * A kept docker container can be removed later (by hand, by an orchestrator,
+ * or by start-command's own cleanup policy). `docker inspect` then exits 1
+ * with `No such object` / `No such container`, which used to be folded into
+ * the same `null` as "docker could not be queried at all", so `--status` kept
+ * reporting the old `containerId` as if the container still held its disk.
+ *
+ * This module tells the two failures apart and records the removal on the
+ * execution record as `containerState: removed` plus
+ * `containerRemovedObservedAt`. `containerId` is kept for history.
+ */
+
+const { spawnSync } = require('child_process');
+
+/** `containerState` of a record whose container no longer exists. */
+const CONTAINER_STATE_REMOVED = 'removed';
+
+/**
+ * Docker's answer for a container that does not exist:
+ * `Error: No such object: <name>` (`docker inspect`, Docker 29 prints it
+ * lower-case) and `Error response from daemon: No such container: <name>`
+ * (`docker container inspect`). A daemon that cannot be reached prints
+ * "Cannot connect" / "failed to connect", which must not match.
+ */
+const DOCKER_NOT_FOUND_PATTERN = /no such (object|container)/i;
+
+/**
+ * Environment variable the detached watcher sets for the finalizer when its
+ * own `docker rm -f` removed the container.
+ */
+const CONTAINER_REMOVED_ENV = 'START_INTERNAL_CONTAINER_REMOVED';
+
+/**
+ * @param {{status?: number|null, stderr?: string|Buffer, error?: Error}} result
+ *   `spawnSync` result of a `docker inspect`
+ * @returns {boolean} True only when docker answered that the object is unknown
+ */
+function isDockerNotFound(result) {
+  if (!result || result.error || result.status === 0) {
+    return false;
+  }
+  return DOCKER_NOT_FOUND_PATTERN.test(String(result.stderr || ''));
+}
+
+/**
+ * Run `docker inspect -f <format> <target>`.
+ * @returns {{stdout: string|null, notFound: boolean}} `stdout` is null when
+ *   the inspect failed; `notFound` says the failure was "no such container"
+ */
+function dockerInspect(dockerCommand, spawnOptions, format, target) {
+  const result = spawnSync(
+    dockerCommand,
+    ['inspect', '-f', format, target],
+    spawnOptions
+  );
+  if (result.error || result.status !== 0 || !result.stdout) {
+    return { stdout: null, notFound: isDockerNotFound(result) };
+  }
+  return { stdout: String(result.stdout), notFound: false };
+}
+
+/**
+ * Set `containerState: removed` on a record, keeping an earlier observation
+ * time when there is one.
+ * @param {Object} record - Execution record (mutated)
+ * @param {string} [observedAt] - When the removal was observed
+ * @returns {Object} The same record
+ */
+function markContainerRemoved(record, observedAt = new Date().toISOString()) {
+  record.containerState = CONTAINER_STATE_REMOVED;
+  record.containerRemovedObservedAt =
+    record.containerRemovedObservedAt || observedAt;
+  return record;
+}
+
+/**
+ * Whether a finished detached docker record should be checked for removal.
+ * Executing records are never touched: a container that is not visible yet
+ * on a slow Docker-in-Docker host is not a removed one (issue #136).
+ */
+function shouldProbeRemoval(record) {
+  const opts = (record && record.options) || {};
+  return (
+    record.status === 'executed' &&
+    opts.isolated === 'docker' &&
+    opts.isolationMode === 'detached' &&
+    Boolean(opts.sessionName || opts.containerId) &&
+    record.containerState !== CONTAINER_STATE_REMOVED
+  );
+}
+
+/**
+ * Attach `containerState: removed` to a finished detached docker record whose
+ * container no longer exists. The container id is preferred over the session
+ * name, so a later container that reuses the name is not mistaken for it.
+ *
+ * @param {Object} record - Enriched execution record
+ * @param {{dockerCommand: string, spawnOptions: Object,
+ *   sessionNotFound?: boolean}} probe - How to run docker, and whether the
+ *   status probe by session name already answered "no such container"
+ * @returns {Object} The record, or a copy carrying the removal
+ */
+function attachContainerRemoval(record, probe) {
+  if (!shouldProbeRemoval(record)) {
+    return record;
+  }
+  const containerId = record.options.containerId;
+  const notFound = containerId
+    ? dockerInspect(
+        probe.dockerCommand,
+        probe.spawnOptions,
+        '{{.Id}}',
+        containerId
+      ).notFound
+    : probe.sessionNotFound === true;
+  if (!notFound) {
+    return record;
+  }
+  const copy = Object.assign(
+    Object.create(Object.getPrototypeOf(record)),
+    record
+  );
+  return markContainerRemoved(copy);
+}
+
+/**
+ * Persist a removal `--status` observed, best effort: a read-only query must
+ * never fail because the store could not be written.
+ * @param {Object} store - ExecutionStore
+ * @param {Object} stored - Record as read from the store
+ * @param {Object} enriched - Record as reported
+ */
+function persistContainerRemoval(store, stored, enriched) {
+  if (
+    !store ||
+    enriched.containerState !== CONTAINER_STATE_REMOVED ||
+    stored.containerState === CONTAINER_STATE_REMOVED ||
+    typeof store.markContainerRemoved !== 'function'
+  ) {
+    return;
+  }
+  try {
+    store.markContainerRemoved(
+      stored.uuid,
+      enriched.containerRemovedObservedAt
+    );
+  } catch {
+    // The removal is still reported; it is simply observed again next time.
+  }
+}
+
+module.exports = {
+  CONTAINER_REMOVED_ENV,
+  CONTAINER_STATE_REMOVED,
+  DOCKER_NOT_FOUND_PATTERN,
+  attachContainerRemoval,
+  dockerInspect,
+  isDockerNotFound,
+  markContainerRemoved,
+  persistContainerRemoval,
+};

package/src/lib/detached-finalize.js

@@ -63,6 +63,8 @@ const WATCHER_LOST_CONTAINER = 'watcher-lost-container';
  * @param {boolean|string|null} [options.running] - `.State.Running`
  * @param {string|null} [options.cgroupMemory] - Last cgroup v2 sample of the
  *   watcher, `<memory.max> <memory.peak> <oom> <oom_kill>` (issue #182)
+ * @param {boolean|string|null} [options.containerRemoved] - The watcher's own
+ *   `docker rm -f` removed the container (issue #225)
  * @returns {{updated: boolean, reason?: string, record?: Object}} Outcome
  */
 function finalizeDetachedExecution(options = {}) {
@@ -96,9 +98,23 @@ function finalizeDetachedExecution(options = {}) {
   ) {
     return { updated: false, reason: 'stale-attempt' };
   }
+  const containerRemoved = normalizeBoolean(options.containerRemoved) === true;
   if (record.status === 'executed' && record.endTime) {
     // Already finalized (e.g. a `--status` query got there first, or the
-    // watcher ran twice after a resume). Nothing to correct.
+    // watcher ran twice after a resume). Only the removal can still be new.
+    if (containerRemoved && record.containerState !== 'removed') {
+      try {
+        const marked = store.markContainerRemoved(
+          record.uuid,
+          new Date().toISOString()
+        );
+        if (marked) {
+          return { updated: true, reason: 'container-removed', record: marked };
+        }
+      } catch {
+        // Bookkeeping only: see the exit-0 contract of `main`.
+      }
+    }
     return { updated: false, reason: 'already-final', record };
   }
 
@@ -171,6 +187,11 @@ function finalizeDetachedExecution(options = {}) {
   if (containerError) {
     record.options = { ...(record.options || {}), containerError };
   }
+  if (containerRemoved) {
+    // The cleanup policy removed the container: say so, and keep its id as
+    // history (issue #225).
+    require('./container-removal').markContainerRemoved(record);
+  }
 
   const {
     appendLifecycle,
@@ -291,7 +312,9 @@ function reconcileFinalizedRecord(store, record) {
  */
 function buildDetachedFinalizeSnippet(executionId, attemptNumber = null) {
   const v = SHELL_VARS;
+  const { CONTAINER_REMOVED_ENV } = require('./container-removal');
   return (
+    `${CONTAINER_REMOVED_ENV}="$${v.removed}" ` +
     `${shellQuote(process.execPath)} ${shellQuote(__filename)} ` +
     `${shellQuote(executionId)} "$${v.exit}" "$${v.oom}" ` +
     `"$${v.started}" "$${v.finished}" "$${v.error}" "$${v.running}" ` +
@@ -301,7 +324,9 @@ function buildDetachedFinalizeSnippet(executionId, attemptNumber = null) {
 
 /**
  * Entry point used by the detached watcher:
- *   <runtime> detached-finalize.js <uuid> <exit> <oom> <started> <finished> <error> <running> <cgroup>
+ *   <runtime> detached-finalize.js <uuid> <exit> <oom> <started> <finished> <error> <running> <cgroup> [attempt]
+ * with `START_INTERNAL_CONTAINER_REMOVED=true` when the watcher removed the
+ * container (issue #225).
  * Always exits 0 — a bookkeeping failure must never turn into a visible error
  * in a log the user is reading for the command's own output.
  * @param {string[]} argv - Positional arguments
@@ -340,6 +365,8 @@ function main(argv) {
       running,
       cgroupMemory,
       attemptNumber: attemptNumber || null,
+      containerRemoved:
+        process.env[require('./container-removal').CONTAINER_REMOVED_ENV],
     });
   } catch {
     // Deliberately silent: see the exit-0 contract above.

package/src/lib/docker-cleanup.js

@@ -381,7 +381,10 @@ function buildDetachedDockerCompletionScript(
   }
   // Everything that assumes the container has exited: cleanup, footer and
   // finalization. Guarded as a whole by `.State.Running` below.
-  const exited = [];
+  // `removed` tells the finalizer whether the cleanup policy removed the
+  // container (issue #225).
+  const removed = SHELL_VARS.removed;
+  const exited = [`${removed}=false`];
 
   if (logPath) {
     const quotedLogPath = shellQuote(logPath);
@@ -394,7 +397,7 @@ function buildDetachedDockerCompletionScript(
       quotedLogPath
     );
     const memory = buildCgroupMemoryLogSnippet(quotedLogPath);
-    const remove = `${imageCleanup.capture}; if docker rm -f ${quotedName} >> ${quotedLogPath} 2>&1; then ${imageCleanup.remove}; fi; ${removalNote}; ${memory}`;
+    const remove = `${imageCleanup.capture}; if docker rm -f ${quotedName} >> ${quotedLogPath} 2>&1; then ${removed}=true; ${imageCleanup.remove}; fi; ${removalNote}; ${memory}`;
     const keep = `${postMortem}; ${memory}; ${buildDockerKeptLogSnippet(containerName, quotedLogPath)}`;
 
     const since = watcherOptions.since
@@ -444,14 +447,14 @@ function buildDetachedDockerCompletionScript(
     parts.push(buildDockerStateSnippet(containerName));
     if (policy === DOCKER_CONTAINER_CLEANUP_POLICY.ALWAYS) {
       exited.push(
-        `${imageCleanup.capture}; if docker rm -f ${quotedName} >/dev/null 2>&1; then ${imageCleanup.remove}; fi`
+        `${imageCleanup.capture}; if docker rm -f ${quotedName} >/dev/null 2>&1; then ${removed}=true; ${imageCleanup.remove}; fi`
       );
     } else if (
       policy === DOCKER_CONTAINER_CLEANUP_POLICY.DEFAULT ||
       policy === DOCKER_CONTAINER_CLEANUP_POLICY.KEEP_ON_FAIL
     ) {
       exited.push(
-        `if ${buildSuccessfulNonOomCondition()}; then ${imageCleanup.capture}; if docker rm -f ${quotedName} >/dev/null 2>&1; then ${imageCleanup.remove}; fi; fi`
+        `if ${buildSuccessfulNonOomCondition()}; then ${imageCleanup.capture}; if docker rm -f ${quotedName} >/dev/null 2>&1; then ${removed}=true; ${imageCleanup.remove}; fi; fi`
       );
     }
   }

package/src/lib/docker-post-mortem.js

@@ -139,6 +139,8 @@ const SHELL_VARS = {
   exitText: '__start_command_exit_text',
   lifetime: '__start_command_lifetime',
   running: '__start_command_running',
+  /** `true` once the watcher's own `docker rm -f` succeeded (issue #225). */
+  removed: '__start_command_removed',
 };
 
 /** `docker inspect -f` template collecting every post-mortem fact at once. */