start-command 0.32.1 → 0.34.0

package/src/lib/execution-resume.js

@@ -0,0 +1,455 @@
+/**
+ * Resume tracked detached executions (issue #162).
+ *
+ * `--resume <id>` restarts a stored execution, and `--resume <id> -- <command>`
+ * runs a *different* command against the same container filesystem. Both keep
+ * the original execution UUID so `--status`, `--list` and `--upload-log` keep
+ * addressing one logical session across restarts.
+ *
+ * Three strategies, chosen from the probed session state:
+ * - DOCKER_START:    the container still exists and the stored command is
+ *                    re-run by `docker start` (its original entrypoint).
+ * - DOCKER_SNAPSHOT: the container still exists but a new command was given,
+ *                    so its filesystem is committed to an image and a derived
+ *                    container runs the new command. This avoids
+ *                    `docker start -ai`, which would re-run the original
+ *                    entrypoint from scratch.
+ * - RELAUNCH:        nothing is left of the session, so the command is
+ *                    launched again through the stored isolation options.
+ */
+
+const {
+  DOCKER_CONTAINER_CLEANUP_POLICY,
+  getDockerCommand,
+  getDockerContainerCleanupPolicy,
+  startDetachedDockerCompletionWatcher,
+} = require('./docker-cleanup');
+const { escapeForLinksNotation } = require('./output-blocks');
+const { runCommand } = require('./execution-control');
+const { SessionState, probeSession } = require('./session-probe');
+const { getDockerNetworks } = require('./docker-network-lifecycle');
+const {
+  ResumeAllAction,
+  resumeAllExecutions,
+} = require('./execution-resume-all');
+
+/**
+ * Strategies `buildResumePlan` can pick.
+ */
+const ResumeMode = {
+  DOCKER_START: 'docker-start',
+  DOCKER_SNAPSHOT: 'docker-snapshot',
+  RELAUNCH: 'relaunch',
+};
+
+/**
+ * Build the docker image name used to snapshot a container before running a
+ * new command in it. Docker repository names must be lowercase and limited to
+ * `[a-z0-9._-]`, so session names are sanitized.
+ * @param {string} sessionName - Original session name
+ * @param {number} attempt - Resume counter (1-based)
+ * @returns {string} Image reference
+ */
+function buildSnapshotImageName(sessionName, attempt) {
+  const sanitized = String(sessionName)
+    .toLowerCase()
+    .replace(/[^a-z0-9._-]+/g, '-')
+    .replace(/^[-.]+/, '');
+  return `start-command-resume/${sanitized || 'session'}:${attempt}`;
+}
+
+/**
+ * Build the container name for a snapshot-based resume.
+ * @param {string} sessionName - Original session name
+ * @param {number} attempt - Resume counter (1-based)
+ * @returns {string} Container name
+ */
+function buildResumedSessionName(sessionName, attempt) {
+  return `${sessionName}-resume-${attempt}`;
+}
+
+/**
+ * Rebuild the isolation options stored on a record so the command can be
+ * launched again with the same configuration.
+ * @param {object} record - Execution record
+ * @returns {object} Options for `runIsolated`
+ */
+function buildLaunchOptions(record) {
+  const opts = record.options || {};
+  return {
+    image: opts.image || null,
+    session: opts.sessionName,
+    detached: true,
+    endpoint: opts.endpoint || null,
+    user: opts.user || false,
+    keepAlive: opts.keepAlive || false,
+    autoRemoveDockerContainer: opts.autoRemoveDockerContainer || false,
+    alwaysCleanupContainer: opts.alwaysCleanupContainer || false,
+    keepContainer: opts.keepContainer || false,
+    keepContainerOnFail: opts.keepContainerOnFail || false,
+    useCommandStream: opts.useCommandStream || false,
+    shell: opts.shell || 'auto',
+    privileged: opts.privileged || false,
+    env: opts.env || [],
+    volumes: opts.volumes || [],
+    mounts: opts.mounts || [],
+    networks: opts.networks || [],
+    networkAliases: opts.networkAliases || [],
+    // Append to the same log so one logical session keeps one gap-free record.
+    logPath: record.logPath || null,
+  };
+}
+
+/**
+ * Build the steps that start a snapshot-derived container.
+ *
+ * `docker run` can only join one network, so a session that was launched on
+ * several networks is rebuilt the same way `runIsolated` builds it: create the
+ * container, connect the additional networks, then start it. Dropping to a
+ * single `docker run` when there is nothing extra to connect keeps the common
+ * case to one command.
+ *
+ * @param {object} params - {containerArgs, extraNetworks, newSessionName, snapshotImage}
+ * @returns {object[]} Resume steps
+ */
+function buildSnapshotStartSteps({
+  containerArgs,
+  extraNetworks,
+  newSessionName,
+  snapshotImage,
+}) {
+  if (extraNetworks.length === 0) {
+    return [
+      {
+        command: getDockerCommand(),
+        args: ['run', '-d', ...containerArgs],
+        description: `Run the new command in ${newSessionName}`,
+      },
+    ];
+  }
+
+  return [
+    {
+      command: getDockerCommand(),
+      args: ['create', ...containerArgs],
+      description: `Create ${newSessionName} from ${snapshotImage}`,
+    },
+    ...extraNetworks.map((network) => ({
+      command: getDockerCommand(),
+      args: ['network', 'connect', network, newSessionName],
+      description: `Connect ${newSessionName} to network ${network}`,
+    })),
+    {
+      command: getDockerCommand(),
+      args: ['start', newSessionName],
+      description: `Run the new command in ${newSessionName}`,
+    },
+  ];
+}
+
+/**
+ * Decide how a stored execution should be resumed.
+ * @param {object} record - Execution record
+ * @param {?string} newCommand - Replacement command (from `-- <command>`)
+ * @param {object} probe - Result of `probeSession`
+ * @returns {object} Resume plan, or {error}
+ */
+function buildResumePlan(record, newCommand, probe = {}) {
+  const opts = (record && record.options) || {};
+  const backend = opts.isolated;
+  const sessionName = opts.sessionName;
+
+  if (!sessionName) {
+    return {
+      error: 'Execution record does not contain an isolation session name.',
+    };
+  }
+
+  if (opts.isolationMode !== 'detached') {
+    return { error: 'Only detached isolated executions can be resumed.' };
+  }
+
+  if (probe.alive) {
+    return {
+      error:
+        `Session "${sessionName}" is still running. ` +
+        `Use \`$ --attach ${record.uuid}\` to re-enter it, or \`$ --stop ${record.uuid}\` first.`,
+    };
+  }
+
+  const command = newCommand || record.command;
+  if (!command) {
+    return {
+      error: `Execution "${record.uuid}" has no stored command to resume.`,
+    };
+  }
+
+  const attempt = (Number(opts.resumeCount) || 0) + 1;
+
+  if (backend === 'docker' && probe.state === SessionState.STOPPED) {
+    if (!newCommand) {
+      return {
+        mode: ResumeMode.DOCKER_START,
+        backend,
+        sessionName,
+        command,
+        attempt,
+        steps: [
+          {
+            command: getDockerCommand(),
+            args: ['start', sessionName],
+            description: `Start stopped container ${sessionName}`,
+          },
+        ],
+        message: `Resumed detached docker container: ${sessionName}`,
+      };
+    }
+
+    // Lazily required: isolation.js pulls in this module's siblings, so a
+    // top-level require would create a cycle.
+    const { buildDockerRuntimeArgs } = require('./isolation');
+    const snapshotImage = buildSnapshotImageName(sessionName, attempt);
+    const newSessionName = buildResumedSessionName(sessionName, attempt);
+    return {
+      mode: ResumeMode.DOCKER_SNAPSHOT,
+      backend,
+      sessionName,
+      newSessionName,
+      snapshotImage,
+      command,
+      attempt,
+      steps: [
+        {
+          command: getDockerCommand(),
+          args: ['commit', sessionName, snapshotImage],
+          description: `Snapshot container ${sessionName} as ${snapshotImage}`,
+        },
+        ...buildSnapshotStartSteps({
+          containerArgs: [
+            '--name',
+            newSessionName,
+            ...(opts.user ? ['--user', opts.user] : []),
+            ...buildDockerRuntimeArgs(opts),
+            snapshotImage,
+            'sh',
+            '-c',
+            command,
+          ],
+          extraNetworks: getDockerNetworks(opts).slice(1),
+          newSessionName,
+          snapshotImage,
+        }),
+      ],
+      message: `Resumed session in new container ${newSessionName} from snapshot of ${sessionName}`,
+    };
+  }
+
+  return {
+    mode: ResumeMode.RELAUNCH,
+    backend,
+    sessionName,
+    command,
+    attempt,
+    steps: [],
+    launchOptions: buildLaunchOptions(record),
+    message: `Relaunched ${backend} session: ${sessionName}`,
+  };
+}
+
+/**
+ * Format a resume result as links notation.
+ * @param {object} result - Resume result fields
+ * @returns {string} Links notation block
+ */
+function formatResumeResultAsLinksNotation(result) {
+  const lines = [
+    'executionResume',
+    `  identifier ${escapeForLinksNotation(result.identifier)}`,
+    `  uuid ${escapeForLinksNotation(result.uuid)}`,
+    `  mode ${escapeForLinksNotation(result.mode)}`,
+    `  backend ${escapeForLinksNotation(result.backend)}`,
+    `  sessionName ${escapeForLinksNotation(result.sessionName)}`,
+  ];
+  if (result.previousSessionName) {
+    lines.push(
+      `  previousSessionName ${escapeForLinksNotation(result.previousSessionName)}`
+    );
+  }
+  if (result.snapshotImage) {
+    lines.push(
+      `  snapshotImage ${escapeForLinksNotation(result.snapshotImage)}`
+    );
+  }
+  lines.push(`  command ${escapeForLinksNotation(result.command)}`);
+  lines.push(`  message ${escapeForLinksNotation(result.message)}`);
+  return lines.join('\n');
+}
+
+function formatResumeResult(result, outputFormat) {
+  if (outputFormat === 'json') {
+    return JSON.stringify(result, null, 2);
+  }
+  if (outputFormat === 'text') {
+    return [
+      `Resume Mode:   ${result.mode}`,
+      `UUID:          ${result.uuid}`,
+      `Backend:       ${result.backend}`,
+      `Session Name:  ${result.sessionName}`,
+      `Command:       ${result.command}`,
+      result.message,
+    ].join('\n');
+  }
+  return formatResumeResultAsLinksNotation(result);
+}
+
+/**
+ * Apply the resume outcome to the stored record, keeping the original UUID so
+ * one logical session stays addressable across restarts.
+ * @param {object} record - Execution record
+ * @param {object} plan - Resume plan
+ * @param {?string} containerId - New container id, when one was created
+ * @returns {object} The updated record
+ */
+function applyResumeToRecord(record, plan, containerId) {
+  const options = { ...(record.options || {}) };
+  options.resumeCount = plan.attempt;
+  options.resumedAt = new Date().toISOString();
+
+  if (plan.newSessionName) {
+    options.sessionNameHistory = [
+      ...(options.sessionNameHistory || []),
+      options.sessionName,
+    ];
+    options.sessionName = plan.newSessionName;
+  }
+  if (plan.snapshotImage) {
+    options.image = plan.snapshotImage;
+  }
+  if (containerId) {
+    options.containerId = containerId;
+  }
+
+  record.options = options;
+  record.command = plan.command;
+  record.status = 'executing';
+  record.exitCode = null;
+  record.endTime = null;
+  record.exitReason = undefined;
+  record.oomKilled = undefined;
+  return record;
+}
+
+function activeSessionName(plan) {
+  return plan.newSessionName || plan.sessionName;
+}
+
+/**
+ * Resume a tracked execution by UUID or session name.
+ * @param {?object} store - Execution store
+ * @param {string} identifier - UUID or session name
+ * @param {object} deps - {command, outputFormat, probe, runner, startWatcher, runIsolated}
+ * @returns {Promise<{success: boolean, output?: string, error?: string}>}
+ */
+async function resumeExecution(store, identifier, deps = {}) {
+  if (!store) {
+    return { success: false, error: 'Execution tracking is disabled.' };
+  }
+
+  const record = store.get(identifier);
+  if (!record) {
+    return {
+      success: false,
+      error: `No execution found with UUID or session name: ${identifier}`,
+    };
+  }
+
+  const runner = deps.runner || runCommand;
+  const probeFn = deps.probe || ((r) => probeSession(r, runner));
+  const plan = buildResumePlan(record, deps.command || null, probeFn(record));
+  if (plan.error) {
+    return { success: false, error: plan.error };
+  }
+
+  let containerId = null;
+
+  if (plan.mode === ResumeMode.RELAUNCH) {
+    const runIsolated = deps.runIsolated || require('./isolation').runIsolated;
+    const launchResult = await runIsolated(
+      plan.backend,
+      plan.command,
+      plan.launchOptions
+    );
+    if (!launchResult || !launchResult.success) {
+      return {
+        success: false,
+        error: `Failed to relaunch ${plan.backend} session "${plan.sessionName}": ${
+          (launchResult && launchResult.message) || 'unknown error'
+        }`,
+      };
+    }
+    containerId = launchResult.containerId || null;
+  } else {
+    for (const step of plan.steps) {
+      const result = runner(step.command, step.args);
+      if (!result.success) {
+        const detail =
+          (result.stderr || '').trim() ||
+          result.error ||
+          `exit code ${result.status}`;
+        return {
+          success: false,
+          error: `Failed to resume ${plan.backend} session "${plan.sessionName}": ${detail}`,
+        };
+      }
+      containerId = (result.stdout || '').trim() || containerId;
+    }
+
+    // The completion watcher died with the previous run (or with the
+    // supervisor), so a new one must follow the resumed container.
+    const startWatcher =
+      deps.startWatcher || startDetachedDockerCompletionWatcher;
+    startWatcher(
+      activeSessionName(plan),
+      getDockerContainerCleanupPolicy(record.options || {}),
+      record.logPath || null,
+      record.uuid || null
+    );
+  }
+
+  const previousSessionName = plan.newSessionName ? plan.sessionName : null;
+  const updated = applyResumeToRecord(record, plan, containerId);
+  store.save(updated);
+
+  return {
+    success: true,
+    output: formatResumeResult(
+      {
+        identifier,
+        uuid: updated.uuid,
+        mode: plan.mode,
+        backend: plan.backend,
+        sessionName: updated.options.sessionName,
+        previousSessionName,
+        snapshotImage: plan.snapshotImage || null,
+        command: plan.command,
+        message: plan.message,
+      },
+      deps.outputFormat
+    ),
+  };
+}
+
+module.exports = {
+  ResumeAllAction,
+  ResumeMode,
+  resumeAllExecutions,
+  applyResumeToRecord,
+  buildLaunchOptions,
+  buildResumePlan,
+  buildResumedSessionName,
+  buildSnapshotImageName,
+  formatResumeResult,
+  formatResumeResultAsLinksNotation,
+  resumeExecution,
+  DOCKER_CONTAINER_CLEANUP_POLICY,
+};

package/src/lib/execution-store.js

@@ -15,13 +15,14 @@
  * - startTime: Timestamp when execution started
  * - endTime: Timestamp when execution completed (null while executing)
  * - oomKilled: Docker resource-exhaustion signal when available
+ * - memoryExhausted: memory-exhaustion observation derived from the log tail
  * - options: Execution options (isolation mode, etc.)
  */
 
 const fs = require('fs');
 const path = require('path');
 const os = require('os');
-const { execSync, spawnSync } = require('child_process');
+const { execFileSync, spawnSync } = require('child_process');
 const crypto = require('crypto');
 
 // Synchronous wrapper using Bun's native ESM support
@@ -74,8 +75,47 @@ class ExecutionRecord {
     this.logPath = options.logPath || '';
     this.startTime = options.startTime || new Date().toISOString();
     this.endTime = options.endTime || null;
+    // Provenance of `endTime` (issue #170.2). `--status` used to fabricate a
+    // finish time with `new Date()` whenever the record had none, which made an
+    // observation indistinguishable from a real finish time. One of
+    // 'docker-finished-at' | 'log-footer' | 'observed-at'.
+    this.endTimeSource =
+      options.endTimeSource !== undefined ? options.endTimeSource : undefined;
+    // When the end of the execution was *noticed*, for the cases where no real
+    // finish time exists. Always accompanies endTimeSource === 'observed-at'.
+    this.observedAt =
+      options.observedAt !== undefined ? options.observedAt : undefined;
+    // When `cleanupStale()` gave up on a record whose process is gone. This is
+    // a detection time, not a finish time, so it is never written to `endTime`
+    // (issue #170.3).
+    this.staleDetectedAt =
+      options.staleDetectedAt !== undefined
+        ? options.staleDetectedAt
+        : undefined;
+    // `docker inspect .State.StartedAt` of the isolation container, kept next
+    // to `endTime` so the container's lifetime stays reconstructible after the
+    // container itself is gone (issue #171).
+    this.containerStartedAt =
+      options.containerStartedAt !== undefined
+        ? options.containerStartedAt
+        : undefined;
     this.oomKilled =
       options.oomKilled !== undefined ? options.oomKilled : undefined;
+    // Query-time hint explaining an opaque exit code (issue #162). Derived from
+    // the log tail by the status formatter; never persisted as a verdict.
+    this.exitReason =
+      options.exitReason !== undefined ? options.exitReason : undefined;
+    // Query-time memory-exhaustion observation (issue #165). A runtime that
+    // aborts on its own heap limit is invisible to `oomKilled`, so the evidence
+    // the runtime printed into the log is surfaced separately.
+    this.memoryExhausted =
+      options.memoryExhausted !== undefined
+        ? options.memoryExhausted
+        : undefined;
+    this.memoryExhaustedReason =
+      options.memoryExhaustedReason !== undefined
+        ? options.memoryExhaustedReason
+        : undefined;
     this.workingDirectory = options.workingDirectory || process.cwd();
     this.shell = options.shell || process.env.SHELL || '/bin/sh';
     this.platform = options.platform || process.platform;
@@ -106,9 +146,36 @@ class ExecutionRecord {
       startTime: this.startTime,
       endTime: this.endTime,
     };
+    if (this.endTimeSource !== undefined && this.endTimeSource !== null) {
+      obj.endTimeSource = this.endTimeSource;
+    }
+    if (this.observedAt !== undefined && this.observedAt !== null) {
+      obj.observedAt = this.observedAt;
+    }
+    if (this.staleDetectedAt !== undefined && this.staleDetectedAt !== null) {
+      obj.staleDetectedAt = this.staleDetectedAt;
+    }
+    if (
+      this.containerStartedAt !== undefined &&
+      this.containerStartedAt !== null
+    ) {
+      obj.containerStartedAt = this.containerStartedAt;
+    }
     if (this.oomKilled !== undefined && this.oomKilled !== null) {
       obj.oomKilled = this.oomKilled;
     }
+    if (this.exitReason !== undefined && this.exitReason !== null) {
+      obj.exitReason = this.exitReason;
+    }
+    if (this.memoryExhausted !== undefined && this.memoryExhausted !== null) {
+      obj.memoryExhausted = this.memoryExhausted;
+    }
+    if (
+      this.memoryExhaustedReason !== undefined &&
+      this.memoryExhaustedReason !== null
+    ) {
+      obj.memoryExhaustedReason = this.memoryExhaustedReason;
+    }
     Object.assign(obj, {
       workingDirectory: this.workingDirectory,
       shell: this.shell,
@@ -389,7 +456,10 @@ class ExecutionStore {
    */
   execClink(query) {
     try {
-      const result = execSync(`clink '${query}' --db "${this.linksDbPath}"`, {
+      // The query embeds record values verbatim; a shell string would let a
+      // single quote in any of them break out of the argument
+      // (CodeQL js/shell-command-constructed-from-input, issue #168).
+      const result = execFileSync('clink', [query, '--db', this.linksDbPath], {
         encoding: 'utf8',
         timeout: 10000,
         stdio: ['pipe', 'pipe', 'pipe'],
@@ -480,7 +550,8 @@ class ExecutionStore {
 
   /**
    * Get an execution record by UUID or session name
-   * First tries exact UUID match, then falls back to session name lookup
+   * First tries exact UUID match, then the current session name, then any
+   * session name the execution used before it was resumed
    * @param {string} identifier - UUID or session name
    * @returns {ExecutionRecord|null}
    */
@@ -495,7 +566,18 @@ class ExecutionStore {
     const bySessionName = records.find(
       (r) => r.options && r.options.sessionName === identifier
     );
-    return bySessionName || null;
+    if (bySessionName) {
+      return bySessionName;
+    }
+    // Finally, honor session names this record used before it was resumed, so
+    // one logical session stays addressable across restarts (issue #162).
+    const byPreviousSessionName = records.find(
+      (r) =>
+        r.options &&
+        Array.isArray(r.options.sessionNameHistory) &&
+        r.options.sessionNameHistory.includes(identifier)
+    );
+    return byPreviousSessionName || null;
   }
 
   /**
@@ -619,10 +701,15 @@ class ExecutionStore {
             (r) => r.uuid === staleRecord.uuid
           );
           if (index >= 0) {
-            // Mark as executed with exit code -1 to indicate abnormal termination
+            // Mark as executed with exit code -1 to indicate abnormal
+            // termination. The real finish time is unknowable here — the
+            // process vanished without writing one — so the record keeps
+            // `endTime: null` and states *when the loss was detected* instead
+            // of passing cleanup time off as a finish time (issue #170.3).
             currentRecords[index].status = ExecutionStatus.EXECUTED;
             currentRecords[index].exitCode = -1;
-            currentRecords[index].endTime = new Date().toISOString();
+            currentRecords[index].endTime = null;
+            currentRecords[index].staleDetectedAt = new Date().toISOString();
             result.cleaned++;
           }
         }