start-command 0.38.0 → 0.39.1

package/test/regression-236.js

@@ -0,0 +1,250 @@
+// Own the snapshot lock when run without the default preloads.
+require('./setup/private-snapshot-lock');
+const { it, expect } = require('bun:test');
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+const { parseArgs } = require('../src/lib/args-parser');
+const {
+  resolveResourceOptions,
+} = require('../src/lib/docker-resource-options');
+const {
+  buildDockerRuntimeArgs,
+  buildDockerRuntimeMetadata,
+} = require('../src/lib/docker-runtime-args');
+const {
+  ExecutionRecord,
+  ExecutionStore,
+} = require('../src/lib/execution-store');
+const { formatRecord } = require('../src/lib/status-formatter');
+const { resumeExecution } = require('../src/lib/execution-resume');
+
+it('parses the reported command and forwards both limits to Docker', () => {
+  for (const flag of [
+    ['--memory-reservation', '64m'],
+    ['--memory-reservation=64m'],
+  ]) {
+    const options = parseArgs([
+      '--isolated',
+      'docker',
+      '--image',
+      'alpine:3.20',
+      '--memory',
+      '256m',
+      ...flag,
+      '--',
+      'echo',
+      'hi',
+    ]).wrapperOptions;
+    expect(options.memoryReservation).toBe('64m');
+    const resolved = resolveResourceOptions(options);
+    const argv = buildDockerRuntimeArgs(resolved);
+    expect(argv).toContain('--memory=268435456');
+    expect(argv).toContain('--memory-reservation=67108864');
+    const metadata = buildDockerRuntimeMetadata({ ...options, ...resolved });
+    const record = ExecutionRecord.fromObject(
+      new ExecutionRecord({ command: 'echo hi', options: metadata }).toObject()
+    );
+    expect(record.options.resourceLimitSpecs.memoryReservation).toBe('64m');
+    expect(record.options.resolvedLimits.memoryReservation).toBe(67108864);
+    for (const format of ['json', 'text', 'links-notation']) {
+      expect(formatRecord(record, format)).toContain('memoryReservation');
+      expect(formatRecord(record, format)).toContain('67108864');
+    }
+  }
+});
+
+it('resolves reservation percentages from daemon RAM once, including ranges', () => {
+  let calls = 0;
+  const resolved = resolveResourceOptions(
+    { memory: '50%', memoryReservation: '10%-20%' },
+    (_, argv) => {
+      calls++;
+      expect(argv[0]).toBe('info');
+      return {
+        success: true,
+        stdout: JSON.stringify({ MemTotal: 10 * 1024 ** 3, NCPU: 8 }),
+      };
+    },
+    () => 0.5
+  );
+  expect(calls).toBe(1);
+  expect(resolved.resolvedLimits.memoryReservation).toBe(1610612736);
+  expect(resolved.resourceLimitSpecs.memoryReservation).toBe('10%-20%');
+  expect(
+    resolveResourceOptions({ resourceLimits: resolved.resourceLimits }, () => {
+      throw Error('must preserve resolved limits');
+    }).resourceLimits
+  ).toEqual(resolved.resourceLimits);
+});
+
+it('rejects equal or larger reservations against new and inherited hard caps', () => {
+  for (const options of [
+    { memory: '256m', memoryReservation: '512m' },
+    { memory: '256m', memoryReservation: '256m' },
+    { memoryReservation: '512m', resourceLimits: ['--memory=256m'] },
+    {
+      memory: '32m',
+      resourceLimits: ['--memory=256m', '--memory-reservation=64m'],
+    },
+  ]) {
+    expect(() => resolveResourceOptions(options)).toThrow(
+      'memory-reservation must be less than memory'
+    );
+  }
+  expect(
+    resolveResourceOptions({ memoryReservation: '64m' }).resourceLimits
+  ).toEqual(['--memory-reservation=67108864']);
+});
+
+it('validates reservation syntax, missing values and Docker isolation', () => {
+  for (const value of ['0m', '0%', '101%', '10watts', '20%-10%', '']) {
+    expect(() =>
+      parseArgs([
+        '--isolated',
+        'docker',
+        `--memory-reservation=${value}`,
+        '--',
+        'true',
+      ])
+    ).toThrow();
+  }
+  expect(() =>
+    parseArgs(['--isolated', 'docker', '--memory-reservation'])
+  ).toThrow();
+  expect(() => parseArgs(['--memory-reservation=64m', '--', 'true'])).toThrow(
+    'Resource options require Docker'
+  );
+});
+
+it('forwards reservations through a nested Docker isolation level', () => {
+  const options = parseArgs([
+    '--isolated',
+    'screen docker',
+    '--memory',
+    '256m',
+    '--memory-reservation',
+    '64m',
+    '--',
+    'work',
+  ]).wrapperOptions;
+  expect(
+    require('../src/lib/command-builder').buildNextLevelCommand(options, 'work')
+  ).toContain('--memory-reservation 64m');
+});
+
+for (const mode of ['start', 'snapshot', 'missing', 'override', 'invalid']) {
+  it(`preserves or overrides the reservation on ${mode} resume`, async () => {
+    const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'memory-236-'));
+    try {
+      const store = new ExecutionStore({ appFolder: dir, useLinks: false });
+      const resolved = resolveResourceOptions({
+        memory: '256m',
+        memoryReservation: '64m',
+      });
+      const record = new ExecutionRecord({
+        command: 'work',
+        status: 'executed',
+        logPath: path.join(dir, 'run.log'),
+        options: {
+          isolated: 'docker',
+          isolationMode: 'detached',
+          sessionName: 'task',
+          image: 'alpine',
+          ...resolved,
+        },
+      });
+      store.save(record);
+      const calls = [];
+      const runner = (_, args) => {
+        calls.push(args);
+        if (args[0] === 'info') {
+          return {
+            success: true,
+            stdout: JSON.stringify({
+              MemTotal: 1024 ** 3,
+              NCPU: 4,
+              DockerRootDir: os.tmpdir(),
+            }),
+          };
+        }
+        if (args.includes('--size')) {
+          return { success: true, stdout: '1024' };
+        }
+        if (args[0] === 'inspect') {
+          return {
+            success: true,
+            stdout: JSON.stringify({
+              Memory: 268435456,
+              MemorySwap: 268435456,
+              MemoryReservation: 67108864,
+            }),
+          };
+        }
+        return { success: true, stdout: 'id' };
+      };
+      const result = await resumeExecution(store, record.uuid, {
+        command: mode === 'snapshot' ? 'new' : null,
+        resourceOptions:
+          mode === 'override'
+            ? { memoryReservation: '32m' }
+            : mode === 'invalid'
+              ? { memory: '32m' }
+              : {},
+        runIsolated: async (_backend, _command, options) => {
+          calls.push(['run', ...options.resourceLimits]);
+          return { success: true };
+        },
+        snapshotOptions: { freeBytes: () => 100 * 1024 ** 3 },
+        runner,
+        probe: () => ({
+          alive: false,
+          state: mode === 'missing' ? 'missing' : 'stopped',
+        }),
+        startWatcher: () => true,
+      });
+      expect(result.success).toBe(mode !== 'invalid');
+      if (mode === 'invalid') {
+        expect(result.error).toContain(
+          'memory-reservation must be less than memory'
+        );
+        expect(
+          calls.some((a) => ['start', 'run', 'update', 'commit'].includes(a[0]))
+        ).toBe(false);
+      } else {
+        const current = store.get(record.uuid);
+        const bytes = mode === 'override' ? 33554432 : 67108864;
+        expect(current.options.resolvedLimits.memoryReservation).toBe(bytes);
+        expect(current.options.resourceLimitSpecs.memoryReservation).toBe(
+          mode === 'override' ? '32m' : '64m'
+        );
+        if (mode === 'override') {
+          expect(calls.find((a) => a[0] === 'update')).toContain(
+            `--memory-reservation=${bytes}`
+          );
+          expect(calls.findIndex((a) => a[0] === 'update')).toBeLessThan(
+            calls.findIndex((a) => a[0] === 'start')
+          );
+        }
+        if (mode === 'snapshot' || mode === 'missing') {
+          const limits = calls.find((a) => a[0] === 'run');
+          expect(
+            limits.some((s) => /^--memory-reservation=(64m|67108864)$/.test(s))
+          ).toBe(true);
+        }
+      }
+    } finally {
+      fs.rmSync(dir, { recursive: true, force: true });
+    }
+  });
+}
+
+it('refuses reservation overrides when resuming a non-Docker execution', () => {
+  expect(() =>
+    require('../src/lib/resume-resources').prepareResumeResources(
+      { options: { isolated: 'screen' } },
+      { state: 'stopped' },
+      { memoryReservation: '64m' }
+    )
+  ).toThrow('Resource overrides require a Docker execution');
+});

package/test/regression-238.js

@@ -0,0 +1,129 @@
+const { describe, it, expect } = require('bun:test');
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+const { spawnSync } = require('child_process');
+const {
+  buildCgroupSamplerStartSnippet,
+  buildCgroupSamplerStopSnippet,
+  buildCgroupMemoryLogSnippet,
+} = require('../src/lib/cgroup-memory');
+
+function sample(mode, interval, clockStep = '5') {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'sampler-238-'));
+  try {
+    const script = path.join(dir, 'sample.sh');
+    fs.writeFileSync(
+      script,
+      [
+        buildCgroupSamplerStartSnippet("task '238'"),
+        'wait "$__start_command_cgroup_sampler"',
+        mode === 'final'
+          ? 'printf "oom 3\\noom_kill 3\\n" > "$START_COMMAND_CGROUP_ROOT/docker/238abc/memory.events"'
+          : ':',
+        buildCgroupSamplerStopSnippet(),
+        'printf "sample=%s\\n" "$__start_command_cgroup"',
+        buildCgroupMemoryLogSnippet('"$TMPDIR/memory.log"'),
+        'cat "$TMPDIR/memory.log" "$TMPDIR/calls"',
+        'sed "s/^/sleep=/" "$TMPDIR/sleeps"',
+        // Stop must remove every temporary sample file.
+        'for file in "$TMPDIR"/start-command-cgroup*; do [ ! -e "$file" ] || exit 1; done',
+      ].join('; ')
+    );
+    const env = { ...process.env, START_SAMPLER_CLOCK_STEP: clockStep };
+    delete env.START_COMMAND_CGROUP_SAMPLE_INTERVAL_SECONDS;
+    if (interval !== undefined) {
+      env.START_COMMAND_CGROUP_SAMPLE_INTERVAL_SECONDS = interval;
+    }
+    const result = spawnSync(
+      '/bin/sh',
+      [
+        path.resolve(
+          __dirname,
+          '../../experiments/issue-238-cgroup-polling.sh'
+        ),
+        mode,
+        script,
+      ],
+      { encoding: 'utf8', env }
+    );
+    expect(result.status).toBe(0);
+    return result.stdout;
+  } finally {
+    fs.rmSync(dir, { recursive: true, force: true });
+  }
+}
+
+const count = (output, text) =>
+  output.split('\n').filter((line) => line.includes(text)).length;
+const posix = process.platform === 'win32' ? describe.skip : describe;
+
+posix('issue #238: bounded Docker cgroup sampling', () => {
+  it('defaults to five seconds and inspects local liveness only after removal', () => {
+    const output = sample('local');
+    expect(count(output, 'sleep=5')).toBe(3);
+    expect(count(output, '{{.State.Running}}')).toBe(1);
+    expect(count(output, '{{.HostConfig.CgroupnsMode}}')).toBe(1);
+    // A readable local cgroup never starts a remote exec, even after removal
+    // (issue #242): the liveness probe ends the loop instead.
+    expect(count(output, "task '238' sh -c")).toBe(0);
+  });
+
+  it('honors an override in the watcher environment and caches remote namespace', () => {
+    const output = sample('remote', '2');
+    expect(count(output, 'sleep=2')).toBe(3);
+    expect(count(output, '{{.HostConfig.CgroupnsMode}}')).toBe(1);
+    // One long-lived exec streams every round; liveness is probed once, after
+    // the stream ends (issue #242).
+    expect(count(output, '{{.State.Running}}')).toBe(1);
+    expect(count(output, "task '238' sh -c")).toBe(1);
+    expect(output).toContain(
+      'memory.max=67108864 memory.peak=33554432 oom=1 oom_kill=3'
+    );
+  });
+
+  it('rejects zero, negative, fractional, malformed and unsafe intervals', () => {
+    for (const interval of [
+      '',
+      '0',
+      '000',
+      '-1',
+      '0.5',
+      'nope',
+      '1; exit 7',
+      '99999999999999999999999999',
+    ]) {
+      expect(count(sample('remote', interval), 'sleep=5')).toBe(3);
+    }
+  });
+
+  it('keeps unavailable diagnostics and the last sample across remote outages', () => {
+    for (const mode of ['shared', 'no-shell', 'outage']) {
+      const output = sample(mode);
+      expect(count(output, '{{.HostConfig.CgroupnsMode}}')).toBe(1);
+      // A shared namespace or a missing shell ends sampling at once; an outage
+      // falls back to per-round execs, each followed by a liveness probe.
+      expect(count(output, '{{.State.Running}}')).toBe(
+        mode === 'outage' ? 4 : 0
+      );
+      expect(count(output, "task '238' sh -c")).toBe(
+        { shared: 0, 'no-shell': 1, outage: 4 }[mode]
+      );
+      if (mode === 'outage') {
+        expect(output).toContain('sample=67108864 33554432 1 3 0 100');
+      } else {
+        expect(output).toContain('Memory:     unavailable');
+        expect(output).toContain(
+          mode === 'shared' ? 'namespace=host' : 'sh: not found'
+        );
+      }
+    }
+  });
+
+  it('preserves a final read without treating a five-second counter gap as recent OOM', () => {
+    expect(sample('final')).toContain('sample=67108864 33554432 3 3 0 115');
+    expect(sample('final', '1', '1')).toContain(
+      'sample=67108864 33554432 3 3 103 103'
+    );
+  });
+});

package/test/regression-241.js

@@ -0,0 +1,254 @@
+/**
+ * Regression tests for issue #241: a daemon restart was attributed only from
+ * `journalctl -u docker.service`. With a mounted docker.sock
+ * (Docker-outside-of-Docker) there is no host journal, so a task that the
+ * restart force-killed ended with `Exit evidence: unavailable`.
+ *
+ * `docker events` cannot help (its buffer does not survive the restart), but
+ * `State.FinishedAt` / `State.StartedAt` of the other containers do.
+ */
+
+const { describe, it, expect, afterEach } = require('bun:test');
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+
+const evidence = require('../src/lib/exit-evidence');
+const { resolveExitReason } = require('../src/lib/exit-reason');
+
+const TASK = 'a'.repeat(64);
+const PEER = 'b'.repeat(64);
+const FINISHED = '2026-10-21T10:05:00.123456789Z';
+const state = (id, finished, started, exit, oom = 'false') =>
+  `${id} ${finished} ${started} ${exit} ${oom}`;
+// Two containers that finished in the same second, the shape of a restart.
+const TOGETHER = [
+  state(TASK, FINISHED, '2026-10-21T10:00:00Z', 137),
+  state(PEER, '2026-10-21T10:05:00.9Z', '2026-10-21T09:00:00Z', 137),
+].join('\n');
+
+/** Runner answering like a daemon reached through a mounted socket. */
+function socketOnlyRunner(states, calls = []) {
+  return (command, args) => {
+    calls.push([command, ...args]);
+    if (command === 'journalctl') {
+      // No host journal inside the container.
+      return { success: false, stdout: '' };
+    }
+    if (args[0] === 'inspect' && args[2] === '{{.Id}}') {
+      return { success: true, stdout: `${TASK}\n` };
+    }
+    if (args[0] === 'events') {
+      return { success: true, stdout: '' };
+    }
+    if (args[0] === 'ps') {
+      return { success: true, stdout: `${TASK}\n${PEER}\n` };
+    }
+    if (args[0] === 'inspect') {
+      return { success: true, stdout: states };
+    }
+    return { success: false, stdout: '' };
+  };
+}
+
+const collect = (states, extra = {}, calls) =>
+  evidence.collectExitEvidence(
+    {
+      containerName: 'task',
+      sample: '',
+      finishedAt: FINISHED,
+      exitCode: 137,
+      oomKilled: false,
+      ...extra,
+    },
+    socketOnlyRunner(states, calls)
+  );
+
+describe('issue #241: daemon-restart evidence without the host journal', () => {
+  it('attributes two containers that finished in the same second to the daemon', () => {
+    const calls = [];
+    const line = collect(TOGETHER, {}, calls);
+    expect(line).toBe(evidence.DAEMON_RESTART_API);
+    expect(calls).toContainEqual(['docker', 'ps', '-a', '-q', '--no-trunc']);
+    // The task itself is never its own witness.
+    const inspect = calls.find(
+      (call) => call[1] === 'inspect' && call[3] !== '{{.Id}}'
+    );
+    expect(inspect.slice(4)).toEqual([PEER]);
+    expect(
+      resolveExitReason({ exitCode: 137, oomKilled: false, logTail: line })
+    ).toBe('killed (docker daemon restart)');
+  });
+
+  it('reports which source supported the attribution', () => {
+    expect(evidence.fromLog(evidence.DAEMON_RESTART_API)).toEqual({
+      mainOom: false,
+      daemonRestart: true,
+      daemonRestartSource: 'api',
+    });
+    expect(evidence.fromLog(evidence.DAEMON_RESTART)).toEqual({
+      mainOom: false,
+      daemonRestart: true,
+      daemonRestartSource: 'journal',
+    });
+    expect(evidence.fromLog(evidence.MAIN_OOM)).toEqual({
+      mainOom: true,
+      daemonRestart: false,
+    });
+  });
+
+  it('counts a container restarted after it finished, whose ExitCode moby reset to 0', () => {
+    const restarted = state(
+      PEER,
+      '2026-10-21T10:04:52Z',
+      '2026-10-21T10:05:20Z',
+      0
+    );
+    expect(evidence.daemonRestartApiEvidence(restarted, TASK, FINISHED)).toBe(
+      true
+    );
+  });
+
+  for (const [name, peer] of [
+    [
+      'an OOM-killed peer',
+      state(PEER, FINISHED, '2026-10-21T09:00:00Z', 137, 'true'),
+    ],
+    [
+      'a peer that finished 30 s apart',
+      state(PEER, '2026-10-21T10:05:30.2Z', '2026-10-21T09:00:00Z', 137),
+    ],
+    [
+      'a peer that exited on its own',
+      state(PEER, FINISHED, '2026-10-21T09:00:00Z', 0),
+    ],
+    [
+      'a peer that never ran',
+      state(PEER, '0001-01-01T00:00:00Z', '0001-01-01T00:00:00Z', 0),
+    ],
+    [
+      'only the task itself',
+      state(TASK, FINISHED, '2026-10-21T09:00:00Z', 137),
+    ],
+    ['malformed output', 'Error: No such object: gone'],
+  ]) {
+    it(`does not attribute ${name}`, () => {
+      expect(evidence.daemonRestartApiEvidence(peer, TASK, FINISHED)).toBe(
+        false
+      );
+      expect(collect(peer)).toBe(
+        'Exit evidence: unavailable (no attributed exit-time OOM or daemon restart evidence)'
+      );
+    });
+  }
+
+  it('also works for a remote daemon, where no journal is consulted', () => {
+    const original = process.env.DOCKER_HOST;
+    process.env.DOCKER_HOST = 'tcp://docker.example:2376';
+    try {
+      const calls = [];
+      expect(collect(TOGETHER, {}, calls)).toBe(evidence.DAEMON_RESTART_API);
+      expect(calls.some(([command]) => command === 'journalctl')).toBe(false);
+    } finally {
+      if (original === undefined) {
+        delete process.env.DOCKER_HOST;
+      } else {
+        process.env.DOCKER_HOST = original;
+      }
+    }
+  });
+
+  it('keeps journal and attempt-scoped OOM evidence ahead of the API check', () => {
+    const calls = [];
+    const line = collect(
+      TOGETHER,
+      { oomKilled: true, sample: '64 63 3 2 1792577100 1792577100' },
+      calls
+    );
+    expect(line).toBe(evidence.MAIN_OOM);
+    expect(calls.some((call) => call[1] === 'ps')).toBe(false);
+  });
+});
+
+const tempDirs = [];
+afterEach(() => {
+  while (tempDirs.length > 0) {
+    fs.rmSync(tempDirs.pop(), { recursive: true, force: true });
+  }
+});
+
+describe('issue #241: --status collects evidence the lost watcher never wrote', () => {
+  it('reports a daemon restart for a DooD task whose watcher died with the daemon', () => {
+    if (process.platform === 'win32') {
+      console.log('  Skipping: the docker stub needs a POSIX /bin/sh');
+      return;
+    }
+    const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'evidence-241-'));
+    tempDirs.push(dir);
+    const docker = path.join(dir, 'docker');
+    fs.writeFileSync(
+      docker,
+      [
+        '#!/bin/sh',
+        'case "$1 $3" in',
+        `  "inspect {{.Id}}") echo ${TASK} ;;`,
+        `  "inspect {{.Id}} "*) echo '${state(PEER, '2026-10-21T10:05:00.9Z', '2026-10-21T10:05:30Z', 0)}' ;;`,
+        `  "inspect "*) echo 'false 137 false 2026-10-21T10:00:00Z ${FINISHED}' ;;`,
+        `  "ps "*) printf '%s\\n' ${TASK} ${PEER} ;;`,
+        '  "events "*) ;;',
+        '  *) exit 1 ;;',
+        'esac',
+        '',
+      ].join('\n')
+    );
+    fs.chmodSync(docker, 0o755);
+    // No host journal, as inside a container with a mounted docker.sock.
+    fs.writeFileSync(path.join(dir, 'journalctl'), '#!/bin/sh\nexit 1\n');
+    fs.chmodSync(path.join(dir, 'journalctl'), 0o755);
+    const saved = {
+      START_DOCKER_BIN: process.env.START_DOCKER_BIN,
+      PATH: process.env.PATH,
+      DOCKER_HOST: process.env.DOCKER_HOST,
+    };
+    process.env.START_DOCKER_BIN = docker;
+    process.env.PATH = `${dir}${path.delimiter}${process.env.PATH}`;
+    delete process.env.DOCKER_HOST;
+    try {
+      const {
+        ExecutionStore,
+        ExecutionRecord,
+      } = require('../src/lib/execution-store');
+      const { queryStatus } = require('../src/lib/status-formatter');
+      const store = new ExecutionStore({ appFolder: dir, useLinks: false });
+      const record = new ExecutionRecord({
+        command: 'sleep 600',
+        options: {
+          isolated: 'docker',
+          isolationMode: 'detached',
+          sessionName: 'task-241',
+        },
+      });
+      record.logPath = path.join(dir, 'task.log');
+      // The watcher died before it could log anything after the output.
+      fs.writeFileSync(record.logPath, 'working\n');
+      store.save(record);
+      const status = JSON.parse(queryStatus(store, record.uuid, 'json').output);
+      expect(status.status).toBe('executed');
+      expect(status.exitCode).toBe(137);
+      expect(status.exitReason).toBe('killed (docker daemon restart)');
+      expect(status.options.exitEvidence).toEqual({
+        mainOom: false,
+        daemonRestart: true,
+        daemonRestartSource: 'api',
+      });
+    } finally {
+      for (const [key, value] of Object.entries(saved)) {
+        if (value === undefined) {
+          delete process.env[key];
+        } else {
+          process.env[key] = value;
+        }
+      }
+    }
+  });
+});