start-command 0.35.4 → 0.36.0

package/test/regression-190.js

@@ -0,0 +1,111 @@
+const { describe, it, expect } = require('bun:test');
+const { parseArgs } = require('../src/lib/args-parser');
+
+describe('issue #190: launch resource options', () => {
+  it('recognizes resource flags and rejects invalid input before launch', () => {
+    const parsed = parseArgs([
+      '--isolated',
+      'docker',
+      '--image',
+      'alpine',
+      '--memory',
+      '90%-100%',
+      '--cpus=50%',
+      '--',
+      'true',
+    ]);
+    expect(parsed.wrapperOptions.memory).toBe('90%-100%');
+    expect(parsed.wrapperOptions.cpus).toBe('50%');
+    for (const value of ['100%-90%', '101%', '0%', '0g', '3watts']) {
+      expect(() =>
+        parseArgs([
+          '--isolated',
+          'docker',
+          '--image',
+          'alpine',
+          '--memory',
+          value,
+          '--',
+          'true',
+        ])
+      ).toThrow();
+    }
+  });
+});
+
+const {
+  resolveResourceOptions,
+} = require('../src/lib/docker-resource-options');
+it('uses daemon capacity, draws each range once and disables extra swap by default', () => {
+  let draws = 0;
+  const calls = [];
+  const result = resolveResourceOptions(
+    { memory: '70%-80%', cpus: '50%' },
+    (_cmd, args) => {
+      calls.push(args);
+      return {
+        success: true,
+        stdout: JSON.stringify({ MemTotal: 1024 ** 3 * 10, NCPU: 8 }),
+      };
+    },
+    () => {
+      draws++;
+      return 0.5;
+    }
+  );
+  expect(result.resourceLimits).toEqual([
+    '--memory=8053063680',
+    '--cpus=4',
+    '--memory-swap=8053063680',
+  ]);
+  expect(result.resolvedLimits.memorySwap).toBe(result.resolvedLimits.memory);
+  expect(calls).toHaveLength(1);
+  expect(draws).toBe(2);
+});
+it('preserves stored resolutions without another random draw or daemon query', () => {
+  const limits = [
+    '--memory=8053063680',
+    '--memory-swap=8053063680',
+    '--cpus=4',
+  ];
+  const result = resolveResourceOptions(
+    { resourceLimits: limits },
+    () => {
+      throw Error('unexpected query');
+    },
+    () => {
+      throw Error('unexpected draw');
+    }
+  );
+  expect(result.resourceLimits).toEqual(limits);
+});
+it('rejects swap below memory and daemon capacity failures', () => {
+  expect(() =>
+    resolveResourceOptions({ memory: '64m', memorySwap: '32m' })
+  ).toThrow();
+  expect(() =>
+    resolveResourceOptions({ memory: '80%' }, () => ({ success: false }))
+  ).toThrow();
+});
+
+it('forwards raw resource and penalty options through every remaining Docker level', () => {
+  const { buildNextLevelCommand } = require('../src/lib/command-builder');
+  const options = parseArgs([
+    '--isolated',
+    'screen docker',
+    '--memory',
+    '70%-80%',
+    '--cpus',
+    '50%',
+    '--cpu-penalty',
+    '--cpu-penalty-trigger-window',
+    '2s',
+    '--',
+    'work',
+  ]).wrapperOptions;
+  const command = buildNextLevelCommand(options, 'work');
+  const words = require('../src/lib/shell-utils').splitShellWords(command);
+  expect(words[words.indexOf('--memory') + 1]).toBe('70%-80%');
+  expect(words[words.indexOf('--cpus') + 1]).toBe('50%');
+  expect(command).toContain('--cpu-penalty-trigger-window 2000ms');
+});

package/test/regression-191.js

@@ -0,0 +1,152 @@
+const { it, expect } = require('bun:test');
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+const {
+  ExecutionStore,
+  ExecutionRecord,
+} = require('../src/lib/execution-store');
+const { resumeExecution } = require('../src/lib/execution-resume');
+const { recoverKilledExecution } = require('../src/lib/execution-recovery');
+const { SessionState } = require('../src/lib/session-probe');
+
+function fixture() {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'memory-191-'));
+  const store = new ExecutionStore({ appFolder: dir, useLinks: false });
+  const record = new ExecutionRecord({
+    command: 'work',
+    status: 'executed',
+    logPath: path.join(dir, 'run.log'),
+    options: {
+      isolated: 'docker',
+      isolationMode: 'detached',
+      sessionName: 'task',
+      image: 'alpine',
+      onKillResume: 3,
+      onKillResumeMemory: '70%-80%',
+    },
+  });
+  store.save(record);
+  const calls = [];
+  const runner = (_, args) => {
+    calls.push(args);
+    if (args[0] === 'info') {
+      return {
+        success: true,
+        stdout: JSON.stringify({ MemTotal: 1073741824, NCPU: 4 }),
+      };
+    }
+    if (args[0] === 'inspect') {
+      return {
+        success: true,
+        stdout: JSON.stringify({
+          Memory: 67108864,
+          MemorySwap: 67108864,
+          NanoCpus: 3000000000,
+        }),
+      };
+    }
+    return { success: true, stdout: 'id' };
+  };
+  return { dir, store, record, runner, calls };
+}
+it('manual in-place memory override updates before start and records the resolution', async () => {
+  const f = fixture();
+  try {
+    const result = await resumeExecution(f.store, f.record.uuid, {
+      resourceOptions: { memory: '128m' },
+      runner: f.runner,
+      probe: () => ({ alive: false, state: SessionState.STOPPED }),
+      startWatcher: () => true,
+    });
+    expect(result.success).toBe(true);
+    const update = f.calls.find((a) => a[0] === 'update');
+    expect(update).toContain('--memory=134217728');
+    expect(update).toContain('--memory-swap=134217728');
+    expect(f.calls.indexOf(update)).toBeLessThan(
+      f.calls.findIndex((a) => a[0] === 'start')
+    );
+    expect(f.store.get(f.record.uuid).options.resolvedLimits.memory).toBe(
+      134217728
+    );
+  } finally {
+    fs.rmSync(f.dir, { recursive: true, force: true });
+  }
+});
+it('manual snapshot override applies limits to the new container at creation', async () => {
+  const f = fixture();
+  try {
+    const result = await resumeExecution(f.store, f.record.uuid, {
+      command: 'new',
+      resourceOptions: { memory: '128m' },
+      runner: f.runner,
+      probe: () => ({ alive: false, state: SessionState.STOPPED }),
+      startWatcher: () => true,
+    });
+    expect(result.success).toBe(true);
+    expect(f.calls.find((a) => a[0] === 'run')).toContain('--memory=134217728');
+  } finally {
+    fs.rmSync(f.dir, { recursive: true, force: true });
+  }
+});
+for (const [exitCode, oomKilled, sample, change] of [
+  [137, true, '67108864 66000000 1 1 100 100', true],
+  [137, true, '67108864 66000000 1 1 0 100', false],
+  [137, false, '67108864 66000000 0 0 0 100', false],
+  [1, true, '67108864 66000000 1 1 100 100', false],
+]) {
+  it(`automatic memory change requires a qualifying OOM (exit=${exitCode}, fresh=${change})`, () => {
+    const f = fixture();
+    try {
+      const result = recoverKilledExecution({
+        store: f.store,
+        executionId: f.record.uuid,
+        exitCode,
+        oomKilled,
+        cgroupMemory: sample,
+        finishedAt: '1970-01-01T00:01:40Z',
+        runner: f.runner,
+        random: () => 0.5,
+        startWatcher: () => true,
+      });
+      const update = f.calls.find((a) => a[0] === 'update');
+      expect(Boolean(update)).toBe(change);
+      if (change) {
+        expect(result.recovered).toBe(true);
+        expect(update).toContain('--memory=805306368');
+        expect(f.calls.indexOf(update)).toBeLessThan(
+          f.calls.findIndex((a) => a[0] === 'start')
+        );
+        expect(f.store.get(f.record.uuid).options.resolvedLimits.memory).toBe(
+          805306368
+        );
+      }
+      if (exitCode === 1) {
+        expect(result.recovered).toBe(false);
+      }
+    } finally {
+      fs.rmSync(f.dir, { recursive: true, force: true });
+    }
+  });
+}
+
+it('a failed memory update prevents starting the old-limit command', async () => {
+  const f = fixture();
+  try {
+    const result = await resumeExecution(f.store, f.record.uuid, {
+      resourceOptions: { memory: '128m' },
+      runner: (cmd, args) =>
+        args[0] === 'update'
+          ? (f.calls.push(args),
+            { success: false, stderr: 'daemon rejected update' })
+          : f.runner(cmd, args),
+      probe: () => ({ alive: false, state: SessionState.STOPPED }),
+      startWatcher: () => true,
+    });
+    expect(result.success).toBe(false);
+    expect(f.calls.some((args) => args[0] === 'start')).toBe(false);
+    expect(f.store.get(f.record.uuid).status).toBe('executed');
+  } finally {
+    fs.rmSync(f.dir, { recursive: true, force: true });
+  }
+});

package/test/regression-192.js

@@ -0,0 +1,66 @@
+const { it, expect } = require('bun:test');
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+const { spawnSync } = require('child_process');
+const {
+  buildCgroupSamplerStartSnippet,
+  buildCgroupMemoryLogSnippet,
+} = require('../src/lib/cgroup-memory');
+it('samples remote/private cgroups through Docker and reports unavailable counters', () => {
+  expect(buildCgroupSamplerStartSnippet('remote-task')).toContain(
+    'docker exec'
+  );
+  expect(buildCgroupMemoryLogSnippet("'/tmp/log'")).toContain('unavailable');
+  expect(buildCgroupSamplerStartSnippet('remote-task')).toContain(
+    'HostConfig.Memory'
+  );
+});
+
+for (const mode of ['remote', 'no-shell', 'outage', 'shared']) {
+  it(`real watcher shell: ${mode}`, () => {
+    if (process.platform === 'win32') {
+      return;
+    }
+    const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'remote-192-'));
+    try {
+      const script = path.join(dir, 'sample.sh');
+      fs.writeFileSync(
+        script,
+        [
+          buildCgroupSamplerStartSnippet('remote-task'),
+          'sleep 1.2',
+          require('../src/lib/cgroup-memory').buildCgroupSamplerStopSnippet(),
+          buildCgroupMemoryLogSnippet('"$TMPDIR/memory.log"'),
+          'cat "$TMPDIR/memory.log"',
+        ].join('; ')
+      );
+      const result = spawnSync(
+        '/bin/sh',
+        [
+          path.resolve(
+            __dirname,
+            '../../experiments/issue-195-memory-sampler.sh'
+          ),
+          mode,
+          script,
+        ],
+        { encoding: 'utf8' }
+      );
+      expect(result.status).toBe(0);
+      if (mode === 'remote' || mode === 'outage') {
+        expect(result.stdout).toContain(
+          'memory.max=67108864 memory.peak=33554432 oom=1 oom_kill=3'
+        );
+      } else {
+        expect(result.stdout).toContain('Memory:     unavailable');
+        expect(result.stdout).toContain('memory.limit=67108864 (HostConfig)');
+        expect(result.stdout).toContain(
+          mode === 'no-shell' ? 'sh: not found' : 'namespace=host'
+        );
+      }
+    } finally {
+      fs.rmSync(dir, { recursive: true, force: true });
+    }
+  }, 10000);
+}

package/test/regression-193.js

@@ -0,0 +1,241 @@
+const { describe, it, expect } = require('bun:test');
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+const { LockManager, ExecutionRecord } = require('../src/lib/execution-store');
+const { resumeExecution } = require('../src/lib/execution-resume');
+const { SessionState } = require('../src/lib/session-probe');
+
+describe('issue #193: durable store locks and resume', () => {
+  for (const content of [
+    '',
+    '{"pid":',
+    JSON.stringify({
+      pid: String(process.pid),
+      timestamp: Date.now(),
+      hostname: os.hostname(),
+    }),
+  ]) {
+    it(`reclaims an old malformed lock (${JSON.stringify(content)})`, () => {
+      const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'lock-193-'));
+      const file = path.join(dir, 'executions.lock');
+      try {
+        fs.writeFileSync(file, content);
+        const old = new Date(Date.now() - 10000);
+        fs.utimesSync(file, old, old);
+        const lock = new LockManager(file);
+        expect(lock.acquire(300)).toBe(true);
+        lock.release();
+        expect(fs.existsSync(file)).toBe(false);
+      } finally {
+        fs.rmSync(dir, { recursive: true, force: true });
+      }
+    });
+  }
+
+  it('does not replace a fresh incomplete lock', () => {
+    const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'lock-193-'));
+    const file = path.join(dir, 'executions.lock');
+    try {
+      fs.writeFileSync(file, '');
+      expect(new LockManager(file).acquire(100)).toBe(false);
+    } finally {
+      fs.rmSync(dir, { recursive: true, force: true });
+    }
+  });
+
+  it('reserves the replacement session before Docker side effects', async () => {
+    const record = new ExecutionRecord({
+      command: 'old',
+      status: 'executed',
+      exitCode: 137,
+      options: {
+        isolated: 'docker',
+        isolationMode: 'detached',
+        sessionName: 'task',
+      },
+    });
+    let saved;
+    const store = {
+      get: () => saved || record,
+      save: (r) => {
+        saved = globalThis.structuredClone(r);
+      },
+    };
+    const result = await resumeExecution(store, record.uuid, {
+      command: 'new',
+      probe: () => ({ alive: false, state: SessionState.STOPPED }),
+      runner: (_cmd, args) => {
+        if (args[0] !== 'inspect') {
+          expect(saved?.options.sessionName).toBe('task-resume-1');
+          expect(saved?.attempt.launchAcceptedAt).toBeNull();
+        }
+        return { success: true, stdout: args[0] === 'inspect' ? '{}' : 'id' };
+      },
+      startWatcher: () => true,
+    });
+    expect(result.success).toBe(true);
+  });
+
+  it('never launches when the durable reservation fails', async () => {
+    const record = new ExecutionRecord({
+      command: 'old',
+      status: 'executed',
+      options: {
+        isolated: 'docker',
+        isolationMode: 'detached',
+        sessionName: 'task',
+      },
+    });
+    const calls = [];
+    const result = await resumeExecution(
+      {
+        get: () => record,
+        save: () => {
+          throw new Error('ENOSPC');
+        },
+      },
+      record.uuid,
+      {
+        probe: () => ({ alive: false, state: SessionState.STOPPED }),
+        runner: (_cmd, args) => {
+          calls.push(args);
+          return { success: true, stdout: 'id' };
+        },
+      }
+    );
+    expect(result.success).toBe(false);
+    expect(calls.some((args) => args[0] === 'start')).toBe(false);
+  });
+});
+
+it('an ENOSPC while writing lock data never publishes an empty shared lock', () => {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'enospc-193-'));
+  const file = path.join(dir, 'executions.lock');
+  try {
+    const failedFs = {
+      ...fs,
+      writeFileSync: (name) => {
+        fs.writeFileSync(name, '');
+        throw Object.assign(new Error('disk full'), { code: 'ENOSPC' });
+      },
+    };
+    expect(() => new LockManager(file, failedFs).acquire(100)).toThrow(
+      'disk full'
+    );
+    expect(fs.readdirSync(dir)).toEqual([]);
+    const healthy = new LockManager(file);
+    expect(healthy.acquire(100)).toBe(true);
+    healthy.release();
+  } finally {
+    fs.rmSync(dir, { recursive: true, force: true });
+  }
+});
+
+for (const canStop of [true, false]) {
+  it(`attaches the watcher and identifies a postlaunch save failure (stop=${canStop})`, async () => {
+    const record = new ExecutionRecord({
+      command: 'work',
+      status: 'executed',
+      options: {
+        isolated: 'docker',
+        isolationMode: 'detached',
+        sessionName: 'task',
+      },
+    });
+    let saves = 0,
+      saved,
+      watched = false;
+    const calls = [];
+    const store = {
+      get: () => saved || record,
+      save: (r) => {
+        if (++saves > 1) {
+          throw Error('ENOSPC');
+        }
+        saved = globalThis.structuredClone(r);
+      },
+    };
+    const result = await resumeExecution(store, record.uuid, {
+      probe: () => ({ alive: false, state: SessionState.STOPPED }),
+      runner: (_, args) => {
+        calls.push(args);
+        return {
+          success: args[0] !== 'stop' || canStop,
+          stdout: args[0] === 'inspect' ? '{}' : 'id',
+        };
+      },
+      startWatcher: () => {
+        watched = true;
+        return true;
+      },
+    });
+    expect(result.success).toBe(false);
+    expect(watched).toBe(true);
+    expect(calls.some((args) => args[0] === 'stop')).toBe(true);
+    const error = JSON.parse(result.error);
+    expect(error.code).toBe('LAUNCH_PERSISTENCE_FAILED');
+    expect(error.containerName).toBe('task');
+    expect(error.running).toBe(!canStop);
+    expect(saved.options.launchPending).toBe(true);
+  });
+}
+
+it('a partial ENOSPC database write preserves the durable reservation', () => {
+  const { ExecutionStore } = require('../src/lib/execution-store');
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'database-193-'));
+  const store = new ExecutionStore({ appFolder: dir, useLinks: false });
+  const record = new ExecutionRecord({
+    command: 'reserved',
+    options: { launchPending: true, sessionName: 'task-resume-1' },
+  });
+  store.save(record);
+  const original = fs.writeFileSync;
+  try {
+    fs.writeFileSync = (name, content, ...args) => {
+      if (typeof name === 'string' && name.startsWith(store.linoDbPath)) {
+        original(name, String(content).slice(0, 8), ...args);
+        throw Object.assign(Error('disk full'), { code: 'ENOSPC' });
+      }
+      return original(name, content, ...args);
+    };
+    expect(() => store.save(new ExecutionRecord({ command: 'new' }))).toThrow(
+      'disk full'
+    );
+    expect(store.get(record.uuid)?.options.sessionName).toBe('task-resume-1');
+    expect(fs.readdirSync(dir).filter((name) => name.includes('.tmp'))).toEqual(
+      []
+    );
+  } finally {
+    fs.writeFileSync = original;
+    fs.rmSync(dir, { recursive: true, force: true });
+  }
+});
+
+it('a dead launcher does not permanently block a later resume', async () => {
+  const { ExecutionStore } = require('../src/lib/execution-store');
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'owner-193-'));
+  try {
+    const store = new ExecutionStore({ appFolder: dir, useLinks: false });
+    const record = new ExecutionRecord({
+      command: 'work',
+      options: {
+        isolated: 'docker',
+        isolationMode: 'detached',
+        sessionName: 'task',
+        launchPending: true,
+        launchOwner: { pid: 2147483647, hostname: os.hostname() },
+      },
+    });
+    store.save(record);
+    const result = await resumeExecution(store, record.uuid, {
+      probe: () => ({ alive: false, state: SessionState.STOPPED }),
+      runner: () => ({ success: true, stdout: '{}' }),
+      startWatcher: () => true,
+    });
+    expect(result.success).toBe(true);
+    expect(store.get(record.uuid).options.launchPending).toBe(false);
+  } finally {
+    fs.rmSync(dir, { recursive: true, force: true });
+  }
+});