start-command 0.39.0 → 0.39.1

package/CHANGELOG.md

@@ -1,5 +1,13 @@
 # start-command
 
+## 0.39.1
+
+### Patch Changes
+
+- 55c57b0: Stop the Docker cgroup sampler when the counters can never be read (#247). On hosts whose containers have no `memory.events` (the `memory` controller is not delegated, e.g. a root `cgroup.subtree_control` of `cpuset cpu pids`) and whose local cgroup is hidden, the sampler started one `docker exec` and one `docker inspect -f {{.State.Running}}` every interval for the whole life of the task without ever recording a sample. Exit 3 (unreadable counters) from the long-lived stream or from the per-round read now records the reason and stops sampling, as a missing `sh` (exit 126/127) already did; the post-mortem reports `Memory:     unavailable (local cgroup unavailable; Docker exec exit=3: private cgroup v2 counters unreadable)` instead of the last round's error. Other failed rounds are still retried, but each one doubles the wait, up to 60 s (or the interval, if longer); a sample restores the interval. The completion watcher (`docker wait`) still finishes the run.
+
+  Attached `--isolated screen` runs no longer report a failed command as exit 0 when `screen -dmS` returns before `screen -ls` lists the new session. A session missing from `screen -ls` now counts as finished only once its exit-code file is written, or after 20 polls (about 2 s) without the session.
+
 ## 0.39.0
 
 ### Minor Changes

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "start-command",
-  "version": "0.39.0",
+  "version": "0.39.1",
   "description": "Gamification of coding, execute any command with ability to auto-report issues on GitHub",
   "main": "src/bin/cli.js",
   "exports": {

package/src/lib/cgroup-memory.js

@@ -30,6 +30,15 @@ const { shellQuote } = require('./isolation-log-utils');
 /** Default seconds between samples; the watcher environment may override it. */
 const CGROUP_SAMPLE_INTERVAL_SECONDS = 5;
 
+/**
+ * Longest wait between rounds that keep failing (issue #247), unless the
+ * interval itself is longer. Each round without a sample doubles the wait.
+ */
+const CGROUP_SAMPLE_BACKOFF_MAX_SECONDS = 60;
+
+/** Exit status of both in-container reads for unreadable counters. */
+const CGROUP_UNREADABLE_EXIT = 3;
+
 /** Shell variables used by the sampler snippets. */
 const CGROUP_SHELL_VARS = {
   /** Last sample: `<memory.max> <memory.peak> <oom> <oom_kill>`. */
@@ -80,17 +89,18 @@ function buildCgroupFunctionsSnippet() {
  * (`docker exec -i`), so it runs only shell builtins: the task's own CPU
  * limit pays for no `cat`, `sed` or `sleep` processes. Stdin reaches EOF when
  * the exec's client is gone, so the loop never outlives the sampler; dockerd
- * would accept its output even then. Exit 3: unreadable counters.
+ * would accept its output even then. Exit 3: unreadable counters (no
+ * delegated `memory` controller), which no later round can change.
  * @returns {string} Shell script for `sh -c`
  */
 function buildCgroupStreamScript() {
   return [
     'while read -r __scx_tick; do __scx_oom=; __scx_kill=; __scx_max=-; __scx_peak=-',
-    '[ -r "$1/memory.events" ] || exit 3',
+    `[ -r "$1/memory.events" ] || exit ${CGROUP_UNREADABLE_EXIT}`,
     'while read -r __scx_key __scx_value; do case "$__scx_key" in ' +
       'oom) __scx_oom=$__scx_value;; oom_kill) __scx_kill=$__scx_value;; esac; ' +
       'done < "$1/memory.events"',
-    'case "$__scx_oom:$__scx_kill" in *[!0-9:]*|:*|*:) exit 3;; esac',
+    `case "$__scx_oom:$__scx_kill" in *[!0-9:]*|:*|*:) exit ${CGROUP_UNREADABLE_EXIT};; esac`,
     '[ ! -r "$1/memory.max" ] || read -r __scx_max < "$1/memory.max"',
     '[ ! -r "$1/memory.peak" ] || read -r __scx_peak < "$1/memory.peak"',
     'printf \'%s %s %s %s %s\\n\' "${__scx_max:--}" "${__scx_peak:--}" ' +
@@ -116,15 +126,21 @@ const CGROUP_SAMPLING_OFF_REASON =
  * successful read proves the container runs, so Docker is asked for liveness
  * only after a failed round or when the stream ends. Without a usable stream it falls back
  * to one `docker exec` per round; without `sh` in the image (exit 126/127) or
- * without any readable cgroup it records the reason and stops.
- * `START_COMMAND_CGROUP_SAMPLE_INTERVAL_SECONDS=off` disables sampling.
+ * without any readable cgroup (exit 3 from either exec, issue #247) it
+ * records the reason and stops. Other failed rounds are retried, each one
+ * doubling the wait up to CGROUP_SAMPLE_BACKOFF_MAX_SECONDS; a sample restores
+ * the interval. `START_COMMAND_CGROUP_SAMPLE_INTERVAL_SECONDS=off` disables
+ * sampling.
  * @param {string} containerName - Container to sample
  * @returns {string} Shell command
  */
 function buildCgroupSamplerStartSnippet(containerName) {
   const v = CGROUP_SHELL_VARS;
   const name = shellQuote(containerName);
-  const remoteRead = '__start_command_cgroup_read /sys/fs/cgroup';
+  const remoteRead = `__start_command_cgroup_read /sys/fs/cgroup || exit ${CGROUP_UNREADABLE_EXIT}`;
+  // Exit statuses no later round can change: no `sh`, or no counters.
+  const permanent = `${CGROUP_UNREADABLE_EXIT}|126|127`;
+  const backoff = CGROUP_SAMPLE_BACKOFF_MAX_SECONDS;
   const unavailable = (detail) =>
     `printf '%s\\n' "local cgroup unavailable; ${detail}" > "$${v.file}.reason"`;
   const execReason = unavailable(
@@ -155,7 +171,8 @@ function buildCgroupSamplerStartSnippet(containerName) {
       `__scs_dir=$(__start_command_cgroup_dir ${name}); ` +
       // The namespace is fixed at container creation, including when reads fail.
       `__scs_ns=$(docker inspect -f '{{.HostConfig.CgroupnsMode}}' ${name} 2>/dev/null); __scs_stream=1; __scs_reader=''; ` +
-      `while :; do __scs_ok=''; if [ -n "$__scs_dir" ]; then ` +
+      `__scs_delay=$__scs_interval; ` +
+      `while :; do __scs_ok=''; __scs_sampled=''; if [ -n "$__scs_dir" ]; then ` +
       '__scs_line=$(__start_command_cgroup_read "$__scs_dir") && __scs_record "$__scs_line" && __scs_ok=1; ' +
       `elif [ "$__scs_ns" != private ]; then ${unavailable('container cgroup namespace=${__scs_ns:-unavailable}, private required')}; break; ` +
       `elif [ "$__scs_stream" = 1 ]; then ` +
@@ -171,15 +188,20 @@ function buildCgroupSamplerStartSnippet(containerName) {
       `[ -z "$__scs_got" ] || exit 0; ${execReason}; ` +
       `case "$__scs_exit" in ''|0|*[!0-9]*) exit 1;; esac; exit "$__scs_exit"; } & ` +
       `__scs_reader=$!; wait "$__scs_reader"; __scs_exit=$?; trap - TERM; __scs_reader=''; ` +
-      // No shell in the image: a per-round exec cannot succeed either.
-      `case "$__scs_exit" in 0) ;; 126|127) break;; *) __scs_stream=0;; esac; ` +
+      // No shell in the image or no counters: a per-round exec cannot
+      // succeed either.
+      `case "$__scs_exit" in 0) __scs_sampled=1;; ${permanent}) break;; *) __scs_stream=0;; esac; ` +
       `else __scs_remote=$(docker exec ${name} sh -c ${shellQuote(`${buildCgroupFunctionsSnippet()}; ${remoteRead}`)} 2>&1); __scs_exit=$?; ` +
       'if [ "$__scs_exit" = 0 ] && __scs_record "$__scs_remote"; then __scs_ok=1; ' +
-      `else __scs_detail=$(printf "%s" "$__scs_remote" | sed -n "1p" | cut -c 1-240); ${execReason}; fi; fi; ` +
+      `else __scs_detail=$(printf "%s" "$__scs_remote" | sed -n "1p" | cut -c 1-240); ${execReason}; ` +
+      `case "$__scs_exit" in ${permanent}) break;; esac; fi; fi; ` +
       // A successful read needs no Docker liveness probe, and the completion
       // watcher still stops us after docker wait.
       `if [ -z "$__scs_ok" ] && [ "$(docker inspect -f '{{.State.Running}}' ${name} 2>/dev/null)" = false ]; then break; fi; ` +
-      `sleep "$__scs_interval"; done ) >/dev/null 2>&1 & ${v.sampler}=$!`,
+      // Back off while rounds keep failing, never below the interval.
+      `if [ -n "$__scs_ok$__scs_sampled" ]; then __scs_delay=$__scs_interval; else __scs_delay=$((__scs_delay*2)); ` +
+      `[ "$__scs_delay" -le ${backoff} ] || __scs_delay=${backoff}; [ "$__scs_delay" -ge "$__scs_interval" ] || __scs_delay=$__scs_interval; fi; ` +
+      `sleep "$__scs_delay"; done ) >/dev/null 2>&1 & ${v.sampler}=$!`,
   ].join('; ');
 }
 
@@ -373,10 +395,12 @@ function formatCgroupMemory(memory) {
 }
 
 module.exports = {
+  CGROUP_SAMPLE_BACKOFF_MAX_SECONDS,
   CGROUP_SAMPLE_INTERVAL_SECONDS,
   CGROUP_SAMPLING_OFF_REASON,
   CGROUP_SHELL_VARS,
   CGROUP_STREAM_NAME,
+  CGROUP_UNREADABLE_EXIT,
   OOM_SCOPE,
   OOM_SCOPE_NOTES,
   buildCgroupFunctionsSnippet,

package/src/lib/screen-isolation.js

@@ -100,6 +100,28 @@ function supportsLogfileOption() {
   return version.patch >= 1;
 }
 
+/**
+ * Polls without the session in `screen -ls` and without an exit code before
+ * the session counts as gone. `screen -dmS` can return before the session is
+ * listed, so a missing session alone is no proof that the command finished;
+ * its exit code is written just before the session ends.
+ */
+const SCREEN_UNLISTED_GRACE_POLLS = 20;
+
+/**
+ * Whether an attached screen run is over.
+ * @param {boolean} listed - The session is in `screen -ls`
+ * @param {boolean} exitCodeReady - The exit-code sidecar has content (or no
+ *   sidecar is expected, as for a bare shell)
+ * @param {number} unlistedPolls - Consecutive polls without the session
+ * @returns {boolean}
+ */
+function isScreenRunFinished(listed, exitCodeReady, unlistedPolls) {
+  return (
+    !listed && (exitCodeReady || unlistedPolls >= SCREEN_UNLISTED_GRACE_POLLS)
+  );
+}
+
 /**
  * Run command in GNU Screen using detached mode with log capture.
  *
@@ -346,58 +368,64 @@ function runScreenWithLogCapture(
       const checkInterval = 100; // ms
       const maxWait = 300000; // 5 minutes max
       let waited = 0;
+      let unlistedPolls = 0;
+
+      const exitCodeReady = () => {
+        if (isBareShell) {
+          return true;
+        }
+        try {
+          return fs.readFileSync(exitCodeFile, 'utf8').trim() !== '';
+        } catch {
+          return false;
+        }
+      };
+
+      const finish = () => {
+        readAndDisplayOutput().then((output) => {
+          const exitCode = readExitCode();
+          cleanupTempFiles();
+          resolve({
+            success: exitCode === 0,
+            sessionName,
+            message: `Screen session "${sessionName}" exited with code ${exitCode}`,
+            exitCode,
+            output,
+          });
+        });
+      };
 
       const checkCompletion = () => {
+        let listed;
         try {
           // Check if session still exists
-          const sessions = execSync('screen -ls', {
+          listed = execSync('screen -ls', {
             encoding: 'utf8',
             stdio: ['pipe', 'pipe', 'pipe'],
-          });
-
-          if (!sessions.includes(sessionName)) {
-            // Session ended, read output and exit code
-            readAndDisplayOutput().then((output) => {
-              const exitCode = readExitCode();
-              cleanupTempFiles();
-              resolve({
-                success: exitCode === 0,
-                sessionName,
-                message: `Screen session "${sessionName}" exited with code ${exitCode}`,
-                exitCode,
-                output,
-              });
-            });
-            return;
-          }
-
-          waited += checkInterval;
-          if (waited >= maxWait) {
-            cleanupTempFiles();
-            resolve({
-              success: false,
-              sessionName,
-              message: `Screen session "${sessionName}" timed out after ${maxWait / 1000} seconds`,
-              exitCode: 1,
-            });
-            return;
-          }
-
-          setTimeout(checkCompletion, checkInterval);
+          }).includes(sessionName);
         } catch {
           // screen -ls failed, session probably ended
-          readAndDisplayOutput().then((output) => {
-            const exitCode = readExitCode();
-            cleanupTempFiles();
-            resolve({
-              success: exitCode === 0,
-              sessionName,
-              message: `Screen session "${sessionName}" exited with code ${exitCode}`,
-              exitCode,
-              output,
-            });
+          listed = false;
+        }
+        unlistedPolls = listed ? 0 : unlistedPolls + 1;
+        if (isScreenRunFinished(listed, exitCodeReady(), unlistedPolls)) {
+          finish();
+          return;
+        }
+
+        waited += checkInterval;
+        if (waited >= maxWait) {
+          cleanupTempFiles();
+          resolve({
+            success: false,
+            sessionName,
+            message: `Screen session "${sessionName}" timed out after ${maxWait / 1000} seconds`,
+            exitCode: 1,
           });
+          return;
         }
+
+        setTimeout(checkCompletion, checkInterval);
       };
 
       // Start checking after a brief delay
@@ -522,7 +550,9 @@ function resetScreenVersionCache() {
 }
 
 module.exports = {
+  SCREEN_UNLISTED_GRACE_POLLS,
   getScreenVersion,
+  isScreenRunFinished,
   supportsLogfileOption,
   runScreenWithLogCapture,
   startDetachedScreenWithLogCapture,

package/test/isolation.js

@@ -812,3 +812,35 @@ describe('isInteractiveShellCommand (issue #84)', () => {
     assert.strictEqual(isInteractiveShellCommand('  '), false);
   });
 });
+
+describe('isScreenRunFinished', () => {
+  // `screen -dmS` can return before `screen -ls` lists the session; a failed
+  // command then read as exit 0 (Rust CI run 39873000384).
+  const {
+    SCREEN_UNLISTED_GRACE_POLLS,
+    isScreenRunFinished,
+  } = require('../src/lib/screen-isolation');
+
+  it('waits while the session is listed', () => {
+    assert.strictEqual(isScreenRunFinished(true, true, 0), false);
+  });
+
+  it('finishes once the session is gone and its exit code is written', () => {
+    assert.strictEqual(isScreenRunFinished(false, true, 1), true);
+  });
+
+  it('keeps polling an unlisted session without an exit code', () => {
+    assert.strictEqual(isScreenRunFinished(false, false, 1), false);
+    assert.strictEqual(
+      isScreenRunFinished(false, false, SCREEN_UNLISTED_GRACE_POLLS - 1),
+      false
+    );
+  });
+
+  it('gives up after the grace polls', () => {
+    assert.strictEqual(
+      isScreenRunFinished(false, false, SCREEN_UNLISTED_GRACE_POLLS),
+      true
+    );
+  });
+});

package/test/regression-247.js

@@ -0,0 +1,140 @@
+/**
+ * Regression tests for issue #247: with cgroup counters unreadable on both
+ * sides (no local cgroup, no `memory.events` in the container because the
+ * `memory` controller is not delegated), the sampler kept starting one
+ * `docker exec` and one `docker inspect -f {{.State.Running}}` every interval
+ * for the whole life of the task, although no round could ever record a
+ * sample.
+ *
+ * Now exit 3 (unreadable counters) from the stream or from the per-round read
+ * records the reason and stops, as exit 126/127 already did, and other failed
+ * rounds back off exponentially. The no-memory fixture modes run the real
+ * in-container scripts against a cgroup directory without `memory.events`.
+ */
+
+const { describe, it, expect } = require('bun:test');
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+const { spawnSync } = require('child_process');
+const {
+  CGROUP_SAMPLE_BACKOFF_MAX_SECONDS,
+  CGROUP_UNREADABLE_EXIT,
+  buildCgroupSamplerStartSnippet,
+  buildCgroupSamplerStopSnippet,
+  buildCgroupMemoryLogSnippet,
+} = require('../src/lib/cgroup-memory');
+
+/** Runs the shared fixture of issue #238 and returns its trace. */
+function sample(mode, interval) {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'sampler-247-'));
+  try {
+    const script = path.join(dir, 'sample.sh');
+    fs.writeFileSync(
+      script,
+      [
+        buildCgroupSamplerStartSnippet("task '238'"),
+        'wait "$__start_command_cgroup_sampler"',
+        buildCgroupSamplerStopSnippet(),
+        'printf "sample=%s\\n" "$__start_command_cgroup"',
+        buildCgroupMemoryLogSnippet('"$TMPDIR/memory.log"'),
+        'cat "$TMPDIR/memory.log" "$TMPDIR/calls"',
+        'sed "s/^/sleep=/" "$TMPDIR/sleeps"',
+        'for file in "$TMPDIR"/start-command-cgroup*; do [ ! -e "$file" ] || exit 1; done',
+      ].join('; ')
+    );
+    const env = { ...process.env };
+    delete env.START_COMMAND_CGROUP_SAMPLE_INTERVAL_SECONDS;
+    if (interval !== undefined) {
+      env.START_COMMAND_CGROUP_SAMPLE_INTERVAL_SECONDS = interval;
+    }
+    const result = spawnSync(
+      '/bin/sh',
+      [
+        path.resolve(
+          __dirname,
+          '../../experiments/issue-238-cgroup-polling.sh'
+        ),
+        mode,
+        script,
+      ],
+      { encoding: 'utf8', env, timeout: 30000 }
+    );
+    expect(result.status).toBe(0);
+    return result.stdout;
+  } finally {
+    fs.rmSync(dir, { recursive: true, force: true });
+  }
+}
+
+const count = (output, text) =>
+  output.split('\n').filter((line) => line.includes(text)).length;
+const sleeps = (output) =>
+  output
+    .split('\n')
+    .filter((line) => line.startsWith('sleep='))
+    .map((line) => Number(line.slice('sleep='.length)));
+const posix = process.platform === 'win32' ? describe.skip : describe;
+const unreadable = `Memory:     unavailable (local cgroup unavailable; Docker exec exit=${CGROUP_UNREADABLE_EXIT}: private cgroup v2 counters unreadable)`;
+
+posix('issue #247: unreadable cgroup counters stop the sampler', () => {
+  it('stops after the stream exits 3, without per-round execs or liveness probes', () => {
+    const output = sample('no-memory');
+    expect(output).toContain('sample=\n');
+    expect(output).toContain(unreadable);
+    expect(count(output, "exec -i task '238' sh -c")).toBe(1);
+    expect(count(output, "task '238' sh -c")).toBe(1);
+    expect(count(output, '{{.State.Running}}')).toBe(0);
+    expect(sleeps(output)).toEqual([]);
+  });
+
+  it('stops after a per-round read exits 3', () => {
+    // The stream fails for a transient reason first, so one per-round exec
+    // runs; its real read finds no memory.events.
+    const output = sample('no-memory-outage');
+    expect(output).toContain(unreadable);
+    expect(count(output, "exec -i task '238' sh -c")).toBe(1);
+    expect(count(output, "exec task '238' sh -c")).toBe(1);
+    expect(count(output, '{{.State.Running}}')).toBe(1);
+    expect(sleeps(output)).toHaveLength(1);
+  });
+
+  it('stops after a per-round exec finds no shell', () => {
+    const output = sample('no-shell-outage');
+    expect(output).toContain('Docker exec exit=127: sh: not found');
+    expect(count(output, "exec task '238' sh -c")).toBe(1);
+    expect(count(output, '{{.State.Running}}')).toBe(1);
+  });
+
+  it('records the per-round reason as exit 3, not as an exec failure', () => {
+    const snippet = buildCgroupSamplerStartSnippet('task');
+    expect(snippet).toContain(
+      `__start_command_cgroup_read /sys/fs/cgroup || exit ${CGROUP_UNREADABLE_EXIT}`
+    );
+    expect(snippet).toContain('3|126|127) break;;');
+  });
+});
+
+posix('issue #247: failed rounds back off exponentially', () => {
+  it('doubles the wait after each round without a sample', () => {
+    // Round 0 streams a sample; the stream, then per-round execs, fail.
+    expect(sleeps(sample('outage'))).toEqual([5, 10, 20]);
+  });
+
+  it(`caps the wait at ${CGROUP_SAMPLE_BACKOFF_MAX_SECONDS} seconds`, () => {
+    expect(sleeps(sample('outage', '40'))).toEqual([
+      40,
+      CGROUP_SAMPLE_BACKOFF_MAX_SECONDS,
+      CGROUP_SAMPLE_BACKOFF_MAX_SECONDS,
+    ]);
+  });
+
+  it('never waits less than a longer interval', () => {
+    expect(sleeps(sample('outage', '100'))).toEqual([100, 100, 100]);
+  });
+
+  it('keeps the interval while samples arrive', () => {
+    expect(sleeps(sample('remote', '2'))).toEqual([2, 2, 2]);
+    expect(sleeps(sample('local'))).toEqual([5, 5, 5]);
+  });
+});