start-command 0.35.4 → 0.36.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. package/CHANGELOG.md +6 -0
  2. package/README.md +10 -0
  3. package/package.json +1 -1
  4. package/src/bin/cli.js +43 -7
  5. package/src/lib/args-parser.js +12 -0
  6. package/src/lib/atomic-write.js +34 -0
  7. package/src/lib/attached-diagnostics.js +90 -0
  8. package/src/lib/cgroup-memory.js +43 -11
  9. package/src/lib/command-builder.js +40 -0
  10. package/src/lib/cpu-penalty-monitor.js +226 -0
  11. package/src/lib/cpu-penalty.js +175 -0
  12. package/src/lib/detached-finalize.js +16 -0
  13. package/src/lib/docker-cleanup.js +23 -2
  14. package/src/lib/docker-resource-limits.js +28 -0
  15. package/src/lib/docker-resource-options.js +180 -0
  16. package/src/lib/docker-runtime-args.js +15 -0
  17. package/src/lib/execution-recovery.js +145 -29
  18. package/src/lib/execution-resume-all.js +11 -0
  19. package/src/lib/execution-resume.js +142 -29
  20. package/src/lib/execution-store.js +25 -121
  21. package/src/lib/exit-evidence.js +123 -0
  22. package/src/lib/exit-reason.js +24 -6
  23. package/src/lib/isolation.js +28 -3
  24. package/src/lib/launch-owner.js +25 -0
  25. package/src/lib/launch-persistence.js +54 -0
  26. package/src/lib/query-commands.js +14 -3
  27. package/src/lib/resume-resources.js +88 -0
  28. package/src/lib/status-formatter.js +28 -0
  29. package/src/lib/store-lock.js +133 -0
  30. package/src/lib/usage.js +11 -0
  31. package/test/docker-runtime-options.js +6 -0
  32. package/test/execution-resume.js +9 -3
  33. package/test/exit-reason.js +6 -1
  34. package/test/isolation.js +5 -4
  35. package/test/regression-174.js +3 -1
  36. package/test/regression-180.js +17 -10
  37. package/test/regression-182.js +7 -3
  38. package/test/regression-185.js +4 -9
  39. package/test/regression-187.js +2 -1
  40. package/test/regression-189.js +224 -0
  41. package/test/regression-190.js +111 -0
  42. package/test/regression-191.js +152 -0
  43. package/test/regression-192.js +66 -0
  44. package/test/regression-193.js +241 -0
  45. package/test/regression-194.js +182 -0
package/CHANGELOG.md CHANGED
@@ -1,5 +1,11 @@
1
1
  # start-command
2
2
 
3
+ ## 0.36.0
4
+
5
+ ### Minor Changes
6
+
7
+ - dbf7ba6: Add Docker launch/recovery resource limits with daemon percentages and random ranges, opt-in delayed CPU penalties, and remote/DinD memory diagnostics. Reserve launches before side effects, atomically publish store locks and database writes, and require exit-time evidence for OOM verdicts while recognizing attributed Docker daemon restarts.
8
+
3
9
  ## 0.35.4
4
10
 
5
11
  ### Patch Changes
package/README.md CHANGED
@@ -56,3 +56,13 @@ GitHub Releases. The release title carries the `[JavaScript]` prefix, e.g.
56
56
  ## License
57
57
 
58
58
  Released into the public domain under the [Unlicense](../LICENSE).
59
+
60
+ ### Docker resource and recovery controls
61
+
62
+ Launch with `--memory 64m`, `--memory-swap 64m` and `--cpus 1`, or use daemon
63
+ percentages and uniform ranges such as `--memory '70%-80%'`. Memory-swap defaults
64
+ to memory. `--cpu-penalty` enables delayed caps and reports state through status.
65
+ Manual `--resume <id> --memory 128m` updates before starting;
66
+ `--on-kill-resume-memory` changes memory only after a fresh qualifying OOM.
67
+ See the [full resource and recovery guide](../README.md#docker-launch-limits-recovery-limits-and-cpu-penalty)
68
+ and [incident research](../docs/case-studies/issue-195/README.md).
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "start-command",
3
- "version": "0.35.4",
3
+ "version": "0.36.0",
4
4
  "description": "Gamification of coding, execute any command with ability to auto-report issues on GitHub",
5
5
  "main": "src/bin/cli.js",
6
6
  "exports": {
package/src/bin/cli.js CHANGED
@@ -241,6 +241,18 @@ const sessionId = wrapperOptions.sessionId || generateUUID();
241
241
  process.exit(queryExitCode === null ? 0 : queryExitCode);
242
242
  }
243
243
 
244
+ if (wrapperOptions.isolated === 'docker') {
245
+ const {
246
+ resolveResourceOptions,
247
+ } = require('../lib/docker-resource-options');
248
+ Object.assign(
249
+ wrapperOptions,
250
+ resolveResourceOptions(
251
+ wrapperOptions,
252
+ require('../lib/execution-control').runCommand
253
+ )
254
+ );
255
+ }
244
256
  // Check if running in isolation mode or with user isolation
245
257
  if (hasIsolation(wrapperOptions) || wrapperOptions.user) {
246
258
  await runWithIsolation(
@@ -419,14 +431,21 @@ async function runWithIsolation(
419
431
  // Save initial execution record and set global reference for signal cleanup
420
432
  if (executionRecord && store) {
421
433
  currentExecutionRecord = executionRecord;
434
+ require('../lib/launch-owner').markLaunch(executionRecord);
422
435
  try {
423
436
  store.save(executionRecord);
424
437
  } catch (err) {
425
- if (config.verbose) {
426
- console.error(
427
- `[Tracking] Warning: Could not save execution record: ${err.message}`
428
- );
429
- }
438
+ console.error(
439
+ JSON.stringify({
440
+ code: 'LAUNCH_RESERVATION_FAILED',
441
+ uuid: executionRecord.uuid,
442
+ containerName: sessionName,
443
+ error: err.message,
444
+ })
445
+ );
446
+ currentExecutionRecord = null;
447
+ process.exitCode = 1;
448
+ return 1;
430
449
  }
431
450
  }
432
451
 
@@ -458,6 +477,7 @@ async function runWithIsolation(
458
477
  // Note: Isolation environments currently use native spawn/execSync
459
478
  // Future: Add command-stream support with raw() function for multiplexers
460
479
  result = await runIsolated(environment, cmd, {
480
+ ...options,
461
481
  session: sessionName,
462
482
  image: effectiveImage,
463
483
  endpoint: options.endpoint,
@@ -502,6 +522,11 @@ async function runWithIsolation(
502
522
 
503
523
  // Update execution record: detached keeps "executing" (resolved at query time)
504
524
  if (executionRecord && store) {
525
+ executionRecord.options.launchPending = false;
526
+ if (result.diagnostics) {
527
+ executionRecord.cgroupMemory = result.diagnostics.cgroupMemory;
528
+ executionRecord.options.exitEvidence = result.diagnostics.exitEvidence;
529
+ }
505
530
  if (result.containerId) {
506
531
  executionRecord.options.containerId = result.containerId;
507
532
  }
@@ -515,11 +540,22 @@ async function runWithIsolation(
515
540
  : executionRecord
516
541
  );
517
542
  } catch (err) {
518
- if (config.verbose) {
543
+ if (environment === 'docker' && mode === 'detached' && result.success) {
519
544
  console.error(
520
- `[Tracking] Warning: Could not update execution record: ${err.message}`
545
+ require('../lib/launch-persistence').failLaunchedPersistence(
546
+ store,
547
+ executionRecord,
548
+ require('../lib/execution-control').runCommand,
549
+ err
550
+ )
521
551
  );
552
+ currentExecutionRecord = null;
553
+ process.exitCode = 1;
554
+ return 1;
522
555
  }
556
+ console.error(
557
+ `[Tracking] Could not update execution record: ${err.message}`
558
+ );
523
559
  }
524
560
  currentExecutionRecord = null;
525
561
  }
@@ -44,6 +44,8 @@
44
44
  const { getDefaultDockerImage } = require('./docker-utils');
45
45
  const dockerNetworkOptions = require('./docker-network-options');
46
46
  const dockerRecoveryOptions = require('./docker-recovery-options');
47
+ const cpuPenalty = require('./cpu-penalty');
48
+ const dockerResourceOptions = require('./docker-resource-options');
47
49
  const { parseSequence, isSequence } = require('./sequence-parser');
48
50
  const { buildCommandString } = require('./shell-utils');
49
51
  const {
@@ -414,6 +416,14 @@ function parseOption(args, index, options) {
414
416
  return networkOption;
415
417
  }
416
418
 
419
+ const cpuConsumed = cpuPenalty.parse(args, index, options);
420
+ if (cpuConsumed) {
421
+ return cpuConsumed;
422
+ }
423
+ const resourceOption = dockerResourceOptions.parse(args, index, options);
424
+ if (resourceOption) {
425
+ return resourceOption;
426
+ }
417
427
  const recoveryOption = dockerRecoveryOptions.parse(args, index, options);
418
428
  if (recoveryOption) {
419
429
  return recoveryOption;
@@ -756,6 +766,8 @@ function validateOptions(options) {
756
766
  }
757
767
 
758
768
  dockerRecoveryOptions.validate(options);
769
+ dockerResourceOptions.validate(options);
770
+ cpuPenalty.validate(options);
759
771
 
760
772
  // Session name is only valid with isolation
761
773
  if (options.session && !options.isolated) {
@@ -0,0 +1,34 @@
1
+ /** Keep the previous database intact when a write fails, including ENOSPC. */
2
+ const fs = require('fs');
3
+ const path = require('path');
4
+ const crypto = require('crypto');
5
+
6
+ function atomicWrite(file, content) {
7
+ const temporary = `${file}.${process.pid}.${crypto.randomUUID()}.tmp`;
8
+ try {
9
+ fs.writeFileSync(temporary, content, { flag: 'wx' });
10
+ const handle = fs.openSync(temporary, 'r+');
11
+ try {
12
+ fs.fsyncSync(handle);
13
+ } finally {
14
+ fs.closeSync(handle);
15
+ }
16
+ fs.renameSync(temporary, file);
17
+ if (process.platform !== 'win32') {
18
+ const directory = fs.openSync(path.dirname(file), 'r');
19
+ try {
20
+ fs.fsyncSync(directory);
21
+ } finally {
22
+ fs.closeSync(directory);
23
+ }
24
+ }
25
+ } finally {
26
+ try {
27
+ fs.unlinkSync(temporary);
28
+ } catch {
29
+ /* Renamed or never created. */
30
+ }
31
+ }
32
+ }
33
+
34
+ module.exports = { atomicWrite };
@@ -0,0 +1,90 @@
1
+ /** Keep the same memory and CPU monitors alive during an attached Docker run. */
2
+ const { spawn, spawnSync } = require('child_process');
3
+ const fs = require('fs');
4
+ const os = require('os');
5
+ const path = require('path');
6
+ const crypto = require('crypto');
7
+ const cgroup = require('./cgroup-memory');
8
+ const cpu = require('./cpu-penalty-monitor');
9
+ const evidence = require('./exit-evidence');
10
+ const { shellQuote } = require('./isolation-log-utils');
11
+
12
+ function startAttachedDiagnostics(name, options) {
13
+ const file = path.join(os.tmpdir(), `start-attached-${crypto.randomUUID()}`);
14
+ let worker;
15
+ {
16
+ const script = [
17
+ cgroup.buildCgroupSamplerStartSnippet(name),
18
+ cpu.startSnippet(options.executionId, name, 1, options),
19
+ 'read __start_stop',
20
+ cgroup.buildCgroupSamplerStopSnippet(),
21
+ cpu.stopSnippet(),
22
+ `printf '%s' "$__start_command_cgroup" > ${shellQuote(file)}`,
23
+ `printf '%s' "$__start_command_memory_unavailable" > ${shellQuote(`${file}.reason`)}`,
24
+ ]
25
+ .filter(Boolean)
26
+ .join('; ');
27
+ worker = spawn('sh', ['-c', script], {
28
+ stdio: ['pipe', 'ignore', 'ignore'],
29
+ });
30
+ worker.on('error', () => {});
31
+ // Subscribe before a fast command finishes, including shell startup failure.
32
+ worker.done = new Promise((resolve) => worker.once('close', resolve));
33
+ worker.stdin.on('error', () => {});
34
+ }
35
+ return {
36
+ async finish(state) {
37
+ if (worker) {
38
+ worker.stdin.end();
39
+ await worker.done;
40
+ }
41
+ let sample = '';
42
+ let unavailable = 'sampler could not start (sh unavailable)';
43
+ try {
44
+ sample = fs.readFileSync(file, 'utf8');
45
+ unavailable =
46
+ fs.readFileSync(`${file}.reason`, 'utf8') ||
47
+ 'sampler stopped before a reading';
48
+ } catch {
49
+ /* Optional diagnostics are best effort. */
50
+ }
51
+ try {
52
+ fs.unlinkSync(file);
53
+ fs.unlinkSync(`${file}.reason`);
54
+ } catch {
55
+ /* Optional diagnostics are best effort. */
56
+ }
57
+ const docker = require('./docker-cleanup').getDockerCommand();
58
+ const limit = spawnSync(
59
+ docker,
60
+ ['inspect', '-f', '{{.HostConfig.Memory}}', name],
61
+ { encoding: 'utf8' }
62
+ );
63
+ const line = evidence.collectExitEvidence({
64
+ containerName: name,
65
+ sample,
66
+ finishedAt: state?.finishedAt,
67
+ exitCode: state?.exitCode,
68
+ oomKilled: state?.oomKilled,
69
+ });
70
+ if (options.logPath) {
71
+ try {
72
+ fs.appendFileSync(
73
+ options.logPath,
74
+ `${cgroup.formatCgroupMemoryLogLine(sample, limit.status === 0 ? limit.stdout.trim() : 'unknown', unavailable)}\n${line}\n`
75
+ );
76
+ } catch (error) {
77
+ if (process.env.START_DEBUG === '1') {
78
+ console.error(`[DEBUG] Attached diagnostics: ${error.message}`);
79
+ }
80
+ }
81
+ }
82
+ return {
83
+ cgroupMemory: cgroup.parseCgroupMemorySample(sample),
84
+ exitEvidence: evidence.fromLog(line),
85
+ };
86
+ },
87
+ };
88
+ }
89
+
90
+ module.exports = { startAttachedDiagnostics };
@@ -67,6 +67,7 @@ function buildCgroupFunctionsSnippet() {
67
67
  '__start_command_cgroup_read() { __scr_events=$(cat "$1/memory.events" 2>/dev/null) || return 1; ' +
68
68
  "__scr_oom=$(printf '%s\\n' \"$__scr_events\" | sed -n 's/^oom //p'); " +
69
69
  "__scr_kill=$(printf '%s\\n' \"$__scr_events\" | sed -n 's/^oom_kill //p'); " +
70
+ 'case "$__scr_oom:$__scr_kill" in *[!0-9:]*|:*|*:) return 1;; esac; ' +
70
71
  '__scr_max=$(cat "$1/memory.max" 2>/dev/null); __scr_peak=$(cat "$1/memory.peak" 2>/dev/null); ' +
71
72
  'printf \'%s %s %s %s %s\\n\' "${__scr_max:--}" "${__scr_peak:--}" ' +
72
73
  '"${__scr_oom:--}" "${__scr_kill:--}" "$1"; }',
@@ -82,13 +83,33 @@ function buildCgroupFunctionsSnippet() {
82
83
  */
83
84
  function buildCgroupSamplerStartSnippet(containerName) {
84
85
  const v = CGROUP_SHELL_VARS;
86
+ const name = shellQuote(containerName);
87
+ const remoteRead = '__start_command_cgroup_read /sys/fs/cgroup';
85
88
  return [
86
89
  buildCgroupFunctionsSnippet(),
87
90
  `${v.file}="\${TMPDIR:-/tmp}/start-command-cgroup.$$"`,
88
- `rm -f "$${v.file}" "$${v.file}.tmp"`,
89
- `( __scs_dir=$(__start_command_cgroup_dir ${shellQuote(containerName)}) || exit 0; ` +
90
- 'while __scs_line=$(__start_command_cgroup_read "$__scs_dir"); do ' +
91
- `printf '%s\\n' "$__scs_line" > "$${v.file}.tmp" && mv -f "$${v.file}.tmp" "$${v.file}"; ` +
91
+ `rm -f "$${v.file}" "$${v.file}.tmp" "$${v.file}.reason"`,
92
+ `__start_command_memory_limit=$(docker inspect -f '{{.HostConfig.Memory}}' ${name} 2>/dev/null)`,
93
+ `__start_command_sample_name=${name}`,
94
+ // The first counter is a baseline, including after watcher restart. Only
95
+ // changes between adjacent successful samples supply exit-time evidence.
96
+ `( export START_COMMAND_CGROUP_SAMPLING=1; __scs_previous=''; __scs_previous_at=0; __scs_changed=0; ` +
97
+ `__scs_dir=$(__start_command_cgroup_dir ${name}); while :; do ` +
98
+ '__scs_line=""; if [ -n "$__scs_dir" ]; then __scs_line=$(__start_command_cgroup_read "$__scs_dir"); fi; ' +
99
+ `if [ -z "$__scs_line" ]; then __scs_ns=$(docker inspect -f '{{.HostConfig.CgroupnsMode}}' ${name} 2>/dev/null); ` +
100
+ `if [ "$__scs_ns" = private ]; then ` +
101
+ `__scs_remote=$(docker exec ${name} sh -c ${shellQuote(`${buildCgroupFunctionsSnippet()}; ${remoteRead}`)} 2>&1); __scs_exit=$?; ` +
102
+ 'if [ "$__scs_exit" = 0 ]; then __scs_line=$__scs_remote; ' +
103
+ 'else __scs_detail=$(printf "%s" "$__scs_remote" | sed -n "1p" | cut -c 1-240); ' +
104
+ `printf '%s\n' "local cgroup unavailable; Docker exec exit=$__scs_exit: \${__scs_detail:-private cgroup v2 counters unreadable}" > "$${v.file}.reason"; fi; ` +
105
+ `else printf '%s\n' "local cgroup unavailable; container cgroup namespace=\${__scs_ns:-unavailable}, private required" > "$${v.file}.reason"; fi; fi; ` +
106
+ '__scs_now=$(date +%s); if [ -n "$__scs_line" ]; then ' +
107
+ '__scs_kill=$(printf "%s" "$__scs_line" | cut -d " " -f 4); ' +
108
+ 'if [ -n "$__scs_previous" ] && [ "$__scs_kill" -gt "$__scs_previous" ] 2>/dev/null && [ "$((__scs_now-__scs_previous_at))" -le 3 ]; then __scs_changed=$__scs_now; fi; ' +
109
+ '__scs_previous=$__scs_kill; __scs_previous_at=$__scs_now; ' +
110
+ '__scs_fields=$(printf "%s" "$__scs_line" | cut -d " " -f 1-4); ' +
111
+ `printf '%s %s %s %s\\n' "$__scs_fields" "$__scs_changed" "$__scs_now" "$__scs_dir" > "$${v.file}.tmp" && mv -f "$${v.file}.tmp" "$${v.file}"; fi; ` +
112
+ `if [ "$(docker inspect -f '{{.State.Running}}' ${name} 2>/dev/null)" = false ]; then break; fi; ` +
92
113
  `sleep ${CGROUP_SAMPLE_INTERVAL_SECONDS}; done ) >/dev/null 2>&1 & ${v.sampler}=$!`,
93
114
  ].join('; ');
94
115
  }
@@ -106,10 +127,16 @@ function buildCgroupSamplerStopSnippet() {
106
127
  `kill "$${v.sampler}" 2>/dev/null`,
107
128
  `wait "$${v.sampler}" 2>/dev/null`,
108
129
  `${v.sample}=$(cat "$${v.file}" 2>/dev/null)`,
109
- `__scs_dir=$(printf '%s' "$${v.sample}" | cut -s -d ' ' -f 5-)`,
110
- `if [ -n "$__scs_dir" ] && __scs_line=$(__start_command_cgroup_read "$__scs_dir"); then ${v.sample}=$__scs_line; fi`,
111
- `rm -f "$${v.file}" "$${v.file}.tmp"`,
112
- `${v.sample}=$(printf '%s' "$${v.sample}" | cut -d ' ' -f 1-4)`,
130
+ `__start_command_memory_unavailable=$(cat "$${v.file}.reason" 2>/dev/null)`,
131
+ `__scs_dir=$(printf '%s' "$${v.sample}" | cut -s -d ' ' -f 7-)`,
132
+ `if [ -n "$__scs_dir" ] && __scs_line=$(__start_command_cgroup_read "$__scs_dir"); then ` +
133
+ `__scs_old=$(printf '%s' "$${v.sample}" | cut -d ' ' -f 4); __scs_at=$(printf '%s' "$${v.sample}" | cut -d ' ' -f 6); ` +
134
+ `__scs_changed=$(printf '%s' "$${v.sample}" | cut -d ' ' -f 5); __scs_now=$(date +%s); ` +
135
+ `__scs_new=$(printf '%s' "$__scs_line" | cut -d ' ' -f 4); ` +
136
+ `if [ "$__scs_new" -gt "$__scs_old" ] 2>/dev/null && [ "$((__scs_now-__scs_at))" -le 3 ]; then __scs_changed=$__scs_now; fi; ` +
137
+ `${v.sample}="$(printf '%s' "$__scs_line" | cut -d ' ' -f 1-4) $__scs_changed $__scs_now"; fi`,
138
+ `rm -f "$${v.file}" "$${v.file}.tmp" "$${v.file}.reason"`,
139
+ `${v.sample}=$(printf '%s' "$${v.sample}" | cut -d ' ' -f 1-6)`,
113
140
  ].join('; ');
114
141
  }
115
142
 
@@ -128,7 +155,8 @@ function buildCgroupMemoryLogSnippet(quotedLogPath) {
128
155
  `__scm_note=' (${OOM_SCOPE_NOTES[OOM_SCOPE.UNKNOWN]})'; fi; ` +
129
156
  `printf 'Memory: memory.max=%s memory.peak=%s oom=%s oom_kill=%s%s\\n' ` +
130
157
  `"$1" "$2" "$3" "$4" "$__scm_note"; }; ` +
131
- `if [ -n "$${v.sample}" ]; then __start_command_cgroup_log $${v.sample} >> ${quotedLogPath}; fi`
158
+ `if [ -n "$${v.sample}" ]; then __start_command_cgroup_log $${v.sample} >> ${quotedLogPath}; ` +
159
+ `else printf 'Memory: unavailable (%s) memory.limit=%s (HostConfig)\\n' "\${__start_command_memory_unavailable:-sampler stopped before a reading; local cgroup unavailable}" "\${__start_command_memory_limit:-unknown}" >> ${quotedLogPath}; fi`
132
160
  );
133
161
  }
134
162
 
@@ -138,10 +166,14 @@ function buildCgroupMemoryLogSnippet(quotedLogPath) {
138
166
  * @param {string|null|undefined} text - `<memory.max> <memory.peak> <oom> <oom_kill>`
139
167
  * @returns {?string} Line without a trailing newline, or null without a sample
140
168
  */
141
- function formatCgroupMemoryLogLine(text) {
169
+ function formatCgroupMemoryLogLine(
170
+ text,
171
+ memoryLimit = 'unknown',
172
+ unavailable = 'sampler unavailable; local cgroup hidden or unreadable'
173
+ ) {
142
174
  const counters = parseCgroupMemorySample(text);
143
175
  if (!counters) {
144
- return null;
176
+ return `Memory: unavailable (${unavailable}) memory.limit=${memoryLimit} (HostConfig)`;
145
177
  }
146
178
  const [max, peak, oom, kill] = String(text).trim().split(/\s+/);
147
179
  const scope = describeCgroupOomScope(counters);
@@ -57,6 +57,46 @@ function buildNextLevelCommand(options, command) {
57
57
  // Docker runtime options are flat (not per-level); forward them only when a
58
58
  // remaining level still uses docker so the nested $ invocation can apply them.
59
59
  if (remainingStack.includes('docker')) {
60
+ for (const [key, flag] of [
61
+ ['memory', '--memory'],
62
+ ['memorySwap', '--memory-swap'],
63
+ ['cpus', '--cpus'],
64
+ ['onKillResumeMemory', '--on-kill-resume-memory'],
65
+ ]) {
66
+ if (options[key] !== null && options[key] !== undefined) {
67
+ parts.push(
68
+ `${flag} ${require('./shell-utils').quoteShellArg(String(options[key]))}`
69
+ );
70
+ }
71
+ }
72
+ if (options.cpuPenaltyConfig) {
73
+ parts.push('--cpu-penalty');
74
+ for (const [key, flag] of [
75
+ ['cpus', '--cpu-penalty-cpus'],
76
+ ['trigger', '--cpu-penalty-trigger'],
77
+ ['triggerWindowMs', '--cpu-penalty-trigger-window'],
78
+ ['release', '--cpu-penalty-release'],
79
+ ['releaseWindowMs', '--cpu-penalty-release-window'],
80
+ ]) {
81
+ parts.push(
82
+ `${flag} ${options.cpuPenaltyConfig[key]}${key.endsWith('Ms') ? 'ms' : ''}`
83
+ );
84
+ }
85
+ }
86
+ if (options.onKillResume !== null && options.onKillResume !== undefined) {
87
+ parts.push(`--on-kill-resume ${options.onKillResume}`);
88
+ }
89
+ if (
90
+ options.onKillResumeDelay !== null &&
91
+ options.onKillResumeDelay !== undefined
92
+ ) {
93
+ parts.push(`--on-kill-resume-delay ${options.onKillResumeDelay}`);
94
+ }
95
+ if (options.recoveryCommand) {
96
+ parts.push(
97
+ `--recovery-command ${require('./shell-utils').quoteShellArg(options.recoveryCommand)}`
98
+ );
99
+ }
60
100
  for (const volume of options.volumes || []) {
61
101
  parts.push(`--volume "${volume}"`);
62
102
  }
@@ -0,0 +1,226 @@
1
+ /** A watcher-owned monitor. Sidecars avoid writing the store for every sample. */
2
+ const fs = require('fs');
3
+ const path = require('path');
4
+ const { spawnSync } = require('child_process');
5
+ const { initialState, evaluate } = require('./cpu-penalty');
6
+ const { LockManager } = require('./store-lock');
7
+ const { shellQuote } = require('./isolation-log-utils');
8
+ const { cpuCount, cpuUpdateArgs } = require('./docker-resource-limits');
9
+ function statePath(record) {
10
+ return `${record.logPath}.cpu-penalty.json`;
11
+ }
12
+ function readState(record) {
13
+ if (!record.options?.cpuPenaltyConfig || !record.logPath) {
14
+ return null;
15
+ }
16
+ try {
17
+ const state = JSON.parse(fs.readFileSync(statePath(record), 'utf8'));
18
+ if (
19
+ state.containerName !== record.options.sessionName ||
20
+ state.attemptNumber !== (record.attempt?.number || 1)
21
+ ) {
22
+ return null;
23
+ }
24
+ if (state.phase === 'penalized' && Number.isFinite(state.updatedAt)) {
25
+ const until = record.endTime ? Date.parse(record.endTime) : Date.now();
26
+ state.penalizedMs += Math.max(0, until - state.updatedAt);
27
+ state.updatedAt = Math.max(until, state.updatedAt);
28
+ }
29
+ return state;
30
+ } catch {
31
+ return null;
32
+ }
33
+ }
34
+ function publicState(state) {
35
+ const result = { ...state };
36
+ delete result.samples;
37
+ delete result.lastAt;
38
+ delete result.capacity;
39
+ return result;
40
+ }
41
+ function run(command, args) {
42
+ const result = spawnSync(command, args, { encoding: 'utf8' });
43
+ return { success: result.status === 0, stdout: result.stdout || '' };
44
+ }
45
+ async function monitor(uuid, name, attemptNumber, standalone) {
46
+ const { ExecutionStore } = require('./execution-store');
47
+ const store = uuid
48
+ ? new ExecutionStore(
49
+ process.env.START_APP_FOLDER
50
+ ? { appFolder: path.resolve(process.env.START_APP_FOLDER) }
51
+ : {}
52
+ )
53
+ : null;
54
+ const record = store ? store.get(uuid) : JSON.parse(standalone || 'null');
55
+ if (
56
+ !record?.options.cpuPenaltyConfig ||
57
+ !record.logPath ||
58
+ record.options.sessionName !== name
59
+ ) {
60
+ return;
61
+ }
62
+ const config = record.options.cpuPenaltyConfig;
63
+ const lock = new LockManager(`${statePath(record)}.lock`);
64
+ if (!lock.acquire(100)) {
65
+ return;
66
+ }
67
+ const docker = require('./docker-cleanup').getDockerCommand();
68
+ const baseFlags =
69
+ record.options.baseResourceLimits || record.options.resourceLimits || [];
70
+ const baseCpus = cpuCount(baseFlags);
71
+ let state = initialState(baseCpus, Date.now(), readState(record));
72
+ const interval = Math.min(
73
+ 30000,
74
+ Math.max(50, Math.min(config.triggerWindowMs, config.releaseWindowMs) / 8)
75
+ );
76
+ const publish = () => {
77
+ const file = statePath(record),
78
+ tmp = `${file}.${process.pid}.tmp`;
79
+ try {
80
+ fs.writeFileSync(
81
+ tmp,
82
+ JSON.stringify({
83
+ ...publicState(state),
84
+ containerName: name,
85
+ attemptNumber: Number(attemptNumber),
86
+ updatedAt: Date.now(),
87
+ })
88
+ );
89
+ fs.renameSync(tmp, file);
90
+ } finally {
91
+ try {
92
+ fs.unlinkSync(tmp);
93
+ } catch {
94
+ /* Optional diagnostics are best effort. */
95
+ }
96
+ }
97
+ };
98
+ const log = (text) => {
99
+ try {
100
+ fs.appendFileSync(record.logPath, `${text}\n`);
101
+ } catch {
102
+ /* Optional diagnostics are best effort. */
103
+ }
104
+ };
105
+ const started = Date.now();
106
+ try {
107
+ publish();
108
+ while (true) {
109
+ const current = store ? store.get(uuid) : record;
110
+ if (
111
+ !current ||
112
+ current.options.sessionName !== name ||
113
+ (current.attempt?.number || 1) !== Number(attemptNumber)
114
+ ) {
115
+ break;
116
+ }
117
+ const live = run(docker, ['inspect', '-f', '{{.State.Running}}', name]);
118
+ if (live.success && live.stdout.trim() === 'false') {
119
+ break;
120
+ }
121
+ if (!live.success && Date.now() - started > 30000) {
122
+ break;
123
+ }
124
+ const info = run(docker, ['info', '--format', '{{json .}}']);
125
+ const stats = run(docker, [
126
+ 'stats',
127
+ '--no-stream',
128
+ '--format',
129
+ '{{json .}}',
130
+ ]);
131
+ let daemonCpus = NaN,
132
+ cores = NaN;
133
+ try {
134
+ if (info.success) {
135
+ daemonCpus = JSON.parse(info.stdout).NCPU;
136
+ }
137
+ if (stats.success) {
138
+ const row = stats.stdout
139
+ .trim()
140
+ .split('\n')
141
+ .map((s) => JSON.parse(s))
142
+ .find((s) => s.Name === name);
143
+ if (row && /^\d+(?:\.\d+)?%$/.test(row.CPUPerc)) {
144
+ cores = Number(row.CPUPerc.slice(0, -1)) / 100;
145
+ }
146
+ }
147
+ } catch {
148
+ /* Optional diagnostics are best effort. */
149
+ }
150
+ const decision = evaluate(
151
+ state,
152
+ {
153
+ now: Date.now(),
154
+ cores,
155
+ daemonCpus,
156
+ maxGapMs: Math.max(5000, interval * 3),
157
+ },
158
+ config
159
+ );
160
+ if (decision.action) {
161
+ const action = decision.action;
162
+ const update = run(docker, [
163
+ 'update',
164
+ ...cpuUpdateArgs(baseFlags, action.cpus),
165
+ name,
166
+ ]);
167
+ if (update.success) {
168
+ state = decision.state;
169
+ log(
170
+ `[start] CPU penalty ${action.kind === 'apply' ? 'applied' : 'lifted'}: ${action.cpus} CPUs (${action.average === null || action.average === undefined ? 'capacity changed' : `${action.average.toFixed(2)} of ${action.capacity} CPUs`})`
171
+ );
172
+ } else {
173
+ state = {
174
+ ...decision.state,
175
+ phase: state.phase,
176
+ since: state.since,
177
+ limitCpus: state.limitCpus,
178
+ penaltyCount: state.penaltyCount,
179
+ samples: [],
180
+ };
181
+ }
182
+ } else {
183
+ state = decision.state;
184
+ }
185
+ publish();
186
+ await new Promise((resolve) => globalThis.setTimeout(resolve, interval));
187
+ }
188
+ } finally {
189
+ lock.release();
190
+ }
191
+ }
192
+ function startSnippet(uuid, name, attempt = 1, options = null) {
193
+ const standalone =
194
+ !uuid && options?.cpuPenaltyConfig && options.logPath
195
+ ? JSON.stringify({
196
+ logPath: options.logPath,
197
+ options: {
198
+ sessionName: name,
199
+ cpuPenaltyConfig: options.cpuPenaltyConfig,
200
+ baseResourceLimits: options.resourceLimits,
201
+ },
202
+ })
203
+ : null;
204
+ if (!uuid && !standalone) {
205
+ return '';
206
+ }
207
+ return `${shellQuote(process.execPath)} ${shellQuote(__filename)} ${shellQuote(uuid || '')} ${shellQuote(name)} ${Number(attempt)} ${shellQuote(standalone || '')} >/dev/null 2>&1 & __start_cpu_monitor=$!`;
208
+ }
209
+ function stopSnippet() {
210
+ return 'if [ -n "$__start_cpu_monitor" ]; then kill "$__start_cpu_monitor" 2>/dev/null; wait "$__start_cpu_monitor" 2>/dev/null; fi';
211
+ }
212
+ module.exports = {
213
+ statePath,
214
+ readState,
215
+ publicState,
216
+ monitor,
217
+ startSnippet,
218
+ stopSnippet,
219
+ };
220
+ if (require.main === module) {
221
+ monitor(...process.argv.slice(2)).catch((error) => {
222
+ if (process.env.START_DEBUG === '1') {
223
+ console.error(`[DEBUG] CPU monitor: ${error.stack}`);
224
+ }
225
+ });
226
+ }