start-command 0.35.4 → 0.36.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/README.md +10 -0
- package/package.json +1 -1
- package/src/bin/cli.js +43 -7
- package/src/lib/args-parser.js +12 -0
- package/src/lib/atomic-write.js +34 -0
- package/src/lib/attached-diagnostics.js +90 -0
- package/src/lib/cgroup-memory.js +43 -11
- package/src/lib/command-builder.js +40 -0
- package/src/lib/cpu-penalty-monitor.js +226 -0
- package/src/lib/cpu-penalty.js +175 -0
- package/src/lib/detached-finalize.js +16 -0
- package/src/lib/docker-cleanup.js +23 -2
- package/src/lib/docker-resource-limits.js +28 -0
- package/src/lib/docker-resource-options.js +180 -0
- package/src/lib/docker-runtime-args.js +15 -0
- package/src/lib/execution-recovery.js +145 -29
- package/src/lib/execution-resume-all.js +11 -0
- package/src/lib/execution-resume.js +142 -29
- package/src/lib/execution-store.js +25 -121
- package/src/lib/exit-evidence.js +123 -0
- package/src/lib/exit-reason.js +24 -6
- package/src/lib/isolation.js +28 -3
- package/src/lib/launch-owner.js +25 -0
- package/src/lib/launch-persistence.js +54 -0
- package/src/lib/query-commands.js +14 -3
- package/src/lib/resume-resources.js +88 -0
- package/src/lib/status-formatter.js +28 -0
- package/src/lib/store-lock.js +133 -0
- package/src/lib/usage.js +11 -0
- package/test/docker-runtime-options.js +6 -0
- package/test/execution-resume.js +9 -3
- package/test/exit-reason.js +6 -1
- package/test/isolation.js +5 -4
- package/test/regression-174.js +3 -1
- package/test/regression-180.js +17 -10
- package/test/regression-182.js +7 -3
- package/test/regression-185.js +4 -9
- package/test/regression-187.js +2 -1
- package/test/regression-189.js +224 -0
- package/test/regression-190.js +111 -0
- package/test/regression-191.js +152 -0
- package/test/regression-192.js +66 -0
- package/test/regression-193.js +241 -0
- package/test/regression-194.js +182 -0
|
@@ -326,6 +326,9 @@ function applyEndTime(enriched, dockerState, footerFinishedAt) {
|
|
|
326
326
|
}
|
|
327
327
|
|
|
328
328
|
function resolveDetachedStatus(record, logTail) {
|
|
329
|
+
if (require('./launch-owner').hasActiveLaunch(record)) {
|
|
330
|
+
return record;
|
|
331
|
+
}
|
|
329
332
|
const footer = parseFooterFromTail(logTail);
|
|
330
333
|
const footerExit = footer.exitCode;
|
|
331
334
|
const dockerState = isDetachedDockerRecord(record)
|
|
@@ -444,6 +447,7 @@ function attachExitReason(record, logTail) {
|
|
|
444
447
|
logTail,
|
|
445
448
|
oomKilled: record.oomKilled,
|
|
446
449
|
cgroupMemory: record.cgroupMemory,
|
|
450
|
+
exitEvidence: record.options?.exitEvidence,
|
|
447
451
|
});
|
|
448
452
|
|
|
449
453
|
if (!exitReason) {
|
|
@@ -490,9 +494,22 @@ function attachMemoryExhaustion(record, logTail) {
|
|
|
490
494
|
logTail,
|
|
491
495
|
oomKilled: record.oomKilled,
|
|
492
496
|
cgroupMemory: record.cgroupMemory,
|
|
497
|
+
exitEvidence: record.options?.exitEvidence,
|
|
493
498
|
});
|
|
494
499
|
|
|
495
500
|
if (!memory) {
|
|
501
|
+
if (
|
|
502
|
+
record.memoryExhaustedReason?.startsWith('Docker reported') ||
|
|
503
|
+
record.memoryExhaustedReason?.startsWith('cgroup memory.events')
|
|
504
|
+
) {
|
|
505
|
+
const cleared = Object.assign(
|
|
506
|
+
Object.create(Object.getPrototypeOf(record)),
|
|
507
|
+
record
|
|
508
|
+
);
|
|
509
|
+
delete cleared.memoryExhausted;
|
|
510
|
+
delete cleared.memoryExhaustedReason;
|
|
511
|
+
return cleared;
|
|
512
|
+
}
|
|
496
513
|
return record;
|
|
497
514
|
}
|
|
498
515
|
|
|
@@ -517,6 +534,14 @@ function enrichDetachedStatus(record) {
|
|
|
517
534
|
if (!record) {
|
|
518
535
|
return record;
|
|
519
536
|
}
|
|
537
|
+
const cpuPenalty = require('./cpu-penalty-monitor').readState(record);
|
|
538
|
+
if (cpuPenalty) {
|
|
539
|
+
record = Object.assign(
|
|
540
|
+
Object.create(Object.getPrototypeOf(record)),
|
|
541
|
+
record
|
|
542
|
+
);
|
|
543
|
+
record.cpuPenalty = cpuPenalty;
|
|
544
|
+
}
|
|
520
545
|
const {
|
|
521
546
|
readAttemptActivity,
|
|
522
547
|
readAttemptLogTail,
|
|
@@ -693,6 +718,9 @@ function formatRecordAsText(record) {
|
|
|
693
718
|
...(formatCgroupMemory(obj.cgroupMemory)
|
|
694
719
|
? [`Cgroup Memory: ${formatCgroupMemory(obj.cgroupMemory)}`]
|
|
695
720
|
: []),
|
|
721
|
+
...(obj.cpuPenalty
|
|
722
|
+
? [`CPU Penalty: ${JSON.stringify(obj.cpuPenalty)}`]
|
|
723
|
+
: []),
|
|
696
724
|
`PID: ${obj.pid !== null ? obj.pid : 'N/A'}`,
|
|
697
725
|
`Working Directory: ${obj.workingDirectory}`,
|
|
698
726
|
`Shell: ${obj.shell}`,
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
/** Populate before publishing: a failed write must never poison the shared lock. */
|
|
2
|
+
const fs = require('fs');
|
|
3
|
+
const os = require('os');
|
|
4
|
+
const crypto = require('crypto');
|
|
5
|
+
const LOCK_TIMEOUT_MS = 30000;
|
|
6
|
+
const LOCK_STALE_MS = 60000;
|
|
7
|
+
const MALFORMED_LOCK_GRACE_MS = 3000;
|
|
8
|
+
|
|
9
|
+
class LockManager {
|
|
10
|
+
constructor(lockFilePath, filesystem = fs) {
|
|
11
|
+
this.lockFilePath = lockFilePath;
|
|
12
|
+
this.fs = filesystem;
|
|
13
|
+
this.lockAcquired = false;
|
|
14
|
+
this.identity = null;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
acquire(timeout = LOCK_TIMEOUT_MS) {
|
|
18
|
+
const started = Date.now();
|
|
19
|
+
const debug = (message) => {
|
|
20
|
+
if (process.env.START_DEBUG === '1') {
|
|
21
|
+
console.error(`[DEBUG] Store lock ${this.lockFilePath}: ${message}`);
|
|
22
|
+
}
|
|
23
|
+
};
|
|
24
|
+
while (Date.now() - started < timeout) {
|
|
25
|
+
const temp = `${this.lockFilePath}.${process.pid}.${crypto.randomUUID()}`;
|
|
26
|
+
try {
|
|
27
|
+
try {
|
|
28
|
+
const stat = this.fs.statSync(this.lockFilePath);
|
|
29
|
+
if (this.isLockStale(this.readLockFile(), stat)) {
|
|
30
|
+
const current = this.fs.statSync(this.lockFilePath);
|
|
31
|
+
if (current.ino === stat.ino && current.mtimeMs === stat.mtimeMs) {
|
|
32
|
+
debug('reclaiming stale or malformed lock');
|
|
33
|
+
this.fs.unlinkSync(this.lockFilePath);
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
} catch (error) {
|
|
37
|
+
if (error.code !== 'ENOENT') {
|
|
38
|
+
throw error;
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
this.fs.writeFileSync(
|
|
42
|
+
temp,
|
|
43
|
+
JSON.stringify({
|
|
44
|
+
pid: process.pid,
|
|
45
|
+
timestamp: Date.now(),
|
|
46
|
+
hostname: os.hostname(),
|
|
47
|
+
}),
|
|
48
|
+
{ flag: 'wx' }
|
|
49
|
+
);
|
|
50
|
+
const handle = this.fs.openSync(temp, 'r+');
|
|
51
|
+
try {
|
|
52
|
+
this.fs.fsyncSync(handle);
|
|
53
|
+
} finally {
|
|
54
|
+
this.fs.closeSync(handle);
|
|
55
|
+
}
|
|
56
|
+
const identity = this.fs.statSync(temp);
|
|
57
|
+
this.fs.linkSync(temp, this.lockFilePath);
|
|
58
|
+
this.identity = identity;
|
|
59
|
+
this.lockAcquired = true;
|
|
60
|
+
debug(`acquired after ${Date.now() - started}ms`);
|
|
61
|
+
return true;
|
|
62
|
+
} catch (error) {
|
|
63
|
+
if (error.code !== 'EEXIST') {
|
|
64
|
+
throw error;
|
|
65
|
+
}
|
|
66
|
+
this.sleep(100);
|
|
67
|
+
} finally {
|
|
68
|
+
try {
|
|
69
|
+
this.fs.unlinkSync(temp);
|
|
70
|
+
} catch {
|
|
71
|
+
/* May not have been created. */
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
debug('acquisition timed out');
|
|
76
|
+
return false;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
release() {
|
|
80
|
+
if (!this.lockAcquired) {
|
|
81
|
+
return;
|
|
82
|
+
}
|
|
83
|
+
try {
|
|
84
|
+
const current = this.fs.statSync(this.lockFilePath);
|
|
85
|
+
if (
|
|
86
|
+
current.dev === this.identity.dev &&
|
|
87
|
+
current.ino === this.identity.ino
|
|
88
|
+
) {
|
|
89
|
+
this.fs.unlinkSync(this.lockFilePath);
|
|
90
|
+
}
|
|
91
|
+
} catch {
|
|
92
|
+
/* The lock may already be gone. */
|
|
93
|
+
}
|
|
94
|
+
this.lockAcquired = false;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
readLockFile() {
|
|
98
|
+
try {
|
|
99
|
+
return JSON.parse(this.fs.readFileSync(this.lockFilePath, 'utf8'));
|
|
100
|
+
} catch {
|
|
101
|
+
return null;
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
isLockStale(data, stat) {
|
|
106
|
+
if (
|
|
107
|
+
!data ||
|
|
108
|
+
!Number.isFinite(data.timestamp) ||
|
|
109
|
+
!Number.isInteger(data.pid) ||
|
|
110
|
+
data.pid <= 0 ||
|
|
111
|
+
data.pid > 2147483647
|
|
112
|
+
) {
|
|
113
|
+
return Boolean(
|
|
114
|
+
stat && Date.now() - stat.mtimeMs >= MALFORMED_LOCK_GRACE_MS
|
|
115
|
+
);
|
|
116
|
+
}
|
|
117
|
+
if (data.hostname === os.hostname()) {
|
|
118
|
+
try {
|
|
119
|
+
process.kill(data.pid, 0);
|
|
120
|
+
return false;
|
|
121
|
+
} catch (error) {
|
|
122
|
+
return error.code === 'ESRCH';
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
return Date.now() - data.timestamp > LOCK_STALE_MS;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
sleep(ms) {
|
|
129
|
+
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
module.exports = { LockManager, LOCK_TIMEOUT_MS, MALFORMED_LOCK_GRACE_MS };
|
package/src/lib/usage.js
CHANGED
|
@@ -19,6 +19,17 @@ Options:
|
|
|
19
19
|
--on-kill-resume <N> Resume a killed (exit 137 / OOM) detached docker session up to N times
|
|
20
20
|
--recovery-command <cmd> Command to run in the same container on such a resume
|
|
21
21
|
--on-kill-resume-delay <min[-max]> Wait a random number of seconds before each such resume (default 0)
|
|
22
|
+
--memory <size|N%|MIN%-MAX%> Docker RAM cap; percentages use daemon capacity
|
|
23
|
+
--memory-swap <spec> Combined memory+swap cap (defaults to memory, no extra swap)
|
|
24
|
+
--cpus <n|N%|MIN%-MAX%> Docker CPU cap; ranges draw once per creation
|
|
25
|
+
--on-kill-resume-memory <spec> New memory cap before a qualifying OOM recovery
|
|
26
|
+
--cpu-penalty Enable delayed Docker CPU penalty (default off)
|
|
27
|
+
--cpu-penalty-cpus <n> Penalty cap (default 2 CPUs)
|
|
28
|
+
--cpu-penalty-trigger <p%> Busy threshold (default 95% of usable CPUs)
|
|
29
|
+
--cpu-penalty-trigger-window <d> Fully covered busy window (default 15m)
|
|
30
|
+
--cpu-penalty-release <p%> Quiet threshold (default 65% of penalty cap)
|
|
31
|
+
--cpu-penalty-release-window <d> Minimum quiet/capped window (default 15m)
|
|
32
|
+
Durations accept ms, s, m, h; --status reports penalty state
|
|
22
33
|
--endpoint <endpoint> SSH endpoint (required for ssh isolation, e.g., user@host)
|
|
23
34
|
--isolated-user, -u [name] Create isolated user with same permissions
|
|
24
35
|
--keep-user Keep isolated user after command completes
|
|
@@ -305,6 +305,9 @@ describe('buildDockerRuntimeMetadata', () => {
|
|
|
305
305
|
networks: null,
|
|
306
306
|
networkAliases: null,
|
|
307
307
|
resourceLimits: null,
|
|
308
|
+
resolvedLimits: null,
|
|
309
|
+
resourceLimitSpecs: null,
|
|
310
|
+
onKillResumeMemory: null,
|
|
308
311
|
onKillResume: null,
|
|
309
312
|
recoveryCommand: null,
|
|
310
313
|
onKillResumeDelay: null,
|
|
@@ -331,6 +334,9 @@ describe('buildDockerRuntimeMetadata', () => {
|
|
|
331
334
|
networks: ['hive-formal-ai', 'public-egress'],
|
|
332
335
|
networkAliases: ['formal-ai', 'checker'],
|
|
333
336
|
resourceLimits: null,
|
|
337
|
+
resolvedLimits: null,
|
|
338
|
+
resourceLimitSpecs: null,
|
|
339
|
+
onKillResumeMemory: null,
|
|
334
340
|
onKillResume: null,
|
|
335
341
|
recoveryCommand: null,
|
|
336
342
|
onKillResumeDelay: null,
|
package/test/execution-resume.js
CHANGED
|
@@ -270,11 +270,14 @@ describe('resumeExecution', () => {
|
|
|
270
270
|
watchers.push([name, policy, logPath]),
|
|
271
271
|
});
|
|
272
272
|
assert.strictEqual(result.success, true, result.error);
|
|
273
|
-
assert.deepStrictEqual(
|
|
273
|
+
assert.deepStrictEqual(
|
|
274
|
+
calls.filter((call) => call[1] !== 'inspect'),
|
|
275
|
+
[['docker', 'start', 'box']]
|
|
276
|
+
);
|
|
274
277
|
assert.strictEqual(watchers.length, 1);
|
|
275
278
|
assert.strictEqual(watchers[0][0], 'box');
|
|
276
279
|
assert.strictEqual(watchers[0][2], '/tmp/session.log');
|
|
277
|
-
assert.strictEqual(store.saved.length,
|
|
280
|
+
assert.strictEqual(store.saved.length, 3);
|
|
278
281
|
assert.strictEqual(store.saved[0].uuid, record.uuid);
|
|
279
282
|
assert.strictEqual(store.saved[0].status, 'executing');
|
|
280
283
|
assert.strictEqual(store.saved[0].options.resumeCount, 1);
|
|
@@ -320,7 +323,10 @@ describe('resumeExecution', () => {
|
|
|
320
323
|
});
|
|
321
324
|
assert.strictEqual(result.success, false);
|
|
322
325
|
assert.match(result.error, /no such container/);
|
|
323
|
-
assert.strictEqual(store.saved.length,
|
|
326
|
+
assert.strictEqual(store.saved.length, 2);
|
|
327
|
+
assert.strictEqual(store.saved.at(-1).status, 'executing');
|
|
328
|
+
assert.strictEqual(store.saved.at(-1).options.sessionName, 'box');
|
|
329
|
+
assert.strictEqual(store.saved.at(-1).options.launchPending, undefined);
|
|
324
330
|
});
|
|
325
331
|
|
|
326
332
|
test('relaunches through the isolation backend when the container is gone', async () => {
|
package/test/exit-reason.js
CHANGED
|
@@ -111,7 +111,11 @@ describe('resolveExitReason', () => {
|
|
|
111
111
|
});
|
|
112
112
|
|
|
113
113
|
it('should report the cgroup OOM observation when there is no marker', () => {
|
|
114
|
-
const reason = resolveExitReason({
|
|
114
|
+
const reason = resolveExitReason({
|
|
115
|
+
exitCode: 137,
|
|
116
|
+
oomKilled: true,
|
|
117
|
+
exitEvidence: { mainOom: true },
|
|
118
|
+
});
|
|
115
119
|
assert.strictEqual(reason, 'memory-exhaustion (cgroup-oom-killer)');
|
|
116
120
|
});
|
|
117
121
|
|
|
@@ -173,6 +177,7 @@ describe('resolveMemoryExhaustion', () => {
|
|
|
173
177
|
resolveMemoryExhaustion({
|
|
174
178
|
exitCode: 137,
|
|
175
179
|
logTail: 'killed\n',
|
|
180
|
+
exitEvidence: { mainOom: true },
|
|
176
181
|
oomKilled: true,
|
|
177
182
|
}),
|
|
178
183
|
{
|
package/test/isolation.js
CHANGED
|
@@ -16,6 +16,7 @@ const {
|
|
|
16
16
|
supportsLogfileOption,
|
|
17
17
|
resetScreenVersionCache,
|
|
18
18
|
getDefaultDockerImage,
|
|
19
|
+
canRunLinuxDockerImages,
|
|
19
20
|
} = require('../src/lib/isolation');
|
|
20
21
|
|
|
21
22
|
describe('Isolation Module', () => {
|
|
@@ -665,8 +666,8 @@ describe('detectShellInEnvironment', () => {
|
|
|
665
666
|
});
|
|
666
667
|
|
|
667
668
|
it('should auto-detect shell in docker if docker is available', () => {
|
|
668
|
-
if (!
|
|
669
|
-
console.log(' Skipping:
|
|
669
|
+
if (!canRunLinuxDockerImages()) {
|
|
670
|
+
console.log(' Skipping: Docker cannot run Linux container images');
|
|
670
671
|
return;
|
|
671
672
|
}
|
|
672
673
|
// Use alpine:latest which is commonly available and has sh
|
|
@@ -703,8 +704,8 @@ describe('Shell option forwarding to isolation runners', () => {
|
|
|
703
704
|
|
|
704
705
|
describe('runInDocker shell option', () => {
|
|
705
706
|
it('should accept shell option in options object', async () => {
|
|
706
|
-
if (!
|
|
707
|
-
console.log(' Skipping:
|
|
707
|
+
if (!canRunLinuxDockerImages()) {
|
|
708
|
+
console.log(' Skipping: Docker cannot run Linux container images');
|
|
708
709
|
return;
|
|
709
710
|
}
|
|
710
711
|
|
package/test/regression-174.js
CHANGED
|
@@ -100,9 +100,11 @@ function runWatcher({ policy, running, state, withLog = true }) {
|
|
|
100
100
|
[
|
|
101
101
|
'#!/bin/sh',
|
|
102
102
|
`dir='${dir}'`,
|
|
103
|
+
'[ "$START_COMMAND_CGROUP_SAMPLING" = 1 ] && exit 1',
|
|
103
104
|
'case "$1" in',
|
|
104
105
|
' inspect)',
|
|
105
106
|
' case "$3" in',
|
|
107
|
+
' *HostConfig*) exit 1 ;;',
|
|
106
108
|
' *State.Running*)',
|
|
107
109
|
' answer=$(head -n 1 "$dir/running")',
|
|
108
110
|
' if [ "$(wc -l < "$dir/running")" -gt 1 ]; then',
|
|
@@ -173,7 +175,7 @@ describe('issue #174: docker logs -f returning is not the container exiting', ()
|
|
|
173
175
|
});
|
|
174
176
|
|
|
175
177
|
expect(run.status).toBe(0);
|
|
176
|
-
expect(run.calls).toContain('
|
|
178
|
+
expect(run.calls).toContain('inspect-running false');
|
|
177
179
|
expect(run.calls.indexOf('wait')).toBeGreaterThan(
|
|
178
180
|
run.calls.indexOf('logs')
|
|
179
181
|
);
|
package/test/regression-180.js
CHANGED
|
@@ -70,20 +70,20 @@ describe('issue #180: the sticky OOMKilled flag is not an exit reason', () => {
|
|
|
70
70
|
{
|
|
71
71
|
exitCode: 137,
|
|
72
72
|
oomKilled: true,
|
|
73
|
-
reason:
|
|
74
|
-
memory:
|
|
73
|
+
reason: 'signal (SIGKILL; cause unknown)',
|
|
74
|
+
memory: false,
|
|
75
75
|
},
|
|
76
76
|
{
|
|
77
77
|
exitCode: 137,
|
|
78
78
|
oomKilled: false,
|
|
79
|
-
reason: 'signal (SIGKILL)',
|
|
79
|
+
reason: 'signal (SIGKILL; cause unknown)',
|
|
80
80
|
memory: false,
|
|
81
81
|
},
|
|
82
82
|
{
|
|
83
83
|
exitCode: -1,
|
|
84
84
|
oomKilled: true,
|
|
85
|
-
reason:
|
|
86
|
-
memory:
|
|
85
|
+
reason: null,
|
|
86
|
+
memory: false,
|
|
87
87
|
},
|
|
88
88
|
{
|
|
89
89
|
exitCode: 139,
|
|
@@ -104,10 +104,17 @@ describe('issue #180: the sticky OOMKilled flag is not an exit reason', () => {
|
|
|
104
104
|
});
|
|
105
105
|
}
|
|
106
106
|
|
|
107
|
-
it('
|
|
108
|
-
expect(isOomKillOfCommand({ exitCode: 137, oomKilled: true })).toBe(
|
|
109
|
-
expect(
|
|
110
|
-
|
|
107
|
+
it('requires fresh attribution in addition to SIGKILL or an unknown exit', () => {
|
|
108
|
+
expect(isOomKillOfCommand({ exitCode: 137, oomKilled: true })).toBe(false);
|
|
109
|
+
expect(
|
|
110
|
+
isOomKillOfCommand({
|
|
111
|
+
exitCode: 137,
|
|
112
|
+
oomKilled: true,
|
|
113
|
+
exitEvidence: { mainOom: true },
|
|
114
|
+
})
|
|
115
|
+
).toBe(true);
|
|
116
|
+
expect(isOomKillOfCommand({ exitCode: null, oomKilled: true })).toBe(false);
|
|
117
|
+
expect(isOomKillOfCommand({ exitCode: -1, oomKilled: true })).toBe(false);
|
|
111
118
|
expect(isOomKillOfCommand({ exitCode: 0, oomKilled: true })).toBe(false);
|
|
112
119
|
expect(isOomKillOfCommand({ exitCode: 1, oomKilled: true })).toBe(false);
|
|
113
120
|
expect(isOomKillOfCommand({ exitCode: 137, oomKilled: false })).toBe(false);
|
|
@@ -191,6 +198,6 @@ describe('issue #180: the sticky OOMKilled flag is not an exit reason', () => {
|
|
|
191
198
|
oomKilled: true,
|
|
192
199
|
logPath: null,
|
|
193
200
|
});
|
|
194
|
-
expect(main).toContain('Docker reports it was OOM-killed.');
|
|
201
|
+
expect(main).toContain('Docker reports a process in it was OOM-killed.');
|
|
195
202
|
});
|
|
196
203
|
});
|
package/test/regression-182.js
CHANGED
|
@@ -285,9 +285,13 @@ describe('issue #182: the watcher samples the cgroup while the container runs',
|
|
|
285
285
|
timeout: 20000,
|
|
286
286
|
});
|
|
287
287
|
expect(result.status).toBe(0);
|
|
288
|
-
expect(
|
|
289
|
-
|
|
290
|
-
|
|
288
|
+
expect(
|
|
289
|
+
fs
|
|
290
|
+
.readFileSync(path.join(host.dir, 'sample'), 'utf8')
|
|
291
|
+
.split(' ')
|
|
292
|
+
.slice(0, 4)
|
|
293
|
+
.join(' ')
|
|
294
|
+
).toBe(OOM_SAMPLE);
|
|
291
295
|
expect(fs.readFileSync(logPath, 'utf8')).toBe(
|
|
292
296
|
`${formatCgroupMemoryLogLine(OOM_SAMPLE)}\n`
|
|
293
297
|
);
|
package/test/regression-185.js
CHANGED
|
@@ -97,15 +97,10 @@ describe('issue #185: counter-only OOM scope is unknown', () => {
|
|
|
97
97
|
}
|
|
98
98
|
for (const exitCode of [137, null, -1]) {
|
|
99
99
|
const input = { exitCode, oomKilled: false, cgroupMemory };
|
|
100
|
-
expect(resolveExitReason(input)).toBe(CGROUP_OOM_EXIT_REASON);
|
|
101
|
-
expect(resolveMemoryExhaustion(input)).
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
: {
|
|
105
|
-
memoryExhausted: true,
|
|
106
|
-
memoryExhaustedReason: 'cgroup memory.events reported oom_kill=3',
|
|
107
|
-
}
|
|
108
|
-
);
|
|
100
|
+
expect(resolveExitReason(input)).not.toBe(CGROUP_OOM_EXIT_REASON);
|
|
101
|
+
expect(resolveMemoryExhaustion(input)).toBeNull();
|
|
102
|
+
const attributed = { ...input, exitEvidence: { mainOom: true } };
|
|
103
|
+
expect(resolveExitReason(attributed)).toBe(CGROUP_OOM_EXIT_REASON);
|
|
109
104
|
}
|
|
110
105
|
});
|
|
111
106
|
});
|
package/test/regression-187.js
CHANGED
|
@@ -368,7 +368,8 @@ esac
|
|
|
368
368
|
probe: () => ({ alive: false, state: SessionState.MISSING }),
|
|
369
369
|
runIsolated: (backend, command, options) => {
|
|
370
370
|
expect(options.deferCompletionWatcher).toBe(true);
|
|
371
|
-
expect(store.get(record.uuid).status).toBe('
|
|
371
|
+
expect(store.get(record.uuid).status).toBe('executing');
|
|
372
|
+
expect(store.get(record.uuid).options.launchPending).toBe(true);
|
|
372
373
|
return { success: true, containerId: 'replacement-container' };
|
|
373
374
|
},
|
|
374
375
|
startWatcher: () => {
|
|
@@ -0,0 +1,224 @@
|
|
|
1
|
+
const { it, expect } = require('bun:test');
|
|
2
|
+
const { parseArgs } = require('../src/lib/args-parser');
|
|
3
|
+
it('CPU penalty is opt-in and accepts durations and percentage settings', () => {
|
|
4
|
+
const options = parseArgs([
|
|
5
|
+
'--isolated',
|
|
6
|
+
'docker',
|
|
7
|
+
'--cpu-penalty',
|
|
8
|
+
'--cpu-penalty-trigger-window',
|
|
9
|
+
'1s',
|
|
10
|
+
'--',
|
|
11
|
+
'true',
|
|
12
|
+
]).wrapperOptions;
|
|
13
|
+
expect(options.cpuPenaltyConfig.triggerWindowMs).toBe(1000);
|
|
14
|
+
expect(options.cpuPenaltyConfig.release).toBe(65);
|
|
15
|
+
});
|
|
16
|
+
const { initialState, evaluate, average } = require('../src/lib/cpu-penalty');
|
|
17
|
+
const config = {
|
|
18
|
+
cpus: 2,
|
|
19
|
+
trigger: 95,
|
|
20
|
+
triggerWindowMs: 100,
|
|
21
|
+
release: 65,
|
|
22
|
+
releaseWindowMs: 100,
|
|
23
|
+
};
|
|
24
|
+
it('requires complete time-weighted windows, then lifts and observes afresh', () => {
|
|
25
|
+
let state = initialState(null, 0);
|
|
26
|
+
const tick = (now, cores) => {
|
|
27
|
+
const result = evaluate(
|
|
28
|
+
state,
|
|
29
|
+
{ now, cores, daemonCpus: 6, maxGapMs: 50 },
|
|
30
|
+
config
|
|
31
|
+
);
|
|
32
|
+
state = result.state;
|
|
33
|
+
return result.action;
|
|
34
|
+
};
|
|
35
|
+
expect(tick(0, 6)).toBeNull();
|
|
36
|
+
expect(tick(40, 6)).toBeNull();
|
|
37
|
+
expect(tick(80, 6)).toBeNull();
|
|
38
|
+
expect(tick(100, 0.4).kind).toBe('apply');
|
|
39
|
+
expect(state.penaltyCount).toBe(1);
|
|
40
|
+
expect(tick(120, 0.4)).toBeNull();
|
|
41
|
+
expect(tick(160, 0.4)).toBeNull();
|
|
42
|
+
expect(tick(200, 0.4)).toBeNull();
|
|
43
|
+
expect(tick(220, 6).kind).toBe('lift');
|
|
44
|
+
expect(state.penalizedMs).toBe(120);
|
|
45
|
+
expect(tick(240, 6)).toBeNull();
|
|
46
|
+
expect(tick(280, 6)).toBeNull();
|
|
47
|
+
expect(tick(320, 6)).toBeNull();
|
|
48
|
+
expect(tick(340, 6).kind).toBe('apply');
|
|
49
|
+
expect(state.penaltyCount).toBe(2);
|
|
50
|
+
});
|
|
51
|
+
it('resets coverage on outage, gaps, resize and restart, and skips ineffective caps', () => {
|
|
52
|
+
let state = initialState(4, 0);
|
|
53
|
+
for (const [now, cores, daemonCpus] of [
|
|
54
|
+
[0, 4, 6],
|
|
55
|
+
[40, 4, 6],
|
|
56
|
+
[80, NaN, 6],
|
|
57
|
+
[120, 4, 6],
|
|
58
|
+
[160, 4, 6],
|
|
59
|
+
[200, 4, 8],
|
|
60
|
+
]) {
|
|
61
|
+
const result = evaluate(
|
|
62
|
+
state,
|
|
63
|
+
{ now, cores, daemonCpus, maxGapMs: 50 },
|
|
64
|
+
config
|
|
65
|
+
);
|
|
66
|
+
state = result.state;
|
|
67
|
+
expect(result.action).toBeNull();
|
|
68
|
+
}
|
|
69
|
+
const restarted = initialState(4, 240, state);
|
|
70
|
+
expect(
|
|
71
|
+
evaluate(
|
|
72
|
+
restarted,
|
|
73
|
+
{ now: 240, cores: 4, daemonCpus: 8, maxGapMs: 50 },
|
|
74
|
+
config
|
|
75
|
+
).action
|
|
76
|
+
).toBeNull();
|
|
77
|
+
expect(
|
|
78
|
+
evaluate(state, { now: 400, cores: 4, daemonCpus: 8, maxGapMs: 50 }, config)
|
|
79
|
+
.action
|
|
80
|
+
).toBeNull();
|
|
81
|
+
state = initialState(1, 0);
|
|
82
|
+
for (const now of [0, 40, 80, 120]) {
|
|
83
|
+
const r = evaluate(
|
|
84
|
+
state,
|
|
85
|
+
{ now, cores: 1, daemonCpus: 6, maxGapMs: 50 },
|
|
86
|
+
config
|
|
87
|
+
);
|
|
88
|
+
state = r.state;
|
|
89
|
+
expect(r.action).toBeNull();
|
|
90
|
+
}
|
|
91
|
+
});
|
|
92
|
+
it('weights timestamps rather than giving short intervals equal weight', () => {
|
|
93
|
+
expect(
|
|
94
|
+
average(
|
|
95
|
+
[
|
|
96
|
+
{ at: 0, cores: 6 },
|
|
97
|
+
{ at: 90, cores: 0 },
|
|
98
|
+
{ at: 100, cores: 0 },
|
|
99
|
+
],
|
|
100
|
+
100,
|
|
101
|
+
100
|
|
102
|
+
)
|
|
103
|
+
).toBe(5.4);
|
|
104
|
+
});
|
|
105
|
+
it('resume restores the original base within resized daemon capacity', () => {
|
|
106
|
+
const { prepareResumeResources } = require('../src/lib/resume-resources');
|
|
107
|
+
const record = {
|
|
108
|
+
options: {
|
|
109
|
+
isolated: 'docker',
|
|
110
|
+
sessionName: 'task',
|
|
111
|
+
cpuPenaltyConfig: config,
|
|
112
|
+
baseResourceLimits: ['--cpus=3'],
|
|
113
|
+
resourceLimits: ['--cpus=0.5'],
|
|
114
|
+
resolvedLimits: { cpus: 3 },
|
|
115
|
+
},
|
|
116
|
+
};
|
|
117
|
+
const result = prepareResumeResources(
|
|
118
|
+
record,
|
|
119
|
+
{ state: 'stopped' },
|
|
120
|
+
{},
|
|
121
|
+
(_cmd, args) => ({
|
|
122
|
+
success: true,
|
|
123
|
+
stdout: JSON.stringify(
|
|
124
|
+
args[0] === 'info'
|
|
125
|
+
? { MemTotal: 1073741824, NCPU: 2 }
|
|
126
|
+
: { NanoCpus: 500000000 }
|
|
127
|
+
),
|
|
128
|
+
})
|
|
129
|
+
);
|
|
130
|
+
expect(result.resourceLimits).toContain('--cpus=2');
|
|
131
|
+
expect(result.resolvedLimits.cpus).toBe(2);
|
|
132
|
+
expect(record.options.baseResourceLimits).toEqual(['--cpus=3']);
|
|
133
|
+
});
|
|
134
|
+
|
|
135
|
+
it('monitor applies, lifts and reapplies with tracking disabled', () => {
|
|
136
|
+
if (process.platform === 'win32') {
|
|
137
|
+
return;
|
|
138
|
+
}
|
|
139
|
+
const fs = require('fs'),
|
|
140
|
+
path = require('path'),
|
|
141
|
+
os = require('os');
|
|
142
|
+
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cpu-monitor-189-'));
|
|
143
|
+
try {
|
|
144
|
+
const logPath = path.join(dir, 'task.log');
|
|
145
|
+
fs.writeFileSync(logPath, '');
|
|
146
|
+
const record = {
|
|
147
|
+
logPath,
|
|
148
|
+
options: {
|
|
149
|
+
sessionName: 'cpu-task',
|
|
150
|
+
cpuPenaltyConfig: {
|
|
151
|
+
...config,
|
|
152
|
+
triggerWindowMs: 200,
|
|
153
|
+
releaseWindowMs: 200,
|
|
154
|
+
},
|
|
155
|
+
baseResourceLimits: [],
|
|
156
|
+
},
|
|
157
|
+
};
|
|
158
|
+
const result = require('child_process').spawnSync(
|
|
159
|
+
'/bin/sh',
|
|
160
|
+
[
|
|
161
|
+
path.resolve(__dirname, '../../experiments/issue-195-cpu-monitor.sh'),
|
|
162
|
+
process.execPath,
|
|
163
|
+
require.resolve('../src/lib/cpu-penalty-monitor'),
|
|
164
|
+
'',
|
|
165
|
+
'cpu-task',
|
|
166
|
+
'1',
|
|
167
|
+
JSON.stringify(record),
|
|
168
|
+
],
|
|
169
|
+
{ encoding: 'utf8' }
|
|
170
|
+
);
|
|
171
|
+
expect(result.status).toBe(0);
|
|
172
|
+
const lines = fs.readFileSync(logPath, 'utf8').trim().split('\n');
|
|
173
|
+
expect(lines.length).toBe(3);
|
|
174
|
+
expect(lines[0]).toContain('CPU penalty applied: 2');
|
|
175
|
+
expect(lines[1]).toContain('CPU penalty lifted: 6');
|
|
176
|
+
expect(lines[2]).toContain('CPU penalty applied: 2');
|
|
177
|
+
const state = JSON.parse(
|
|
178
|
+
fs.readFileSync(`${logPath}.cpu-penalty.json`, 'utf8')
|
|
179
|
+
);
|
|
180
|
+
expect(state.penaltyCount).toBe(2);
|
|
181
|
+
expect(state.phase).toBe('penalized');
|
|
182
|
+
expect(state.penalizedMs).toBeGreaterThan(200);
|
|
183
|
+
} finally {
|
|
184
|
+
fs.rmSync(dir, { recursive: true, force: true });
|
|
185
|
+
}
|
|
186
|
+
}, 15000);
|
|
187
|
+
|
|
188
|
+
it('preserves legacy CPU quota format and the lower original base', () => {
|
|
189
|
+
const limits = require('../src/lib/docker-resource-limits');
|
|
190
|
+
const base = ['--cpu-period=200000', '--cpu-quota=100000'];
|
|
191
|
+
expect(limits.cpuCount(base)).toBe(0.5);
|
|
192
|
+
expect(limits.withCpuLimit(base, 0.25)).toEqual([
|
|
193
|
+
'--cpu-period=100000',
|
|
194
|
+
'--cpu-quota=25000',
|
|
195
|
+
]);
|
|
196
|
+
expect(limits.cpuUpdateArgs(base, 0.25)).toEqual([
|
|
197
|
+
'--cpu-period=100000',
|
|
198
|
+
'--cpu-quota=25000',
|
|
199
|
+
]);
|
|
200
|
+
const { clampCpuToDaemon } = require('../src/lib/resume-resources');
|
|
201
|
+
expect(
|
|
202
|
+
clampCpuToDaemon(base, () => ({
|
|
203
|
+
success: true,
|
|
204
|
+
stdout: '{"NCPU":4,"MemTotal":1073741824}',
|
|
205
|
+
}))
|
|
206
|
+
).toEqual(['--cpu-period=100000', '--cpu-quota=50000']);
|
|
207
|
+
const { prepareResumeResources } = require('../src/lib/resume-resources');
|
|
208
|
+
const overridden = prepareResumeResources(
|
|
209
|
+
{
|
|
210
|
+
options: {
|
|
211
|
+
isolated: 'docker',
|
|
212
|
+
sessionName: 'task',
|
|
213
|
+
resourceLimits: base,
|
|
214
|
+
},
|
|
215
|
+
},
|
|
216
|
+
{ state: 'stopped' },
|
|
217
|
+
{ cpus: '0.25' },
|
|
218
|
+
() => ({ success: true, stdout: '{"CpuPeriod":200000,"CpuQuota":100000}' })
|
|
219
|
+
);
|
|
220
|
+
expect(overridden.resourceLimits).toEqual([
|
|
221
|
+
'--cpu-period=100000',
|
|
222
|
+
'--cpu-quota=25000',
|
|
223
|
+
]);
|
|
224
|
+
});
|