start-command 0.35.4 → 0.37.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +14 -0
- package/README.md +16 -0
- package/package.json +1 -1
- package/src/bin/cli.js +48 -7
- package/src/lib/args-parser-queries.js +14 -0
- package/src/lib/args-parser.js +32 -1
- package/src/lib/atomic-write.js +34 -0
- package/src/lib/attached-diagnostics.js +90 -0
- package/src/lib/cgroup-memory.js +43 -11
- package/src/lib/command-builder.js +43 -0
- package/src/lib/cpu-penalty-monitor.js +226 -0
- package/src/lib/cpu-penalty.js +175 -0
- package/src/lib/detached-finalize.js +16 -0
- package/src/lib/docker-cleanup.js +35 -7
- package/src/lib/docker-command-handoff.js +75 -0
- package/src/lib/docker-labels.js +33 -0
- package/src/lib/docker-resource-limits.js +28 -0
- package/src/lib/docker-resource-options.js +180 -0
- package/src/lib/docker-runtime-args.js +23 -0
- package/src/lib/docker-snapshot-safety.js +164 -0
- package/src/lib/execution-recovery.js +153 -30
- package/src/lib/execution-resume-all.js +11 -0
- package/src/lib/execution-resume.js +314 -54
- package/src/lib/execution-store.js +25 -121
- package/src/lib/exit-evidence.js +123 -0
- package/src/lib/exit-reason.js +24 -6
- package/src/lib/failure-handler.js +15 -5
- package/src/lib/isolation.js +48 -7
- package/src/lib/launch-owner.js +25 -0
- package/src/lib/launch-persistence.js +54 -0
- package/src/lib/log-sanitizer.js +267 -0
- package/src/lib/log-uploader.js +38 -19
- package/src/lib/query-commands.js +18 -6
- package/src/lib/resume-resources.js +88 -0
- package/src/lib/shell-utils.js +67 -17
- package/src/lib/status-formatter.js +28 -0
- package/src/lib/store-lock.js +133 -0
- package/src/lib/usage.js +16 -1
- package/test/ci-workflow-invariants.js +15 -14
- package/test/docker-command-handoff.js +107 -0
- package/test/docker-runtime-options.js +10 -0
- package/test/docker-snapshot-safety.js +106 -0
- package/test/execution-resume.js +19 -11
- package/test/exit-reason.js +6 -1
- package/test/failure-handler.js +11 -0
- package/test/isolation.js +5 -4
- package/test/javascript-first-ci.js +216 -0
- package/test/log-sanitizer.js +306 -0
- package/test/regression-174.js +3 -1
- package/test/regression-176.js +7 -0
- package/test/regression-180.js +17 -10
- package/test/regression-182.js +7 -3
- package/test/regression-185.js +4 -9
- package/test/regression-187.js +11 -2
- package/test/regression-189.js +224 -0
- package/test/regression-190.js +111 -0
- package/test/regression-191.js +160 -0
- package/test/regression-192.js +66 -0
- package/test/regression-193.js +251 -0
- package/test/regression-194.js +182 -0
- package/test/regression-198.js +281 -0
- package/test/regression-199.js +79 -0
- package/test/regression-201.js +25 -0
- package/test/regression-202.js +92 -0
- package/test/regression-91.js +23 -20
- package/test/shared-golden.js +54 -0
- package/test/status-query.js +97 -15
|
@@ -0,0 +1,226 @@
|
|
|
1
|
+
/** A watcher-owned monitor. Sidecars avoid writing the store for every sample. */
|
|
2
|
+
const fs = require('fs');
|
|
3
|
+
const path = require('path');
|
|
4
|
+
const { spawnSync } = require('child_process');
|
|
5
|
+
const { initialState, evaluate } = require('./cpu-penalty');
|
|
6
|
+
const { LockManager } = require('./store-lock');
|
|
7
|
+
const { shellQuote } = require('./isolation-log-utils');
|
|
8
|
+
const { cpuCount, cpuUpdateArgs } = require('./docker-resource-limits');
|
|
9
|
+
function statePath(record) {
|
|
10
|
+
return `${record.logPath}.cpu-penalty.json`;
|
|
11
|
+
}
|
|
12
|
+
function readState(record) {
|
|
13
|
+
if (!record.options?.cpuPenaltyConfig || !record.logPath) {
|
|
14
|
+
return null;
|
|
15
|
+
}
|
|
16
|
+
try {
|
|
17
|
+
const state = JSON.parse(fs.readFileSync(statePath(record), 'utf8'));
|
|
18
|
+
if (
|
|
19
|
+
state.containerName !== record.options.sessionName ||
|
|
20
|
+
state.attemptNumber !== (record.attempt?.number || 1)
|
|
21
|
+
) {
|
|
22
|
+
return null;
|
|
23
|
+
}
|
|
24
|
+
if (state.phase === 'penalized' && Number.isFinite(state.updatedAt)) {
|
|
25
|
+
const until = record.endTime ? Date.parse(record.endTime) : Date.now();
|
|
26
|
+
state.penalizedMs += Math.max(0, until - state.updatedAt);
|
|
27
|
+
state.updatedAt = Math.max(until, state.updatedAt);
|
|
28
|
+
}
|
|
29
|
+
return state;
|
|
30
|
+
} catch {
|
|
31
|
+
return null;
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
function publicState(state) {
|
|
35
|
+
const result = { ...state };
|
|
36
|
+
delete result.samples;
|
|
37
|
+
delete result.lastAt;
|
|
38
|
+
delete result.capacity;
|
|
39
|
+
return result;
|
|
40
|
+
}
|
|
41
|
+
function run(command, args) {
|
|
42
|
+
const result = spawnSync(command, args, { encoding: 'utf8' });
|
|
43
|
+
return { success: result.status === 0, stdout: result.stdout || '' };
|
|
44
|
+
}
|
|
45
|
+
async function monitor(uuid, name, attemptNumber, standalone) {
|
|
46
|
+
const { ExecutionStore } = require('./execution-store');
|
|
47
|
+
const store = uuid
|
|
48
|
+
? new ExecutionStore(
|
|
49
|
+
process.env.START_APP_FOLDER
|
|
50
|
+
? { appFolder: path.resolve(process.env.START_APP_FOLDER) }
|
|
51
|
+
: {}
|
|
52
|
+
)
|
|
53
|
+
: null;
|
|
54
|
+
const record = store ? store.get(uuid) : JSON.parse(standalone || 'null');
|
|
55
|
+
if (
|
|
56
|
+
!record?.options.cpuPenaltyConfig ||
|
|
57
|
+
!record.logPath ||
|
|
58
|
+
record.options.sessionName !== name
|
|
59
|
+
) {
|
|
60
|
+
return;
|
|
61
|
+
}
|
|
62
|
+
const config = record.options.cpuPenaltyConfig;
|
|
63
|
+
const lock = new LockManager(`${statePath(record)}.lock`);
|
|
64
|
+
if (!lock.acquire(100)) {
|
|
65
|
+
return;
|
|
66
|
+
}
|
|
67
|
+
const docker = require('./docker-cleanup').getDockerCommand();
|
|
68
|
+
const baseFlags =
|
|
69
|
+
record.options.baseResourceLimits || record.options.resourceLimits || [];
|
|
70
|
+
const baseCpus = cpuCount(baseFlags);
|
|
71
|
+
let state = initialState(baseCpus, Date.now(), readState(record));
|
|
72
|
+
const interval = Math.min(
|
|
73
|
+
30000,
|
|
74
|
+
Math.max(50, Math.min(config.triggerWindowMs, config.releaseWindowMs) / 8)
|
|
75
|
+
);
|
|
76
|
+
const publish = () => {
|
|
77
|
+
const file = statePath(record),
|
|
78
|
+
tmp = `${file}.${process.pid}.tmp`;
|
|
79
|
+
try {
|
|
80
|
+
fs.writeFileSync(
|
|
81
|
+
tmp,
|
|
82
|
+
JSON.stringify({
|
|
83
|
+
...publicState(state),
|
|
84
|
+
containerName: name,
|
|
85
|
+
attemptNumber: Number(attemptNumber),
|
|
86
|
+
updatedAt: Date.now(),
|
|
87
|
+
})
|
|
88
|
+
);
|
|
89
|
+
fs.renameSync(tmp, file);
|
|
90
|
+
} finally {
|
|
91
|
+
try {
|
|
92
|
+
fs.unlinkSync(tmp);
|
|
93
|
+
} catch {
|
|
94
|
+
/* Optional diagnostics are best effort. */
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
};
|
|
98
|
+
const log = (text) => {
|
|
99
|
+
try {
|
|
100
|
+
fs.appendFileSync(record.logPath, `${text}\n`);
|
|
101
|
+
} catch {
|
|
102
|
+
/* Optional diagnostics are best effort. */
|
|
103
|
+
}
|
|
104
|
+
};
|
|
105
|
+
const started = Date.now();
|
|
106
|
+
try {
|
|
107
|
+
publish();
|
|
108
|
+
while (true) {
|
|
109
|
+
const current = store ? store.get(uuid) : record;
|
|
110
|
+
if (
|
|
111
|
+
!current ||
|
|
112
|
+
current.options.sessionName !== name ||
|
|
113
|
+
(current.attempt?.number || 1) !== Number(attemptNumber)
|
|
114
|
+
) {
|
|
115
|
+
break;
|
|
116
|
+
}
|
|
117
|
+
const live = run(docker, ['inspect', '-f', '{{.State.Running}}', name]);
|
|
118
|
+
if (live.success && live.stdout.trim() === 'false') {
|
|
119
|
+
break;
|
|
120
|
+
}
|
|
121
|
+
if (!live.success && Date.now() - started > 30000) {
|
|
122
|
+
break;
|
|
123
|
+
}
|
|
124
|
+
const info = run(docker, ['info', '--format', '{{json .}}']);
|
|
125
|
+
const stats = run(docker, [
|
|
126
|
+
'stats',
|
|
127
|
+
'--no-stream',
|
|
128
|
+
'--format',
|
|
129
|
+
'{{json .}}',
|
|
130
|
+
]);
|
|
131
|
+
let daemonCpus = NaN,
|
|
132
|
+
cores = NaN;
|
|
133
|
+
try {
|
|
134
|
+
if (info.success) {
|
|
135
|
+
daemonCpus = JSON.parse(info.stdout).NCPU;
|
|
136
|
+
}
|
|
137
|
+
if (stats.success) {
|
|
138
|
+
const row = stats.stdout
|
|
139
|
+
.trim()
|
|
140
|
+
.split('\n')
|
|
141
|
+
.map((s) => JSON.parse(s))
|
|
142
|
+
.find((s) => s.Name === name);
|
|
143
|
+
if (row && /^\d+(?:\.\d+)?%$/.test(row.CPUPerc)) {
|
|
144
|
+
cores = Number(row.CPUPerc.slice(0, -1)) / 100;
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
} catch {
|
|
148
|
+
/* Optional diagnostics are best effort. */
|
|
149
|
+
}
|
|
150
|
+
const decision = evaluate(
|
|
151
|
+
state,
|
|
152
|
+
{
|
|
153
|
+
now: Date.now(),
|
|
154
|
+
cores,
|
|
155
|
+
daemonCpus,
|
|
156
|
+
maxGapMs: Math.max(5000, interval * 3),
|
|
157
|
+
},
|
|
158
|
+
config
|
|
159
|
+
);
|
|
160
|
+
if (decision.action) {
|
|
161
|
+
const action = decision.action;
|
|
162
|
+
const update = run(docker, [
|
|
163
|
+
'update',
|
|
164
|
+
...cpuUpdateArgs(baseFlags, action.cpus),
|
|
165
|
+
name,
|
|
166
|
+
]);
|
|
167
|
+
if (update.success) {
|
|
168
|
+
state = decision.state;
|
|
169
|
+
log(
|
|
170
|
+
`[start] CPU penalty ${action.kind === 'apply' ? 'applied' : 'lifted'}: ${action.cpus} CPUs (${action.average === null || action.average === undefined ? 'capacity changed' : `${action.average.toFixed(2)} of ${action.capacity} CPUs`})`
|
|
171
|
+
);
|
|
172
|
+
} else {
|
|
173
|
+
state = {
|
|
174
|
+
...decision.state,
|
|
175
|
+
phase: state.phase,
|
|
176
|
+
since: state.since,
|
|
177
|
+
limitCpus: state.limitCpus,
|
|
178
|
+
penaltyCount: state.penaltyCount,
|
|
179
|
+
samples: [],
|
|
180
|
+
};
|
|
181
|
+
}
|
|
182
|
+
} else {
|
|
183
|
+
state = decision.state;
|
|
184
|
+
}
|
|
185
|
+
publish();
|
|
186
|
+
await new Promise((resolve) => globalThis.setTimeout(resolve, interval));
|
|
187
|
+
}
|
|
188
|
+
} finally {
|
|
189
|
+
lock.release();
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
function startSnippet(uuid, name, attempt = 1, options = null) {
|
|
193
|
+
const standalone =
|
|
194
|
+
!uuid && options?.cpuPenaltyConfig && options.logPath
|
|
195
|
+
? JSON.stringify({
|
|
196
|
+
logPath: options.logPath,
|
|
197
|
+
options: {
|
|
198
|
+
sessionName: name,
|
|
199
|
+
cpuPenaltyConfig: options.cpuPenaltyConfig,
|
|
200
|
+
baseResourceLimits: options.resourceLimits,
|
|
201
|
+
},
|
|
202
|
+
})
|
|
203
|
+
: null;
|
|
204
|
+
if (!uuid && !standalone) {
|
|
205
|
+
return '';
|
|
206
|
+
}
|
|
207
|
+
return `${shellQuote(process.execPath)} ${shellQuote(__filename)} ${shellQuote(uuid || '')} ${shellQuote(name)} ${Number(attempt)} ${shellQuote(standalone || '')} >/dev/null 2>&1 & __start_cpu_monitor=$!`;
|
|
208
|
+
}
|
|
209
|
+
function stopSnippet() {
|
|
210
|
+
return 'if [ -n "$__start_cpu_monitor" ]; then kill "$__start_cpu_monitor" 2>/dev/null; wait "$__start_cpu_monitor" 2>/dev/null; fi';
|
|
211
|
+
}
|
|
212
|
+
module.exports = {
|
|
213
|
+
statePath,
|
|
214
|
+
readState,
|
|
215
|
+
publicState,
|
|
216
|
+
monitor,
|
|
217
|
+
startSnippet,
|
|
218
|
+
stopSnippet,
|
|
219
|
+
};
|
|
220
|
+
if (require.main === module) {
|
|
221
|
+
monitor(...process.argv.slice(2)).catch((error) => {
|
|
222
|
+
if (process.env.START_DEBUG === '1') {
|
|
223
|
+
console.error(`[DEBUG] CPU monitor: ${error.stack}`);
|
|
224
|
+
}
|
|
225
|
+
});
|
|
226
|
+
}
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
/** Docker CPU hysteresis with timestamp-weighted, fully covered windows. */
|
|
2
|
+
const DEFAULTS = {
|
|
3
|
+
cpus: 2,
|
|
4
|
+
trigger: 95,
|
|
5
|
+
triggerWindowMs: 900000,
|
|
6
|
+
release: 65,
|
|
7
|
+
releaseWindowMs: 900000,
|
|
8
|
+
};
|
|
9
|
+
const FIELDS = {
|
|
10
|
+
'--cpu-penalty-cpus': 'cpus',
|
|
11
|
+
'--cpu-penalty-trigger': 'trigger',
|
|
12
|
+
'--cpu-penalty-trigger-window': 'triggerWindowMs',
|
|
13
|
+
'--cpu-penalty-release': 'release',
|
|
14
|
+
'--cpu-penalty-release-window': 'releaseWindowMs',
|
|
15
|
+
};
|
|
16
|
+
function duration(value) {
|
|
17
|
+
const m = /^(\d+(?:\.\d+)?)(ms|s|m|h)$/.exec(value);
|
|
18
|
+
const n = m
|
|
19
|
+
? Number(m[1]) * { ms: 1, s: 1000, m: 60000, h: 3600000 }[m[2]]
|
|
20
|
+
: NaN;
|
|
21
|
+
if (!Number.isSafeInteger(n) || n <= 0) {
|
|
22
|
+
throw Error(`Invalid CPU penalty duration: ${value}`);
|
|
23
|
+
}
|
|
24
|
+
return n;
|
|
25
|
+
}
|
|
26
|
+
function parse(args, index, options) {
|
|
27
|
+
const [flag, inline] = args[index].split('=', 2);
|
|
28
|
+
if (flag === '--cpu-penalty') {
|
|
29
|
+
if (inline !== undefined) {
|
|
30
|
+
throw Error('--cpu-penalty does not take a value');
|
|
31
|
+
}
|
|
32
|
+
options.cpuPenalty = true;
|
|
33
|
+
return 1;
|
|
34
|
+
}
|
|
35
|
+
const field = FIELDS[flag];
|
|
36
|
+
if (!field) {
|
|
37
|
+
return 0;
|
|
38
|
+
}
|
|
39
|
+
const value = inline ?? args[index + 1];
|
|
40
|
+
if (!value) {
|
|
41
|
+
throw Error(`${flag} requires a value`);
|
|
42
|
+
}
|
|
43
|
+
const number = field.endsWith('Ms')
|
|
44
|
+
? duration(value)
|
|
45
|
+
: Number(field === 'cpus' ? value : value.replace(/%$/, ''));
|
|
46
|
+
if (
|
|
47
|
+
!(number > 0) ||
|
|
48
|
+
!Number.isFinite(number) ||
|
|
49
|
+
(field !== 'cpus' && !field.endsWith('Ms') && number > 100)
|
|
50
|
+
) {
|
|
51
|
+
throw Error(`Invalid ${flag}: ${value}`);
|
|
52
|
+
}
|
|
53
|
+
options.cpuPenaltyConfig = {
|
|
54
|
+
...DEFAULTS,
|
|
55
|
+
...options.cpuPenaltyConfig,
|
|
56
|
+
[field]: number,
|
|
57
|
+
};
|
|
58
|
+
return inline === undefined ? 2 : 1;
|
|
59
|
+
}
|
|
60
|
+
function validate(options) {
|
|
61
|
+
if (!options.cpuPenalty && !options.cpuPenaltyConfig) {
|
|
62
|
+
return;
|
|
63
|
+
}
|
|
64
|
+
if (!options.cpuPenalty) {
|
|
65
|
+
throw Error('CPU penalty settings require --cpu-penalty');
|
|
66
|
+
}
|
|
67
|
+
if (!(options.isolatedStack || [options.isolated]).includes('docker')) {
|
|
68
|
+
throw Error('--cpu-penalty requires Docker isolation');
|
|
69
|
+
}
|
|
70
|
+
options.cpuPenaltyConfig = { ...DEFAULTS, ...options.cpuPenaltyConfig };
|
|
71
|
+
}
|
|
72
|
+
function initialState(baseCpus = null, now = Date.now(), saved = null) {
|
|
73
|
+
return {
|
|
74
|
+
phase: saved?.phase === 'penalized' ? 'penalized' : 'observing',
|
|
75
|
+
since: saved?.phase === 'penalized' ? saved.since : now,
|
|
76
|
+
limitCpus: saved?.phase === 'penalized' ? saved.limitCpus : baseCpus,
|
|
77
|
+
baseCpus,
|
|
78
|
+
penaltyCount: saved?.penaltyCount || 0,
|
|
79
|
+
penalizedMs: saved?.penalizedMs || 0,
|
|
80
|
+
samples: [],
|
|
81
|
+
lastAt: now,
|
|
82
|
+
capacity: null,
|
|
83
|
+
};
|
|
84
|
+
}
|
|
85
|
+
function average(samples, now, window) {
|
|
86
|
+
const start = now - window;
|
|
87
|
+
if (
|
|
88
|
+
samples.length < 2 ||
|
|
89
|
+
samples[0].at > start ||
|
|
90
|
+
samples.at(-1).at !== now
|
|
91
|
+
) {
|
|
92
|
+
return null;
|
|
93
|
+
}
|
|
94
|
+
let weighted = 0;
|
|
95
|
+
for (let i = 1; i < samples.length; i++) {
|
|
96
|
+
const left = Math.max(start, samples[i - 1].at),
|
|
97
|
+
right = samples[i].at;
|
|
98
|
+
if (right > left) {
|
|
99
|
+
weighted += (right - left) * samples[i - 1].cores;
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
return weighted / window;
|
|
103
|
+
}
|
|
104
|
+
function evaluate(
|
|
105
|
+
state,
|
|
106
|
+
{ now, cores, daemonCpus, maxGapMs },
|
|
107
|
+
config = DEFAULTS
|
|
108
|
+
) {
|
|
109
|
+
const s = { ...state, samples: [...state.samples] };
|
|
110
|
+
if (s.phase === 'penalized') {
|
|
111
|
+
s.penalizedMs += Math.max(0, now - s.lastAt);
|
|
112
|
+
}
|
|
113
|
+
s.lastAt = now;
|
|
114
|
+
const capacity = Number.isFinite(daemonCpus)
|
|
115
|
+
? Math.min(s.baseCpus || Infinity, daemonCpus)
|
|
116
|
+
: NaN;
|
|
117
|
+
if (
|
|
118
|
+
!(cores >= 0) ||
|
|
119
|
+
!Number.isFinite(cores) ||
|
|
120
|
+
!(capacity > 0) ||
|
|
121
|
+
now <= (s.samples.at(-1)?.at ?? -Infinity) ||
|
|
122
|
+
(s.samples.length && now - s.samples.at(-1).at > maxGapMs) ||
|
|
123
|
+
s.capacity !== capacity
|
|
124
|
+
) {
|
|
125
|
+
s.samples = [];
|
|
126
|
+
}
|
|
127
|
+
s.capacity = capacity;
|
|
128
|
+
if (!(cores >= 0) || !Number.isFinite(cores) || !(capacity > 0)) {
|
|
129
|
+
return { state: s, action: null };
|
|
130
|
+
}
|
|
131
|
+
s.samples.push({ at: now, cores });
|
|
132
|
+
const window =
|
|
133
|
+
s.phase === 'penalized' ? config.releaseWindowMs : config.triggerWindowMs;
|
|
134
|
+
while (s.samples.length > 2 && s.samples[1].at <= now - window) {
|
|
135
|
+
s.samples.shift();
|
|
136
|
+
}
|
|
137
|
+
const avg = average(s.samples, now, window);
|
|
138
|
+
let action = null;
|
|
139
|
+
if (
|
|
140
|
+
s.phase === 'observing' &&
|
|
141
|
+
config.cpus < capacity &&
|
|
142
|
+
avg !== null &&
|
|
143
|
+
avg >= (config.trigger / 100) * capacity
|
|
144
|
+
) {
|
|
145
|
+
action = { kind: 'apply', cpus: config.cpus, average: avg, capacity };
|
|
146
|
+
s.phase = 'penalized';
|
|
147
|
+
s.since = now;
|
|
148
|
+
s.limitCpus = config.cpus;
|
|
149
|
+
s.penaltyCount++;
|
|
150
|
+
} else if (
|
|
151
|
+
s.phase === 'penalized' &&
|
|
152
|
+
(config.cpus >= capacity ||
|
|
153
|
+
(now - s.since >= config.releaseWindowMs &&
|
|
154
|
+
avg !== null &&
|
|
155
|
+
avg < (config.release / 100) * config.cpus))
|
|
156
|
+
) {
|
|
157
|
+
action = { kind: 'lift', cpus: capacity, average: avg, capacity };
|
|
158
|
+
s.phase = 'observing';
|
|
159
|
+
s.since = now;
|
|
160
|
+
s.limitCpus = capacity;
|
|
161
|
+
}
|
|
162
|
+
if (action) {
|
|
163
|
+
s.samples = [];
|
|
164
|
+
}
|
|
165
|
+
return { state: s, action };
|
|
166
|
+
}
|
|
167
|
+
module.exports = {
|
|
168
|
+
DEFAULTS,
|
|
169
|
+
parse,
|
|
170
|
+
validate,
|
|
171
|
+
duration,
|
|
172
|
+
initialState,
|
|
173
|
+
average,
|
|
174
|
+
evaluate,
|
|
175
|
+
};
|
|
@@ -146,6 +146,21 @@ function finalizeDetachedExecution(options = {}) {
|
|
|
146
146
|
record.cgroupMemory = cgroupMemory;
|
|
147
147
|
}
|
|
148
148
|
|
|
149
|
+
const evidence = require('./exit-evidence');
|
|
150
|
+
const logged = evidence.fromLog(
|
|
151
|
+
record.logPath
|
|
152
|
+
? require('./execution-attempt').readAttemptLogTail(record, 65536)
|
|
153
|
+
: null
|
|
154
|
+
);
|
|
155
|
+
record.options.exitEvidence = {
|
|
156
|
+
daemonRestart: logged.daemonRestart,
|
|
157
|
+
mainOom:
|
|
158
|
+
!logged.daemonRestart &&
|
|
159
|
+
(logged.mainOom ||
|
|
160
|
+
(evidence.recentOomDelta(options.cgroupMemory, options.finishedAt) &&
|
|
161
|
+
record.oomKilled === true &&
|
|
162
|
+
(record.exitCode === 137 || record.exitCode < 0))),
|
|
163
|
+
};
|
|
149
164
|
const exitReason =
|
|
150
165
|
resolveReason(record) || (finishedAt ? null : WATCHER_LOST_CONTAINER);
|
|
151
166
|
if (exitReason) {
|
|
@@ -197,6 +212,7 @@ function resolveReason(record) {
|
|
|
197
212
|
logTail,
|
|
198
213
|
oomKilled: record.oomKilled,
|
|
199
214
|
cgroupMemory: record.cgroupMemory,
|
|
215
|
+
exitEvidence: record.options?.exitEvidence,
|
|
200
216
|
});
|
|
201
217
|
} catch {
|
|
202
218
|
return null;
|
|
@@ -83,7 +83,7 @@ function getDockerContainerCleanupInstructions(containerName) {
|
|
|
83
83
|
`Container kept for investigation: ${containerName}`,
|
|
84
84
|
`Re-enter while running: $ --attach ${containerName}`,
|
|
85
85
|
`Continue the stored command: $ --resume ${containerName}`,
|
|
86
|
-
`Run another command
|
|
86
|
+
`Run another command (legacy containers snapshot the filesystem into a new container): $ --resume ${containerName} -- <command>`,
|
|
87
87
|
`Remove when done: docker rm -f ${containerName}`,
|
|
88
88
|
].join('\n');
|
|
89
89
|
}
|
|
@@ -275,10 +275,13 @@ function buildAttachedDockerKeptMessage({
|
|
|
275
275
|
oomKilled,
|
|
276
276
|
logPath,
|
|
277
277
|
}) {
|
|
278
|
+
const logTail = logPath
|
|
279
|
+
? readLogTail(logPath, FATAL_MARKER_TAIL_BYTES)
|
|
280
|
+
: null;
|
|
278
281
|
let message;
|
|
279
282
|
if (oomKilled !== true) {
|
|
280
283
|
message = `\nContainer kept because the command failed.`;
|
|
281
|
-
} else if (isOomKillOfCommand({ exitCode, oomKilled })) {
|
|
284
|
+
} else if (isOomKillOfCommand({ exitCode, oomKilled, logTail })) {
|
|
282
285
|
message = `\nContainer kept because Docker reports it was OOM-killed.`;
|
|
283
286
|
} else {
|
|
284
287
|
// The flag is container-wide (#180): a child was OOM-killed, the command
|
|
@@ -287,7 +290,7 @@ function buildAttachedDockerKeptMessage({
|
|
|
287
290
|
}
|
|
288
291
|
const memory = resolveMemoryExhaustion({
|
|
289
292
|
exitCode,
|
|
290
|
-
logTail
|
|
293
|
+
logTail,
|
|
291
294
|
oomKilled,
|
|
292
295
|
});
|
|
293
296
|
if (memory) {
|
|
@@ -303,7 +306,7 @@ function buildDockerKeptLogSnippet(containerName, quotedLogPath) {
|
|
|
303
306
|
`printf '\\nContainer kept for investigation: %s\\nReason: %s\\n` +
|
|
304
307
|
`Re-enter while running: $ --attach %s\\n` +
|
|
305
308
|
`Continue the stored command: $ --resume %s\\n` +
|
|
306
|
-
`Run another command
|
|
309
|
+
`Run another command (legacy containers snapshot the filesystem into a new container): $ --resume %s -- <command>\\n` +
|
|
307
310
|
`Remove when done: docker rm -f %s\\n' ` +
|
|
308
311
|
`${quotedName} "$__start_command_reason" ` +
|
|
309
312
|
`${quotedName} ${quotedName} ${quotedName} ${quotedName} >> ${quotedLogPath}`
|
|
@@ -358,9 +361,24 @@ function buildDetachedDockerCompletionScript(
|
|
|
358
361
|
watcherOptions = {}
|
|
359
362
|
) {
|
|
360
363
|
const quotedName = shellQuote(containerName);
|
|
364
|
+
const imageCleanup =
|
|
365
|
+
require('./docker-snapshot-safety').snapshotImageCleanupSnippet(
|
|
366
|
+
containerName,
|
|
367
|
+
logPath ? `>> ${shellQuote(logPath)} 2>&1` : '>/dev/null 2>&1'
|
|
368
|
+
);
|
|
361
369
|
// The container's cgroup disappears with it, so its memory counters are
|
|
362
370
|
// sampled while it runs (issue #182).
|
|
371
|
+
const cpuMonitor = require('./cpu-penalty-monitor');
|
|
363
372
|
const parts = [buildCgroupSamplerStartSnippet(containerName)];
|
|
373
|
+
const cpuStart = cpuMonitor.startSnippet(
|
|
374
|
+
executionId,
|
|
375
|
+
containerName,
|
|
376
|
+
watcherOptions.attemptNumber || 1,
|
|
377
|
+
watcherOptions.cpuOptions
|
|
378
|
+
);
|
|
379
|
+
if (cpuStart) {
|
|
380
|
+
parts.push(cpuStart);
|
|
381
|
+
}
|
|
364
382
|
// Everything that assumes the container has exited: cleanup, footer and
|
|
365
383
|
// finalization. Guarded as a whole by `.State.Running` below.
|
|
366
384
|
const exited = [];
|
|
@@ -376,7 +394,7 @@ function buildDetachedDockerCompletionScript(
|
|
|
376
394
|
quotedLogPath
|
|
377
395
|
);
|
|
378
396
|
const memory = buildCgroupMemoryLogSnippet(quotedLogPath);
|
|
379
|
-
const remove =
|
|
397
|
+
const remove = `${imageCleanup.capture}; if docker rm -f ${quotedName} >> ${quotedLogPath} 2>&1; then ${imageCleanup.remove}; fi; ${removalNote}; ${memory}`;
|
|
380
398
|
const keep = `${postMortem}; ${memory}; ${buildDockerKeptLogSnippet(containerName, quotedLogPath)}`;
|
|
381
399
|
|
|
382
400
|
const since = watcherOptions.since
|
|
@@ -393,8 +411,15 @@ function buildDetachedDockerCompletionScript(
|
|
|
393
411
|
);
|
|
394
412
|
}
|
|
395
413
|
parts.push(buildDockerWaitForExitSnippet(containerName, quotedLogPath));
|
|
414
|
+
parts.push(cpuMonitor.stopSnippet());
|
|
396
415
|
parts.push(buildCgroupSamplerStopSnippet());
|
|
397
416
|
parts.push(buildDockerStateSnippet(containerName));
|
|
417
|
+
parts.push(
|
|
418
|
+
require('./exit-evidence').buildExitEvidenceSnippet(
|
|
419
|
+
containerName,
|
|
420
|
+
quotedLogPath
|
|
421
|
+
)
|
|
422
|
+
);
|
|
398
423
|
if (policy === DOCKER_CONTAINER_CLEANUP_POLICY.ALWAYS) {
|
|
399
424
|
exited.push(remove);
|
|
400
425
|
} else if (
|
|
@@ -414,16 +439,19 @@ function buildDetachedDockerCompletionScript(
|
|
|
414
439
|
} else {
|
|
415
440
|
parts.push(`docker wait ${quotedName} >/dev/null 2>&1`);
|
|
416
441
|
parts.push(buildDockerWaitForExitSnippet(containerName));
|
|
442
|
+
parts.push(cpuMonitor.stopSnippet());
|
|
417
443
|
parts.push(buildCgroupSamplerStopSnippet());
|
|
418
444
|
parts.push(buildDockerStateSnippet(containerName));
|
|
419
445
|
if (policy === DOCKER_CONTAINER_CLEANUP_POLICY.ALWAYS) {
|
|
420
|
-
exited.push(
|
|
446
|
+
exited.push(
|
|
447
|
+
`${imageCleanup.capture}; if docker rm -f ${quotedName} >/dev/null 2>&1; then ${imageCleanup.remove}; fi`
|
|
448
|
+
);
|
|
421
449
|
} else if (
|
|
422
450
|
policy === DOCKER_CONTAINER_CLEANUP_POLICY.DEFAULT ||
|
|
423
451
|
policy === DOCKER_CONTAINER_CLEANUP_POLICY.KEEP_ON_FAIL
|
|
424
452
|
) {
|
|
425
453
|
exited.push(
|
|
426
|
-
`if ${buildSuccessfulNonOomCondition()}; then docker rm -f ${quotedName} >/dev/null 2>&1
|
|
454
|
+
`if ${buildSuccessfulNonOomCondition()}; then ${imageCleanup.capture}; if docker rm -f ${quotedName} >/dev/null 2>&1; then ${imageCleanup.remove}; fi; fi`
|
|
427
455
|
);
|
|
428
456
|
}
|
|
429
457
|
}
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
/** Copy only a command script into a stopped container, never its writable layer. */
|
|
2
|
+
const fs = require('fs');
|
|
3
|
+
const os = require('os');
|
|
4
|
+
const path = require('path');
|
|
5
|
+
const { shellQuote } = require('./isolation-log-utils');
|
|
6
|
+
|
|
7
|
+
function commandHandoffPath(sessionName) {
|
|
8
|
+
return `/.start-command-resume-${String(sessionName).replace(/[^A-Za-z0-9_.-]/g, '_')}`;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
function buildCommandHandoffArgs(mainArgs, markerPath) {
|
|
12
|
+
return [
|
|
13
|
+
'sh',
|
|
14
|
+
'-c',
|
|
15
|
+
'p=$1; shift; if [ -f "$p" ]; then exec sh "$p"; fi; exec "$@"',
|
|
16
|
+
'start-command-handoff',
|
|
17
|
+
markerPath,
|
|
18
|
+
...mainArgs,
|
|
19
|
+
];
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
function buildCommandHandoffScript(command, options = {}) {
|
|
23
|
+
const {
|
|
24
|
+
isInteractiveShellCommand,
|
|
25
|
+
isShellInvocationWithArgs,
|
|
26
|
+
buildShellWithArgsCmdArgs,
|
|
27
|
+
toShellWords,
|
|
28
|
+
} = require('./shell-utils');
|
|
29
|
+
if (
|
|
30
|
+
!options.keepAlive &&
|
|
31
|
+
(isInteractiveShellCommand(command) || isShellInvocationWithArgs(command))
|
|
32
|
+
) {
|
|
33
|
+
const args = isShellInvocationWithArgs(command)
|
|
34
|
+
? buildShellWithArgsCmdArgs(command)
|
|
35
|
+
: toShellWords(command);
|
|
36
|
+
return `exec ${args.map(shellQuote).join(' ')}\n`;
|
|
37
|
+
}
|
|
38
|
+
if (!options.shell || options.shell === 'auto') {
|
|
39
|
+
const effective = options.keepAlive
|
|
40
|
+
? `${command}; exec "$__START_COMMAND_HANDOFF_SHELL"`
|
|
41
|
+
: command;
|
|
42
|
+
return `__START_COMMAND_HANDOFF_SHELL=$(command -v bash || command -v zsh || command -v sh); export __START_COMMAND_HANDOFF_SHELL\ncase "$__START_COMMAND_HANDOFF_SHELL" in */bash|bash|*/zsh|zsh) exec "$__START_COMMAND_HANDOFF_SHELL" -i -c ${shellQuote(effective)};; *) exec "$__START_COMMAND_HANDOFF_SHELL" -c ${shellQuote(effective)};; esac\n`;
|
|
43
|
+
}
|
|
44
|
+
const shell = options.shell;
|
|
45
|
+
const interactive = /(?:^|\/)(?:bash|zsh)$/.test(shell) ? ' -i' : '';
|
|
46
|
+
const effective = options.keepAlive
|
|
47
|
+
? `${command}; exec ${shellQuote(shell)}`
|
|
48
|
+
: command;
|
|
49
|
+
return `exec ${shellQuote(shell)}${interactive} -c ${shellQuote(effective)}\n`;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
function writeCommandHandoff(containerName, command, runner, options = {}) {
|
|
53
|
+
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'start-handoff-'));
|
|
54
|
+
const file = path.join(dir, 'command');
|
|
55
|
+
try {
|
|
56
|
+
fs.writeFileSync(file, buildCommandHandoffScript(command, options), {
|
|
57
|
+
mode: 0o644,
|
|
58
|
+
});
|
|
59
|
+
fs.chmodSync(file, 0o644);
|
|
60
|
+
return runner(require('./docker-cleanup').getDockerCommand(), [
|
|
61
|
+
'cp',
|
|
62
|
+
file,
|
|
63
|
+
`${containerName}:${commandHandoffPath(containerName)}`,
|
|
64
|
+
]);
|
|
65
|
+
} finally {
|
|
66
|
+
fs.rmSync(dir, { recursive: true, force: true });
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
module.exports = {
|
|
71
|
+
commandHandoffPath,
|
|
72
|
+
buildCommandHandoffArgs,
|
|
73
|
+
buildCommandHandoffScript,
|
|
74
|
+
writeCommandHandoff,
|
|
75
|
+
};
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
/** Docker attribution shared by initial launches and snapshot resumes (#199). */
|
|
2
|
+
function validateLabel(label) {
|
|
3
|
+
if (typeof label !== 'string' || !/^[^=\s]+=[\s\S]*$/.test(label)) {
|
|
4
|
+
throw new Error('--label requires KEY=VALUE with a nonempty key');
|
|
5
|
+
}
|
|
6
|
+
if (label.startsWith('start-command.')) {
|
|
7
|
+
throw new Error(
|
|
8
|
+
'start-command.* labels are reserved for execution attribution'
|
|
9
|
+
);
|
|
10
|
+
}
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
function dockerLabels(options = {}) {
|
|
14
|
+
const labels = [...(options.labels || [])];
|
|
15
|
+
const session = options.containerName || options.session;
|
|
16
|
+
if (session) {
|
|
17
|
+
const inferredRoot =
|
|
18
|
+
Number(options.resumeCount) > 0
|
|
19
|
+
? session.replace(/(?:-resume-\d+)+$/, '')
|
|
20
|
+
: session;
|
|
21
|
+
labels.push(`start-command.session=${session}`);
|
|
22
|
+
labels.push(
|
|
23
|
+
`start-command.root-session=${options.rootSession || inferredRoot}`
|
|
24
|
+
);
|
|
25
|
+
labels.push(`start-command.resume-count=${options.resumeCount || 0}`);
|
|
26
|
+
}
|
|
27
|
+
if (options.sessionId || options.uuid) {
|
|
28
|
+
labels.push(`start-command.uuid=${options.sessionId || options.uuid}`);
|
|
29
|
+
}
|
|
30
|
+
return labels;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
module.exports = { validateLabel, dockerLabels };
|
|
@@ -200,8 +200,36 @@ function buildResourceLimitsStatusLine(value) {
|
|
|
200
200
|
: null;
|
|
201
201
|
}
|
|
202
202
|
|
|
203
|
+
/** Preserve Docker's CPU representation: an existing quota cannot become NanoCPUs in place. */
|
|
204
|
+
function cpuCount(limits) {
|
|
205
|
+
const get = (prefix) =>
|
|
206
|
+
positive(limits.find((s) => s.startsWith(prefix))?.slice(prefix.length));
|
|
207
|
+
return (
|
|
208
|
+
get('--cpus=') ||
|
|
209
|
+
(get('--cpu-quota=')
|
|
210
|
+
? get('--cpu-quota=') / (get('--cpu-period=') || 100000)
|
|
211
|
+
: null)
|
|
212
|
+
);
|
|
213
|
+
}
|
|
214
|
+
function cpuUpdateArgs(limits, cpus) {
|
|
215
|
+
return !limits.some((s) => s.startsWith('--cpus=')) &&
|
|
216
|
+
limits.some((s) => /^--cpu-(quota|period)=/.test(s))
|
|
217
|
+
? ['--cpu-period=100000', `--cpu-quota=${Math.ceil(cpus * 100000)}`]
|
|
218
|
+
: ['--cpus', String(cpus)];
|
|
219
|
+
}
|
|
220
|
+
function withCpuLimit(limits, cpus) {
|
|
221
|
+
const args = cpuUpdateArgs(limits, cpus);
|
|
222
|
+
return [
|
|
223
|
+
...limits.filter((s) => !/^--(?:cpus|cpu-quota|cpu-period)=/.test(s)),
|
|
224
|
+
...(args[0] === '--cpus' ? [`--cpus=${args[1]}`] : args),
|
|
225
|
+
];
|
|
226
|
+
}
|
|
227
|
+
|
|
203
228
|
module.exports = {
|
|
204
229
|
DEFAULT_SHM_SIZE,
|
|
230
|
+
cpuCount,
|
|
231
|
+
cpuUpdateArgs,
|
|
232
|
+
withCpuLimit,
|
|
205
233
|
buildResourceLimitsStatusLine,
|
|
206
234
|
formatDockerBytes,
|
|
207
235
|
formatDockerCpus,
|