start-command 0.30.0 → 0.30.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +14 -0
- package/package.json +1 -1
- package/src/lib/args-parser.js +5 -5
- package/src/lib/docker-cleanup.js +117 -18
- package/src/lib/execution-control.js +12 -6
- package/src/lib/execution-store.js +11 -2
- package/src/lib/isolation.js +12 -3
- package/src/lib/status-formatter.js +34 -6
- package/src/lib/usage.js +3 -3
- package/test/docker-autoremove.js +47 -0
- package/test/execution-control.js +40 -0
- package/test/run-js-tests.mjs +35 -0
- package/test/session-name-status.js +125 -0
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,19 @@
|
|
|
1
1
|
# start-command
|
|
2
2
|
|
|
3
|
+
## 0.30.2
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- Apply the CI test runner's 30 second timeout consistently so slower integration-style tests do not fail at Bun's default 5 second limit.
|
|
8
|
+
|
|
9
|
+
Surface detached Docker OOMKilled status and preserve abnormal containers under the default cleanup policy.
|
|
10
|
+
|
|
11
|
+
## 0.30.1
|
|
12
|
+
|
|
13
|
+
### Patch Changes
|
|
14
|
+
|
|
15
|
+
- 1a040d6: Use `docker stop` for detached Docker `--stop` control so containers follow Docker's graceful stop lifecycle instead of receiving a raw `SIGINT`.
|
|
16
|
+
|
|
3
17
|
## 0.30.0
|
|
4
18
|
|
|
5
19
|
### Minor Changes
|
package/package.json
CHANGED
package/src/lib/args-parser.js
CHANGED
|
@@ -20,9 +20,9 @@
|
|
|
20
20
|
* --keep-user Keep isolated user after command completes (don't delete)
|
|
21
21
|
* --keep-alive, -k Keep isolation environment alive after command exits
|
|
22
22
|
* --auto-remove-docker-container Always remove docker container after exit (compatibility alias)
|
|
23
|
-
* --always-cleanup-container Always remove docker container after exit
|
|
23
|
+
* --always-cleanup-container Always remove docker container after exit
|
|
24
24
|
* --keep-container Keep docker container filesystem after exit
|
|
25
|
-
* --keep-container-on-fail Remove successful docker containers, keep failed ones
|
|
25
|
+
* --keep-container-on-fail Remove successful docker containers, keep failed or OOM-killed ones
|
|
26
26
|
* --shell <shell> Shell to use in isolation environments: auto, bash, zsh, sh (default: auto)
|
|
27
27
|
* --use-command-stream Use command-stream library for command execution (experimental)
|
|
28
28
|
* --verbose Enable verbose/debug output (sets START_VERBOSE=1)
|
|
@@ -30,7 +30,7 @@
|
|
|
30
30
|
* --list List all tracked command executions
|
|
31
31
|
* --upload-log <uuid-or-session> Upload the stored log for a tracked execution
|
|
32
32
|
* --output-format <format> Output format for status/list (links-notation, json, text)
|
|
33
|
-
* --stop <uuid-or-session-name>
|
|
33
|
+
* --stop <uuid-or-session-name> Ask a detached execution to stop gracefully
|
|
34
34
|
* --terminate <uuid-or-session-name> Terminate a detached execution immediately
|
|
35
35
|
* --cleanup Clean up stale "executing" records (processes that crashed or were killed)
|
|
36
36
|
* --cleanup-dry-run Show stale records that would be cleaned up (without actually cleaning)
|
|
@@ -185,9 +185,9 @@ function parseArgs(args) {
|
|
|
185
185
|
keepUser: false, // Keep isolated user after command completes (don't delete)
|
|
186
186
|
keepAlive: false, // Keep environment alive after command exits
|
|
187
187
|
autoRemoveDockerContainer: false, // Always remove docker container after exit (compatibility alias)
|
|
188
|
-
alwaysCleanupContainer: false, //
|
|
188
|
+
alwaysCleanupContainer: false, // Force docker container cleanup after exit
|
|
189
189
|
keepContainer: false, // Keep docker container filesystem after exit
|
|
190
|
-
keepContainerOnFail: false, // Keep docker container filesystem
|
|
190
|
+
keepContainerOnFail: false, // Keep docker container filesystem when command fails or OOM-kills
|
|
191
191
|
shell: 'auto', // Shell to use in isolation environments: auto, bash, zsh, sh
|
|
192
192
|
useCommandStream: false, // Use command-stream library for command execution
|
|
193
193
|
status: null, // UUID to show status for
|
|
@@ -6,11 +6,23 @@ const {
|
|
|
6
6
|
} = require('./isolation-log-utils');
|
|
7
7
|
|
|
8
8
|
const DOCKER_CONTAINER_CLEANUP_POLICY = {
|
|
9
|
+
DEFAULT: 'default',
|
|
9
10
|
ALWAYS: 'always',
|
|
10
11
|
KEEP: 'keep',
|
|
11
12
|
KEEP_ON_FAIL: 'keep-on-fail',
|
|
12
13
|
};
|
|
13
14
|
|
|
15
|
+
function getDockerCommand() {
|
|
16
|
+
return process.env.START_DOCKER_BIN || 'docker';
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
function getDockerSpawnOptions(options = {}) {
|
|
20
|
+
if (process.platform === 'win32' && process.env.START_DOCKER_BIN) {
|
|
21
|
+
return { ...options, shell: true };
|
|
22
|
+
}
|
|
23
|
+
return options;
|
|
24
|
+
}
|
|
25
|
+
|
|
14
26
|
function getDockerContainerCleanupPolicy(options = {}) {
|
|
15
27
|
if (options.keepContainer) {
|
|
16
28
|
return DOCKER_CONTAINER_CLEANUP_POLICY.KEEP;
|
|
@@ -18,15 +30,25 @@ function getDockerContainerCleanupPolicy(options = {}) {
|
|
|
18
30
|
if (options.keepContainerOnFail) {
|
|
19
31
|
return DOCKER_CONTAINER_CLEANUP_POLICY.KEEP_ON_FAIL;
|
|
20
32
|
}
|
|
21
|
-
|
|
33
|
+
if (options.alwaysCleanupContainer || options.autoRemoveDockerContainer) {
|
|
34
|
+
return DOCKER_CONTAINER_CLEANUP_POLICY.ALWAYS;
|
|
35
|
+
}
|
|
36
|
+
return DOCKER_CONTAINER_CLEANUP_POLICY.DEFAULT;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function isAbnormalDockerExit(exitCode, oomKilled = false) {
|
|
40
|
+
return exitCode !== 0 || oomKilled === true;
|
|
22
41
|
}
|
|
23
42
|
|
|
24
|
-
function shouldCleanupDockerContainer(policy, exitCode) {
|
|
43
|
+
function shouldCleanupDockerContainer(policy, exitCode, oomKilled = false) {
|
|
25
44
|
if (policy === DOCKER_CONTAINER_CLEANUP_POLICY.ALWAYS) {
|
|
26
45
|
return true;
|
|
27
46
|
}
|
|
47
|
+
if (policy === DOCKER_CONTAINER_CLEANUP_POLICY.DEFAULT) {
|
|
48
|
+
return !isAbnormalDockerExit(exitCode, oomKilled);
|
|
49
|
+
}
|
|
28
50
|
if (policy === DOCKER_CONTAINER_CLEANUP_POLICY.KEEP_ON_FAIL) {
|
|
29
|
-
return exitCode
|
|
51
|
+
return !isAbnormalDockerExit(exitCode, oomKilled);
|
|
30
52
|
}
|
|
31
53
|
return false;
|
|
32
54
|
}
|
|
@@ -50,18 +72,53 @@ function appendDockerContainerCleanupPolicyMessage(
|
|
|
50
72
|
if (policy === DOCKER_CONTAINER_CLEANUP_POLICY.KEEP_ON_FAIL) {
|
|
51
73
|
return (
|
|
52
74
|
`${message}\nContainer will be removed after successful completion.` +
|
|
53
|
-
`\nContainer will be kept if the command fails.` +
|
|
75
|
+
`\nContainer will be kept if the command fails or Docker reports OOMKilled.` +
|
|
76
|
+
`\nRemove when done: docker rm -f ${containerName}`
|
|
77
|
+
);
|
|
78
|
+
}
|
|
79
|
+
if (policy === DOCKER_CONTAINER_CLEANUP_POLICY.DEFAULT) {
|
|
80
|
+
return (
|
|
81
|
+
`${message}\nContainer will be removed after successful completion.` +
|
|
82
|
+
`\nContainer will be kept if the command fails or Docker reports OOMKilled.` +
|
|
54
83
|
`\nRemove when done: docker rm -f ${containerName}`
|
|
55
84
|
);
|
|
56
85
|
}
|
|
57
86
|
return `${message}\nContainer will be removed after command completes.`;
|
|
58
87
|
}
|
|
59
88
|
|
|
89
|
+
function readDockerContainerOomKilled(containerName) {
|
|
90
|
+
const result = spawnSync(
|
|
91
|
+
getDockerCommand(),
|
|
92
|
+
['inspect', '-f', '{{.State.OOMKilled}}', containerName],
|
|
93
|
+
getDockerSpawnOptions({
|
|
94
|
+
encoding: 'utf8',
|
|
95
|
+
env: process.env,
|
|
96
|
+
stdio: ['pipe', 'pipe', 'pipe'],
|
|
97
|
+
})
|
|
98
|
+
);
|
|
99
|
+
if (result.error || result.status !== 0) {
|
|
100
|
+
return null;
|
|
101
|
+
}
|
|
102
|
+
const value = String(result.stdout || '').trim();
|
|
103
|
+
if (value === 'true') {
|
|
104
|
+
return true;
|
|
105
|
+
}
|
|
106
|
+
if (value === 'false') {
|
|
107
|
+
return false;
|
|
108
|
+
}
|
|
109
|
+
return null;
|
|
110
|
+
}
|
|
111
|
+
|
|
60
112
|
function removeDockerContainer(containerName, logPath = null) {
|
|
61
|
-
const result = spawnSync(
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
113
|
+
const result = spawnSync(
|
|
114
|
+
getDockerCommand(),
|
|
115
|
+
['rm', '-f', containerName],
|
|
116
|
+
getDockerSpawnOptions({
|
|
117
|
+
encoding: 'utf8',
|
|
118
|
+
env: process.env,
|
|
119
|
+
stdio: ['pipe', 'pipe', 'pipe'],
|
|
120
|
+
})
|
|
121
|
+
);
|
|
65
122
|
const output = `${result.stdout || ''}${result.stderr || ''}`;
|
|
66
123
|
if (logPath && output) {
|
|
67
124
|
appendLogFile(logPath, output.endsWith('\n') ? output : `${output}\n`);
|
|
@@ -69,6 +126,24 @@ function removeDockerContainer(containerName, logPath = null) {
|
|
|
69
126
|
return !result.error && result.status === 0;
|
|
70
127
|
}
|
|
71
128
|
|
|
129
|
+
function buildDockerKeptLogSnippet(containerName, quotedLogPath) {
|
|
130
|
+
const quotedName = shellQuote(containerName);
|
|
131
|
+
return (
|
|
132
|
+
`printf '\\nContainer kept for investigation: %s\\nReason: exitCode=%s oomKilled=%s\\n` +
|
|
133
|
+
`Inspect: docker exec -it %s sh (if running) or docker start -ai %s\\n` +
|
|
134
|
+
`Remove when done: docker rm -f %s\\n' ` +
|
|
135
|
+
`${quotedName} "$__start_command_exit" "$__start_command_oom" ` +
|
|
136
|
+
`${quotedName} ${quotedName} ${quotedName} >> ${quotedLogPath}`
|
|
137
|
+
);
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
function buildSuccessfulNonOomCondition() {
|
|
141
|
+
return (
|
|
142
|
+
'[ "$__start_command_exit" -eq 0 ] 2>/dev/null && ' +
|
|
143
|
+
'[ "$__start_command_oom" != true ]'
|
|
144
|
+
);
|
|
145
|
+
}
|
|
146
|
+
|
|
72
147
|
function buildDetachedDockerCompletionScript(containerName, policy, logPath) {
|
|
73
148
|
const quotedName = shellQuote(containerName);
|
|
74
149
|
const parts = [];
|
|
@@ -77,26 +152,37 @@ function buildDetachedDockerCompletionScript(containerName, policy, logPath) {
|
|
|
77
152
|
const quotedLogPath = shellQuote(logPath);
|
|
78
153
|
parts.push(`docker logs -f ${quotedName} >> ${quotedLogPath} 2>&1`);
|
|
79
154
|
parts.push(
|
|
80
|
-
`
|
|
155
|
+
`__start_command_state=$(docker inspect -f '{{.State.ExitCode}} {{.State.OOMKilled}}' ${quotedName} 2>/dev/null || printf '%s' '-1 false')`
|
|
81
156
|
);
|
|
157
|
+
parts.push('__start_command_exit=${__start_command_state%% *}');
|
|
158
|
+
parts.push('__start_command_oom=${__start_command_state##* }');
|
|
82
159
|
if (policy === DOCKER_CONTAINER_CLEANUP_POLICY.ALWAYS) {
|
|
83
160
|
parts.push(`docker rm -f ${quotedName} >> ${quotedLogPath} 2>&1 || true`);
|
|
84
|
-
} else if (
|
|
161
|
+
} else if (
|
|
162
|
+
policy === DOCKER_CONTAINER_CLEANUP_POLICY.DEFAULT ||
|
|
163
|
+
policy === DOCKER_CONTAINER_CLEANUP_POLICY.KEEP_ON_FAIL
|
|
164
|
+
) {
|
|
165
|
+
const successCondition = buildSuccessfulNonOomCondition();
|
|
85
166
|
parts.push(
|
|
86
|
-
`if
|
|
167
|
+
`if ${successCondition}; then docker rm -f ${quotedName} >> ${quotedLogPath} 2>&1 || true; else ${buildDockerKeptLogSnippet(containerName, quotedLogPath)}; fi`
|
|
87
168
|
);
|
|
88
169
|
}
|
|
89
170
|
parts.push(`${createShellLogFooterSnippet()} >> ${quotedLogPath}`);
|
|
90
171
|
} else {
|
|
91
172
|
parts.push(`docker wait ${quotedName} >/dev/null 2>&1`);
|
|
92
173
|
parts.push(
|
|
93
|
-
`
|
|
174
|
+
`__start_command_state=$(docker inspect -f '{{.State.ExitCode}} {{.State.OOMKilled}}' ${quotedName} 2>/dev/null || printf '%s' '-1 false')`
|
|
94
175
|
);
|
|
176
|
+
parts.push('__start_command_exit=${__start_command_state%% *}');
|
|
177
|
+
parts.push('__start_command_oom=${__start_command_state##* }');
|
|
95
178
|
if (policy === DOCKER_CONTAINER_CLEANUP_POLICY.ALWAYS) {
|
|
96
179
|
parts.push(`docker rm -f ${quotedName} >/dev/null 2>&1 || true`);
|
|
97
|
-
} else if (
|
|
180
|
+
} else if (
|
|
181
|
+
policy === DOCKER_CONTAINER_CLEANUP_POLICY.DEFAULT ||
|
|
182
|
+
policy === DOCKER_CONTAINER_CLEANUP_POLICY.KEEP_ON_FAIL
|
|
183
|
+
) {
|
|
98
184
|
parts.push(
|
|
99
|
-
`if
|
|
185
|
+
`if ${buildSuccessfulNonOomCondition()}; then docker rm -f ${quotedName} >/dev/null 2>&1 || true; fi`
|
|
100
186
|
);
|
|
101
187
|
}
|
|
102
188
|
}
|
|
@@ -118,12 +204,20 @@ function startDetachedDockerCompletionWatcher(containerName, policy, logPath) {
|
|
|
118
204
|
|
|
119
205
|
function spawnAttachedDocker(dockerArgs, logPath) {
|
|
120
206
|
if (!logPath) {
|
|
121
|
-
return spawn(
|
|
207
|
+
return spawn(
|
|
208
|
+
getDockerCommand(),
|
|
209
|
+
dockerArgs,
|
|
210
|
+
getDockerSpawnOptions({ stdio: 'inherit' })
|
|
211
|
+
);
|
|
122
212
|
}
|
|
123
213
|
|
|
124
|
-
const child = spawn(
|
|
125
|
-
|
|
126
|
-
|
|
214
|
+
const child = spawn(
|
|
215
|
+
getDockerCommand(),
|
|
216
|
+
dockerArgs,
|
|
217
|
+
getDockerSpawnOptions({
|
|
218
|
+
stdio: ['inherit', 'pipe', 'pipe'],
|
|
219
|
+
})
|
|
220
|
+
);
|
|
127
221
|
const tee = (chunk, stream) => {
|
|
128
222
|
stream.write(chunk);
|
|
129
223
|
appendLogFile(logPath, chunk.toString());
|
|
@@ -135,11 +229,16 @@ function spawnAttachedDocker(dockerArgs, logPath) {
|
|
|
135
229
|
|
|
136
230
|
module.exports = {
|
|
137
231
|
DOCKER_CONTAINER_CLEANUP_POLICY,
|
|
232
|
+
getDockerCommand,
|
|
233
|
+
getDockerSpawnOptions,
|
|
138
234
|
getDockerContainerCleanupPolicy,
|
|
235
|
+
isAbnormalDockerExit,
|
|
139
236
|
shouldCleanupDockerContainer,
|
|
140
237
|
getDockerContainerCleanupInstructions,
|
|
141
238
|
appendDockerContainerCleanupPolicyMessage,
|
|
239
|
+
readDockerContainerOomKilled,
|
|
142
240
|
removeDockerContainer,
|
|
241
|
+
buildDetachedDockerCompletionScript,
|
|
143
242
|
startDetachedDockerCompletionWatcher,
|
|
144
243
|
spawnAttachedDocker,
|
|
145
244
|
};
|
|
@@ -10,6 +10,7 @@ const {
|
|
|
10
10
|
escapeForLinksNotation,
|
|
11
11
|
formatAsNestedLinksNotation,
|
|
12
12
|
} = require('./output-blocks');
|
|
13
|
+
const { getDockerCommand, getDockerSpawnOptions } = require('./docker-cleanup');
|
|
13
14
|
|
|
14
15
|
const ControlAction = {
|
|
15
16
|
STOP: 'stop',
|
|
@@ -17,10 +18,15 @@ const ControlAction = {
|
|
|
17
18
|
};
|
|
18
19
|
|
|
19
20
|
function runCommand(command, args) {
|
|
20
|
-
const
|
|
21
|
+
const options = {
|
|
21
22
|
encoding: 'utf8',
|
|
22
23
|
stdio: ['ignore', 'pipe', 'pipe'],
|
|
23
|
-
}
|
|
24
|
+
};
|
|
25
|
+
const result = spawnSync(
|
|
26
|
+
command,
|
|
27
|
+
args,
|
|
28
|
+
command === getDockerCommand() ? getDockerSpawnOptions(options) : options
|
|
29
|
+
);
|
|
24
30
|
|
|
25
31
|
return {
|
|
26
32
|
success: !result.error && result.status === 0,
|
|
@@ -143,7 +149,7 @@ function collectProcessIds(record, runner = runCommand) {
|
|
|
143
149
|
addIfPresent(processIds, 'commandPids', commandPids);
|
|
144
150
|
} else if (isolated === 'docker') {
|
|
145
151
|
addIfPresent(processIds, 'containerId', opts.containerId);
|
|
146
|
-
const result = runner(
|
|
152
|
+
const result = runner(getDockerCommand(), [
|
|
147
153
|
'inspect',
|
|
148
154
|
'-f',
|
|
149
155
|
'{{.Id}} {{.State.Pid}}',
|
|
@@ -197,9 +203,9 @@ function getControlCommand(record, action) {
|
|
|
197
203
|
case 'docker':
|
|
198
204
|
return {
|
|
199
205
|
command: 'docker',
|
|
200
|
-
args: ['
|
|
201
|
-
method: '
|
|
202
|
-
message: `
|
|
206
|
+
args: ['stop', sessionName],
|
|
207
|
+
method: 'DOCKER_STOP',
|
|
208
|
+
message: `Requested graceful stop for detached docker container: ${sessionName}`,
|
|
203
209
|
};
|
|
204
210
|
default:
|
|
205
211
|
return {
|
|
@@ -14,6 +14,7 @@
|
|
|
14
14
|
* - logPath: Path to the log file
|
|
15
15
|
* - startTime: Timestamp when execution started
|
|
16
16
|
* - endTime: Timestamp when execution completed (null while executing)
|
|
17
|
+
* - oomKilled: Docker resource-exhaustion signal when available
|
|
17
18
|
* - options: Execution options (isolation mode, etc.)
|
|
18
19
|
*/
|
|
19
20
|
|
|
@@ -73,6 +74,8 @@ class ExecutionRecord {
|
|
|
73
74
|
this.logPath = options.logPath || '';
|
|
74
75
|
this.startTime = options.startTime || new Date().toISOString();
|
|
75
76
|
this.endTime = options.endTime || null;
|
|
77
|
+
this.oomKilled =
|
|
78
|
+
options.oomKilled !== undefined ? options.oomKilled : undefined;
|
|
76
79
|
this.workingDirectory = options.workingDirectory || process.cwd();
|
|
77
80
|
this.shell = options.shell || process.env.SHELL || '/bin/sh';
|
|
78
81
|
this.platform = options.platform || process.platform;
|
|
@@ -93,7 +96,7 @@ class ExecutionRecord {
|
|
|
93
96
|
* Convert to plain object for serialization
|
|
94
97
|
*/
|
|
95
98
|
toObject() {
|
|
96
|
-
|
|
99
|
+
const obj = {
|
|
97
100
|
uuid: this.uuid,
|
|
98
101
|
pid: this.pid,
|
|
99
102
|
status: this.status,
|
|
@@ -102,11 +105,17 @@ class ExecutionRecord {
|
|
|
102
105
|
logPath: this.logPath,
|
|
103
106
|
startTime: this.startTime,
|
|
104
107
|
endTime: this.endTime,
|
|
108
|
+
};
|
|
109
|
+
if (this.oomKilled !== undefined && this.oomKilled !== null) {
|
|
110
|
+
obj.oomKilled = this.oomKilled;
|
|
111
|
+
}
|
|
112
|
+
Object.assign(obj, {
|
|
105
113
|
workingDirectory: this.workingDirectory,
|
|
106
114
|
shell: this.shell,
|
|
107
115
|
platform: this.platform,
|
|
108
116
|
options: this.options,
|
|
109
|
-
};
|
|
117
|
+
});
|
|
118
|
+
return obj;
|
|
110
119
|
}
|
|
111
120
|
|
|
112
121
|
/**
|
package/src/lib/isolation.js
CHANGED
|
@@ -27,6 +27,7 @@ const {
|
|
|
27
27
|
shouldCleanupDockerContainer,
|
|
28
28
|
getDockerContainerCleanupInstructions,
|
|
29
29
|
appendDockerContainerCleanupPolicyMessage,
|
|
30
|
+
readDockerContainerOomKilled,
|
|
30
31
|
removeDockerContainer,
|
|
31
32
|
startDetachedDockerCompletionWatcher,
|
|
32
33
|
spawnAttachedDocker,
|
|
@@ -758,7 +759,11 @@ function runInDocker(command, options = {}) {
|
|
|
758
759
|
message += `\n${hint}`;
|
|
759
760
|
}
|
|
760
761
|
|
|
761
|
-
|
|
762
|
+
const oomKilled = readDockerContainerOomKilled(containerName);
|
|
763
|
+
|
|
764
|
+
if (
|
|
765
|
+
shouldCleanupDockerContainer(cleanupPolicy, exitCode, oomKilled)
|
|
766
|
+
) {
|
|
762
767
|
if (removeDockerContainer(containerName, options.logPath)) {
|
|
763
768
|
message += `\nContainer removed after completion.`;
|
|
764
769
|
} else {
|
|
@@ -768,9 +773,13 @@ function runInDocker(command, options = {}) {
|
|
|
768
773
|
} else if (cleanupPolicy === DOCKER_CONTAINER_CLEANUP_POLICY.KEEP) {
|
|
769
774
|
message += `\n${getDockerContainerCleanupInstructions(containerName)}`;
|
|
770
775
|
} else if (
|
|
771
|
-
cleanupPolicy === DOCKER_CONTAINER_CLEANUP_POLICY.KEEP_ON_FAIL
|
|
776
|
+
cleanupPolicy === DOCKER_CONTAINER_CLEANUP_POLICY.KEEP_ON_FAIL ||
|
|
777
|
+
cleanupPolicy === DOCKER_CONTAINER_CLEANUP_POLICY.DEFAULT
|
|
772
778
|
) {
|
|
773
|
-
message +=
|
|
779
|
+
message +=
|
|
780
|
+
oomKilled === true
|
|
781
|
+
? `\nContainer kept because Docker reports it was OOM-killed.`
|
|
782
|
+
: `\nContainer kept because the command failed.`;
|
|
774
783
|
message += `\nRemove when done: docker rm -f ${containerName}`;
|
|
775
784
|
}
|
|
776
785
|
|
|
@@ -14,6 +14,7 @@ const {
|
|
|
14
14
|
formatAsNestedLinksNotation,
|
|
15
15
|
} = require('./output-blocks');
|
|
16
16
|
const { collectProcessIds } = require('./execution-control');
|
|
17
|
+
const { getDockerCommand, getDockerSpawnOptions } = require('./docker-cleanup');
|
|
17
18
|
|
|
18
19
|
/**
|
|
19
20
|
* Inspect the live state of a detached docker container by name.
|
|
@@ -30,23 +31,34 @@ const { collectProcessIds } = require('./execution-control');
|
|
|
30
31
|
* instead of fabricating a terminal `-1` result.
|
|
31
32
|
*
|
|
32
33
|
* @param {string} sessionName - Container name
|
|
33
|
-
* @returns {{running: boolean, exitCode: number|null}|null} State,
|
|
34
|
-
* the container cannot be inspected (not found yet, removed, or docker error)
|
|
34
|
+
* @returns {{running: boolean, exitCode: number|null, oomKilled: boolean|null}|null} State,
|
|
35
|
+
* or null when the container cannot be inspected (not found yet, removed, or docker error)
|
|
35
36
|
*/
|
|
36
37
|
function inspectDockerState(sessionName) {
|
|
37
38
|
const result = spawnSync(
|
|
38
|
-
|
|
39
|
-
[
|
|
40
|
-
|
|
39
|
+
getDockerCommand(),
|
|
40
|
+
[
|
|
41
|
+
'inspect',
|
|
42
|
+
'-f',
|
|
43
|
+
'{{.State.Running}} {{.State.ExitCode}} {{.State.OOMKilled}}',
|
|
44
|
+
sessionName,
|
|
45
|
+
],
|
|
46
|
+
getDockerSpawnOptions({
|
|
47
|
+
encoding: 'utf8',
|
|
48
|
+
env: process.env,
|
|
49
|
+
stdio: ['pipe', 'pipe', 'pipe'],
|
|
50
|
+
})
|
|
41
51
|
);
|
|
42
52
|
if (result.error || result.status !== 0 || !result.stdout) {
|
|
43
53
|
return null;
|
|
44
54
|
}
|
|
45
|
-
const [runningRaw, exitRaw] = result.stdout.trim().split(/\s+/);
|
|
55
|
+
const [runningRaw, exitRaw, oomKilledRaw] = result.stdout.trim().split(/\s+/);
|
|
46
56
|
const exitCode = Number.parseInt(exitRaw, 10);
|
|
47
57
|
return {
|
|
48
58
|
running: runningRaw === 'true',
|
|
49
59
|
exitCode: Number.isFinite(exitCode) ? exitCode : null,
|
|
60
|
+
oomKilled:
|
|
61
|
+
oomKilledRaw === 'true' ? true : oomKilledRaw === 'false' ? false : null,
|
|
50
62
|
};
|
|
51
63
|
}
|
|
52
64
|
|
|
@@ -67,6 +79,15 @@ function readBackendExitCode(record) {
|
|
|
67
79
|
return state && !state.running ? state.exitCode : null;
|
|
68
80
|
}
|
|
69
81
|
|
|
82
|
+
function readDockerOomKilled(record) {
|
|
83
|
+
const opts = record.options || {};
|
|
84
|
+
if (opts.isolated !== 'docker' || !opts.sessionName) {
|
|
85
|
+
return null;
|
|
86
|
+
}
|
|
87
|
+
const state = inspectDockerState(opts.sessionName);
|
|
88
|
+
return state ? state.oomKilled : null;
|
|
89
|
+
}
|
|
90
|
+
|
|
70
91
|
/**
|
|
71
92
|
* Check if a detached isolation session is still running
|
|
72
93
|
* @param {Object} record - Execution record
|
|
@@ -183,6 +204,10 @@ function enrichDetachedStatus(record) {
|
|
|
183
204
|
}
|
|
184
205
|
|
|
185
206
|
const enriched = cloneRecord();
|
|
207
|
+
const oomKilled = readDockerOomKilled(enriched);
|
|
208
|
+
if (oomKilled !== null) {
|
|
209
|
+
enriched.oomKilled = oomKilled;
|
|
210
|
+
}
|
|
186
211
|
|
|
187
212
|
if (alive && enriched.status === 'executed') {
|
|
188
213
|
// A live `screen -ls` (or `tmux`/`docker`) session does NOT mean the command
|
|
@@ -358,6 +383,9 @@ function formatRecordAsText(record) {
|
|
|
358
383
|
`Status: ${obj.status}`,
|
|
359
384
|
`Command: ${obj.command}`,
|
|
360
385
|
`Exit Code: ${obj.exitCode !== null ? obj.exitCode : 'N/A'}`,
|
|
386
|
+
...(obj.oomKilled !== undefined
|
|
387
|
+
? [`OOM Killed: ${obj.oomKilled}`]
|
|
388
|
+
: []),
|
|
361
389
|
`PID: ${obj.pid !== null ? obj.pid : 'N/A'}`,
|
|
362
390
|
`Working Directory: ${obj.workingDirectory}`,
|
|
363
391
|
`Shell: ${obj.shell}`,
|
package/src/lib/usage.js
CHANGED
|
@@ -19,15 +19,15 @@ Options:
|
|
|
19
19
|
--keep-user Keep isolated user after command completes
|
|
20
20
|
--keep-alive, -k Keep isolation environment alive after command exits
|
|
21
21
|
--auto-remove-docker-container Always remove docker container after exit (compatibility alias)
|
|
22
|
-
--always-cleanup-container Always remove docker container after exit
|
|
22
|
+
--always-cleanup-container Always remove docker container after exit
|
|
23
23
|
--keep-container Keep docker container filesystem after exit
|
|
24
|
-
--keep-container-on-fail Remove successful docker containers, keep failed ones
|
|
24
|
+
--keep-container-on-fail Remove successful docker containers, keep failed or OOM-killed ones
|
|
25
25
|
--shell <shell> Shell to use in isolation environments: auto, bash, zsh, sh (default: auto)
|
|
26
26
|
--use-command-stream Use command-stream library for execution (experimental)
|
|
27
27
|
--status <id> Show status of execution by UUID or session name (--output-format: links-notation|json|text)
|
|
28
28
|
--list List all tracked executions (--output-format: links-notation|json|text)
|
|
29
29
|
--upload-log <id> Upload the stored log for an execution UUID or session name
|
|
30
|
-
--stop <id>
|
|
30
|
+
--stop <id> Ask a detached isolated execution to stop gracefully
|
|
31
31
|
--terminate <id> Terminate a detached isolated execution immediately
|
|
32
32
|
--cleanup Clean up stale "executing" records (crashed/killed processes)
|
|
33
33
|
--cleanup-dry-run Show stale records that would be cleaned up (without cleaning)
|
|
@@ -11,6 +11,12 @@ const path = require('path');
|
|
|
11
11
|
const { isCommandAvailable } = require('../src/lib/isolation');
|
|
12
12
|
const { runInDocker } = require('../src/lib/isolation');
|
|
13
13
|
const { execSync } = require('child_process');
|
|
14
|
+
const {
|
|
15
|
+
DOCKER_CONTAINER_CLEANUP_POLICY,
|
|
16
|
+
buildDetachedDockerCompletionScript,
|
|
17
|
+
getDockerContainerCleanupPolicy,
|
|
18
|
+
shouldCleanupDockerContainer,
|
|
19
|
+
} = require('../src/lib/docker-cleanup');
|
|
14
20
|
|
|
15
21
|
// Helper to wait for a condition with timeout
|
|
16
22
|
async function waitFor(conditionFn, timeout = 5000, interval = 100) {
|
|
@@ -33,6 +39,47 @@ describe('Docker Container Cleanup Policy', () => {
|
|
|
33
39
|
// These tests verify that docker isolation removes finished containers by
|
|
34
40
|
// default while still providing explicit flags to keep them for investigation.
|
|
35
41
|
|
|
42
|
+
describe('cleanup decisions', () => {
|
|
43
|
+
it('should keep abnormal containers under the default policy', () => {
|
|
44
|
+
const policy = getDockerContainerCleanupPolicy({});
|
|
45
|
+
assert.strictEqual(policy, DOCKER_CONTAINER_CLEANUP_POLICY.DEFAULT);
|
|
46
|
+
assert.strictEqual(shouldCleanupDockerContainer(policy, 0, false), true);
|
|
47
|
+
assert.strictEqual(shouldCleanupDockerContainer(policy, 7, false), false);
|
|
48
|
+
assert.strictEqual(shouldCleanupDockerContainer(policy, 0, true), false);
|
|
49
|
+
});
|
|
50
|
+
|
|
51
|
+
it('should keep OOM-killed containers with keepContainerOnFail', () => {
|
|
52
|
+
const policy = getDockerContainerCleanupPolicy({
|
|
53
|
+
keepContainerOnFail: true,
|
|
54
|
+
});
|
|
55
|
+
assert.strictEqual(policy, DOCKER_CONTAINER_CLEANUP_POLICY.KEEP_ON_FAIL);
|
|
56
|
+
assert.strictEqual(shouldCleanupDockerContainer(policy, 0, false), true);
|
|
57
|
+
assert.strictEqual(shouldCleanupDockerContainer(policy, 0, true), false);
|
|
58
|
+
});
|
|
59
|
+
|
|
60
|
+
it('should honor explicit always-cleanup policy', () => {
|
|
61
|
+
const policy = getDockerContainerCleanupPolicy({
|
|
62
|
+
alwaysCleanupContainer: true,
|
|
63
|
+
});
|
|
64
|
+
assert.strictEqual(policy, DOCKER_CONTAINER_CLEANUP_POLICY.ALWAYS);
|
|
65
|
+
assert.strictEqual(shouldCleanupDockerContainer(policy, 7, false), true);
|
|
66
|
+
assert.strictEqual(shouldCleanupDockerContainer(policy, 0, true), true);
|
|
67
|
+
});
|
|
68
|
+
|
|
69
|
+
it('should make the detached watcher inspect OOMKilled before default cleanup', () => {
|
|
70
|
+
const script = buildDetachedDockerCompletionScript(
|
|
71
|
+
'issue144-container',
|
|
72
|
+
DOCKER_CONTAINER_CLEANUP_POLICY.DEFAULT,
|
|
73
|
+
'/tmp/issue144.log'
|
|
74
|
+
);
|
|
75
|
+
assert.match(script, /\.State\.ExitCode.*\.State\.OOMKilled/);
|
|
76
|
+
assert.match(script, /__start_command_oom/);
|
|
77
|
+
assert.match(script, /Container kept for investigation/);
|
|
78
|
+
assert.match(script, /docker rm -f/);
|
|
79
|
+
assert.match(script, /issue144-container/);
|
|
80
|
+
});
|
|
81
|
+
});
|
|
82
|
+
|
|
36
83
|
describe('auto-remove enabled', () => {
|
|
37
84
|
it(
|
|
38
85
|
'should automatically remove container when autoRemoveDockerContainer is true',
|
|
@@ -136,6 +136,46 @@ describe('execution control', () => {
|
|
|
136
136
|
});
|
|
137
137
|
});
|
|
138
138
|
|
|
139
|
+
it('should stop a detached docker container with docker stop', () => {
|
|
140
|
+
const store = createStore();
|
|
141
|
+
store.save(
|
|
142
|
+
createDetachedRecord({
|
|
143
|
+
uuid: 'docker-control-uuid',
|
|
144
|
+
options: {
|
|
145
|
+
isolated: 'docker',
|
|
146
|
+
isolationMode: 'detached',
|
|
147
|
+
sessionName: 'docker-session',
|
|
148
|
+
containerId: 'abc123',
|
|
149
|
+
},
|
|
150
|
+
})
|
|
151
|
+
);
|
|
152
|
+
const runner = createRunner({
|
|
153
|
+
'docker inspect -f {{.Id}} {{.State.Pid}} docker-session': {
|
|
154
|
+
success: true,
|
|
155
|
+
stdout: 'abcdef 0\n',
|
|
156
|
+
stderr: '',
|
|
157
|
+
status: 0,
|
|
158
|
+
error: null,
|
|
159
|
+
},
|
|
160
|
+
});
|
|
161
|
+
|
|
162
|
+
const result = controlExecution(
|
|
163
|
+
store,
|
|
164
|
+
'docker-control-uuid',
|
|
165
|
+
ControlAction.STOP,
|
|
166
|
+
runner
|
|
167
|
+
);
|
|
168
|
+
|
|
169
|
+
assert.strictEqual(result.success, true);
|
|
170
|
+
assert.match(result.output, /action stop/);
|
|
171
|
+
assert.match(result.output, /method DOCKER_STOP/);
|
|
172
|
+
assert.match(result.output, /containerId abcdef/);
|
|
173
|
+
assert.deepStrictEqual(runner.calls[0], {
|
|
174
|
+
command: 'docker',
|
|
175
|
+
args: ['stop', 'docker-session'],
|
|
176
|
+
});
|
|
177
|
+
});
|
|
178
|
+
|
|
139
179
|
it('should send docker terminate through docker kill', () => {
|
|
140
180
|
const store = createStore();
|
|
141
181
|
store.save(
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import { describe, expect, test } from 'bun:test';
|
|
2
|
+
import {
|
|
3
|
+
applyDefaultTimeout,
|
|
4
|
+
hasExplicitTimeout,
|
|
5
|
+
} from '../../scripts/run-js-tests.mjs';
|
|
6
|
+
|
|
7
|
+
describe('run-js-tests timeout defaults', () => {
|
|
8
|
+
test('adds a 30 second timeout when no timeout is supplied', () => {
|
|
9
|
+
expect(applyDefaultTimeout(['--coverage'])).toEqual([
|
|
10
|
+
'--timeout',
|
|
11
|
+
'30000',
|
|
12
|
+
'--coverage',
|
|
13
|
+
]);
|
|
14
|
+
});
|
|
15
|
+
|
|
16
|
+
test('preserves an explicit long timeout flag', () => {
|
|
17
|
+
expect(hasExplicitTimeout(['--timeout', '45000'])).toBe(true);
|
|
18
|
+
expect(applyDefaultTimeout(['--timeout', '45000'])).toEqual([
|
|
19
|
+
'--timeout',
|
|
20
|
+
'45000',
|
|
21
|
+
]);
|
|
22
|
+
});
|
|
23
|
+
|
|
24
|
+
test('preserves an explicit inline timeout flag', () => {
|
|
25
|
+
expect(hasExplicitTimeout(['--timeout=45000'])).toBe(true);
|
|
26
|
+
expect(applyDefaultTimeout(['--timeout=45000'])).toEqual([
|
|
27
|
+
'--timeout=45000',
|
|
28
|
+
]);
|
|
29
|
+
});
|
|
30
|
+
|
|
31
|
+
test('preserves Bun short timeout flag', () => {
|
|
32
|
+
expect(hasExplicitTimeout(['-t', '45000'])).toBe(true);
|
|
33
|
+
expect(applyDefaultTimeout(['-t', '45000'])).toEqual(['-t', '45000']);
|
|
34
|
+
});
|
|
35
|
+
});
|
|
@@ -16,6 +16,7 @@ const {
|
|
|
16
16
|
} = require('../src/lib/execution-store');
|
|
17
17
|
const {
|
|
18
18
|
queryStatus,
|
|
19
|
+
listExecutions,
|
|
19
20
|
isDetachedSessionAlive,
|
|
20
21
|
enrichDetachedStatus,
|
|
21
22
|
attachCurrentTime,
|
|
@@ -55,6 +56,65 @@ function runCli(args, env = {}) {
|
|
|
55
56
|
};
|
|
56
57
|
}
|
|
57
58
|
|
|
59
|
+
function createExecutable(filePath, content) {
|
|
60
|
+
fs.writeFileSync(filePath, content, 'utf8');
|
|
61
|
+
fs.chmodSync(filePath, 0o755);
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function withFakeDockerInspect(stateLine, fn) {
|
|
65
|
+
const fakeBin = fs.mkdtempSync(path.join(os.tmpdir(), 'fake-docker-'));
|
|
66
|
+
const dockerPath = path.join(
|
|
67
|
+
fakeBin,
|
|
68
|
+
process.platform === 'win32' ? 'docker.cmd' : 'docker'
|
|
69
|
+
);
|
|
70
|
+
if (process.platform === 'win32') {
|
|
71
|
+
createExecutable(
|
|
72
|
+
dockerPath,
|
|
73
|
+
[
|
|
74
|
+
'@echo off',
|
|
75
|
+
'if not "%1"=="inspect" exit /b 1',
|
|
76
|
+
'echo %3 | findstr /C:"State.Pid" >nul',
|
|
77
|
+
'if %errorlevel%==0 (',
|
|
78
|
+
' echo fake-container-id 4321',
|
|
79
|
+
' exit /b 0',
|
|
80
|
+
')',
|
|
81
|
+
`echo ${stateLine}`,
|
|
82
|
+
'exit /b 0',
|
|
83
|
+
'',
|
|
84
|
+
].join('\r\n')
|
|
85
|
+
);
|
|
86
|
+
} else {
|
|
87
|
+
createExecutable(
|
|
88
|
+
dockerPath,
|
|
89
|
+
[
|
|
90
|
+
'#!/bin/sh',
|
|
91
|
+
'[ "$1" = "inspect" ] || exit 1',
|
|
92
|
+
'case "$3" in',
|
|
93
|
+
' *State.Pid*) echo "fake-container-id 4321" ;;',
|
|
94
|
+
` *) echo "${stateLine}" ;;`,
|
|
95
|
+
'esac',
|
|
96
|
+
'',
|
|
97
|
+
].join('\n')
|
|
98
|
+
);
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
const originalPath = process.env.PATH;
|
|
102
|
+
const originalDockerBin = process.env.START_DOCKER_BIN;
|
|
103
|
+
process.env.PATH = `${fakeBin}${path.delimiter}${originalPath || ''}`;
|
|
104
|
+
process.env.START_DOCKER_BIN = dockerPath;
|
|
105
|
+
try {
|
|
106
|
+
return fn();
|
|
107
|
+
} finally {
|
|
108
|
+
process.env.PATH = originalPath;
|
|
109
|
+
if (originalDockerBin === undefined) {
|
|
110
|
+
delete process.env.START_DOCKER_BIN;
|
|
111
|
+
} else {
|
|
112
|
+
process.env.START_DOCKER_BIN = originalDockerBin;
|
|
113
|
+
}
|
|
114
|
+
fs.rmSync(fakeBin, { recursive: true, force: true });
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
|
|
58
118
|
describe('Issue #101: --status with session name lookup', () => {
|
|
59
119
|
let store;
|
|
60
120
|
|
|
@@ -552,6 +612,71 @@ describe('Issue #136: detached docker session liveness', () => {
|
|
|
552
612
|
});
|
|
553
613
|
});
|
|
554
614
|
|
|
615
|
+
describe('Issue #144: detached docker OOMKilled status signal', () => {
|
|
616
|
+
let store;
|
|
617
|
+
|
|
618
|
+
beforeEach(() => {
|
|
619
|
+
cleanupTestDir();
|
|
620
|
+
store = new ExecutionStore({
|
|
621
|
+
appFolder: TEST_APP_FOLDER,
|
|
622
|
+
useLinks: false,
|
|
623
|
+
});
|
|
624
|
+
});
|
|
625
|
+
|
|
626
|
+
afterEach(() => {
|
|
627
|
+
cleanupTestDir();
|
|
628
|
+
});
|
|
629
|
+
|
|
630
|
+
function saveDockerRecord() {
|
|
631
|
+
const record = new ExecutionRecord({
|
|
632
|
+
command: 'sh -c "exit 0"',
|
|
633
|
+
logPath: '/tmp/issue-144.log',
|
|
634
|
+
options: {
|
|
635
|
+
sessionName: 'issue144-oom',
|
|
636
|
+
isolated: 'docker',
|
|
637
|
+
isolationMode: 'detached',
|
|
638
|
+
},
|
|
639
|
+
});
|
|
640
|
+
store.save(record);
|
|
641
|
+
return record;
|
|
642
|
+
}
|
|
643
|
+
|
|
644
|
+
it('surfaces oomKilled in status output even when Docker exit code is 0', () => {
|
|
645
|
+
const record = saveDockerRecord();
|
|
646
|
+
|
|
647
|
+
withFakeDockerInspect('false 0 true', () => {
|
|
648
|
+
const jsonResult = queryStatus(store, record.uuid, 'json');
|
|
649
|
+
expect(jsonResult.success).toBe(true);
|
|
650
|
+
const parsed = JSON.parse(jsonResult.output);
|
|
651
|
+
expect(parsed.status).toBe('executed');
|
|
652
|
+
expect(parsed.exitCode).toBe(0);
|
|
653
|
+
expect(parsed.oomKilled).toBe(true);
|
|
654
|
+
|
|
655
|
+
const linksResult = queryStatus(store, record.uuid, 'links-notation');
|
|
656
|
+
expect(linksResult.success).toBe(true);
|
|
657
|
+
expect(linksResult.output).toContain(' oomKilled true');
|
|
658
|
+
|
|
659
|
+
const textResult = queryStatus(store, record.uuid, 'text');
|
|
660
|
+
expect(textResult.success).toBe(true);
|
|
661
|
+
expect(textResult.output).toContain('OOM Killed: true');
|
|
662
|
+
});
|
|
663
|
+
});
|
|
664
|
+
|
|
665
|
+
it('surfaces oomKilled in list output', () => {
|
|
666
|
+
saveDockerRecord();
|
|
667
|
+
|
|
668
|
+
withFakeDockerInspect('false 0 true', () => {
|
|
669
|
+
const result = listExecutions(store, 'json');
|
|
670
|
+
expect(result.success).toBe(true);
|
|
671
|
+
const parsed = JSON.parse(result.output);
|
|
672
|
+
expect(parsed.count).toBe(1);
|
|
673
|
+
expect(parsed.executions[0].status).toBe('executed');
|
|
674
|
+
expect(parsed.executions[0].exitCode).toBe(0);
|
|
675
|
+
expect(parsed.executions[0].oomKilled).toBe(true);
|
|
676
|
+
});
|
|
677
|
+
});
|
|
678
|
+
});
|
|
679
|
+
|
|
555
680
|
describe('Issue #105: attachCurrentTime for executing status', () => {
|
|
556
681
|
it('should add currentTime to serialization when status is executing', () => {
|
|
557
682
|
const record = new ExecutionRecord({
|