start-command 0.36.0 → 0.37.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +14 -0
- package/README.md +6 -0
- package/package.json +1 -1
- package/src/bin/cli.js +5 -0
- package/src/lib/args-parser-queries.js +14 -0
- package/src/lib/args-parser.js +20 -1
- package/src/lib/command-builder.js +3 -0
- package/src/lib/cpu-penalty-monitor.js +209 -87
- package/src/lib/cpu-penalty.js +30 -1
- package/src/lib/docker-cleanup.js +12 -5
- package/src/lib/docker-command-handoff.js +75 -0
- package/src/lib/docker-labels.js +33 -0
- package/src/lib/docker-runtime-args.js +8 -0
- package/src/lib/docker-snapshot-safety.js +164 -0
- package/src/lib/execution-recovery.js +8 -1
- package/src/lib/execution-resume.js +184 -37
- package/src/lib/failure-handler.js +15 -5
- package/src/lib/isolation.js +20 -4
- package/src/lib/log-sanitizer.js +267 -0
- package/src/lib/log-uploader.js +38 -19
- package/src/lib/query-commands.js +4 -3
- package/src/lib/resume-resources.js +15 -3
- package/src/lib/shell-utils.js +67 -17
- package/src/lib/store-lock.js +54 -3
- package/src/lib/usage.js +5 -1
- package/test/ci-workflow-invariants.js +15 -14
- package/test/docker-command-handoff.js +107 -0
- package/test/docker-runtime-options.js +4 -0
- package/test/docker-snapshot-safety.js +106 -0
- package/test/execution-resume.js +10 -8
- package/test/failure-handler.js +11 -0
- package/test/javascript-first-ci.js +216 -0
- package/test/log-sanitizer.js +306 -0
- package/test/regression-176.js +7 -0
- package/test/regression-187.js +9 -1
- package/test/regression-191.js +9 -1
- package/test/regression-193.js +11 -1
- package/test/regression-198.js +281 -0
- package/test/regression-199.js +79 -0
- package/test/regression-201.js +25 -0
- package/test/regression-202.js +92 -0
- package/test/regression-207.js +249 -0
- package/test/regression-91.js +23 -20
- package/test/shared-golden.js +77 -0
- package/test/status-query.js +97 -15
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
/** Copy only a command script into a stopped container, never its writable layer. */
|
|
2
|
+
const fs = require('fs');
|
|
3
|
+
const os = require('os');
|
|
4
|
+
const path = require('path');
|
|
5
|
+
const { shellQuote } = require('./isolation-log-utils');
|
|
6
|
+
|
|
7
|
+
function commandHandoffPath(sessionName) {
|
|
8
|
+
return `/.start-command-resume-${String(sessionName).replace(/[^A-Za-z0-9_.-]/g, '_')}`;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
function buildCommandHandoffArgs(mainArgs, markerPath) {
|
|
12
|
+
return [
|
|
13
|
+
'sh',
|
|
14
|
+
'-c',
|
|
15
|
+
'p=$1; shift; if [ -f "$p" ]; then exec sh "$p"; fi; exec "$@"',
|
|
16
|
+
'start-command-handoff',
|
|
17
|
+
markerPath,
|
|
18
|
+
...mainArgs,
|
|
19
|
+
];
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
function buildCommandHandoffScript(command, options = {}) {
|
|
23
|
+
const {
|
|
24
|
+
isInteractiveShellCommand,
|
|
25
|
+
isShellInvocationWithArgs,
|
|
26
|
+
buildShellWithArgsCmdArgs,
|
|
27
|
+
toShellWords,
|
|
28
|
+
} = require('./shell-utils');
|
|
29
|
+
if (
|
|
30
|
+
!options.keepAlive &&
|
|
31
|
+
(isInteractiveShellCommand(command) || isShellInvocationWithArgs(command))
|
|
32
|
+
) {
|
|
33
|
+
const args = isShellInvocationWithArgs(command)
|
|
34
|
+
? buildShellWithArgsCmdArgs(command)
|
|
35
|
+
: toShellWords(command);
|
|
36
|
+
return `exec ${args.map(shellQuote).join(' ')}\n`;
|
|
37
|
+
}
|
|
38
|
+
if (!options.shell || options.shell === 'auto') {
|
|
39
|
+
const effective = options.keepAlive
|
|
40
|
+
? `${command}; exec "$__START_COMMAND_HANDOFF_SHELL"`
|
|
41
|
+
: command;
|
|
42
|
+
return `__START_COMMAND_HANDOFF_SHELL=$(command -v bash || command -v zsh || command -v sh); export __START_COMMAND_HANDOFF_SHELL\ncase "$__START_COMMAND_HANDOFF_SHELL" in */bash|bash|*/zsh|zsh) exec "$__START_COMMAND_HANDOFF_SHELL" -i -c ${shellQuote(effective)};; *) exec "$__START_COMMAND_HANDOFF_SHELL" -c ${shellQuote(effective)};; esac\n`;
|
|
43
|
+
}
|
|
44
|
+
const shell = options.shell;
|
|
45
|
+
const interactive = /(?:^|\/)(?:bash|zsh)$/.test(shell) ? ' -i' : '';
|
|
46
|
+
const effective = options.keepAlive
|
|
47
|
+
? `${command}; exec ${shellQuote(shell)}`
|
|
48
|
+
: command;
|
|
49
|
+
return `exec ${shellQuote(shell)}${interactive} -c ${shellQuote(effective)}\n`;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
function writeCommandHandoff(containerName, command, runner, options = {}) {
|
|
53
|
+
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'start-handoff-'));
|
|
54
|
+
const file = path.join(dir, 'command');
|
|
55
|
+
try {
|
|
56
|
+
fs.writeFileSync(file, buildCommandHandoffScript(command, options), {
|
|
57
|
+
mode: 0o644,
|
|
58
|
+
});
|
|
59
|
+
fs.chmodSync(file, 0o644);
|
|
60
|
+
return runner(require('./docker-cleanup').getDockerCommand(), [
|
|
61
|
+
'cp',
|
|
62
|
+
file,
|
|
63
|
+
`${containerName}:${commandHandoffPath(containerName)}`,
|
|
64
|
+
]);
|
|
65
|
+
} finally {
|
|
66
|
+
fs.rmSync(dir, { recursive: true, force: true });
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
module.exports = {
|
|
71
|
+
commandHandoffPath,
|
|
72
|
+
buildCommandHandoffArgs,
|
|
73
|
+
buildCommandHandoffScript,
|
|
74
|
+
writeCommandHandoff,
|
|
75
|
+
};
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
/** Docker attribution shared by initial launches and snapshot resumes (#199). */
|
|
2
|
+
function validateLabel(label) {
|
|
3
|
+
if (typeof label !== 'string' || !/^[^=\s]+=[\s\S]*$/.test(label)) {
|
|
4
|
+
throw new Error('--label requires KEY=VALUE with a nonempty key');
|
|
5
|
+
}
|
|
6
|
+
if (label.startsWith('start-command.')) {
|
|
7
|
+
throw new Error(
|
|
8
|
+
'start-command.* labels are reserved for execution attribution'
|
|
9
|
+
);
|
|
10
|
+
}
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
function dockerLabels(options = {}) {
|
|
14
|
+
const labels = [...(options.labels || [])];
|
|
15
|
+
const session = options.containerName || options.session;
|
|
16
|
+
if (session) {
|
|
17
|
+
const inferredRoot =
|
|
18
|
+
Number(options.resumeCount) > 0
|
|
19
|
+
? session.replace(/(?:-resume-\d+)+$/, '')
|
|
20
|
+
: session;
|
|
21
|
+
labels.push(`start-command.session=${session}`);
|
|
22
|
+
labels.push(
|
|
23
|
+
`start-command.root-session=${options.rootSession || inferredRoot}`
|
|
24
|
+
);
|
|
25
|
+
labels.push(`start-command.resume-count=${options.resumeCount || 0}`);
|
|
26
|
+
}
|
|
27
|
+
if (options.sessionId || options.uuid) {
|
|
28
|
+
labels.push(`start-command.uuid=${options.sessionId || options.uuid}`);
|
|
29
|
+
}
|
|
30
|
+
return labels;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
module.exports = { validateLabel, dockerLabels };
|
|
@@ -42,6 +42,9 @@ function buildDockerRuntimeArgs(options = {}) {
|
|
|
42
42
|
for (const envVar of options.env || []) {
|
|
43
43
|
args.push('-e', envVar);
|
|
44
44
|
}
|
|
45
|
+
for (const label of require('./docker-labels').dockerLabels(options)) {
|
|
46
|
+
args.push('--label', label);
|
|
47
|
+
}
|
|
45
48
|
for (const volume of options.volumes || []) {
|
|
46
49
|
args.push('-v', volume);
|
|
47
50
|
}
|
|
@@ -69,6 +72,9 @@ function buildDockerRuntimeArgs(options = {}) {
|
|
|
69
72
|
*/
|
|
70
73
|
function buildDockerRuntimeStatusLines(options = {}) {
|
|
71
74
|
const lines = [];
|
|
75
|
+
if (options.labels && options.labels.length > 0) {
|
|
76
|
+
lines.push(`[Isolation] Labels: ${options.labels.join(', ')}`);
|
|
77
|
+
}
|
|
72
78
|
if (options.volumes && options.volumes.length > 0) {
|
|
73
79
|
lines.push(`[Isolation] Volumes: ${options.volumes.join(', ')}`);
|
|
74
80
|
}
|
|
@@ -129,6 +135,8 @@ function buildDockerRuntimeMetadata(options = {}) {
|
|
|
129
135
|
options.volumes && options.volumes.length > 0 ? options.volumes : null,
|
|
130
136
|
mounts: options.mounts && options.mounts.length > 0 ? options.mounts : null,
|
|
131
137
|
env: options.env && options.env.length > 0 ? options.env : null,
|
|
138
|
+
labels: options.labels && options.labels.length > 0 ? options.labels : null,
|
|
139
|
+
commandHandoff: options.detached ? true : null,
|
|
132
140
|
privileged: options.privileged || null,
|
|
133
141
|
network: networks[0] || null,
|
|
134
142
|
networks: networks.length > 0 ? networks : null,
|
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
/** Serialize costly snapshot resumes and check every image-store filesystem. */
|
|
2
|
+
const fs = require('fs');
|
|
3
|
+
const os = require('os');
|
|
4
|
+
const path = require('path');
|
|
5
|
+
const { LockManager } = require('./store-lock');
|
|
6
|
+
|
|
7
|
+
const GiB = 1024 ** 3;
|
|
8
|
+
const SNAPSHOT_RESERVE_BYTES = 10 * GiB;
|
|
9
|
+
const SNAPSHOT_IMAGE_LABEL = 'start-command.snapshot-image';
|
|
10
|
+
|
|
11
|
+
function acquireSnapshotLock(options = {}) {
|
|
12
|
+
const lockPath =
|
|
13
|
+
options.lockPath ||
|
|
14
|
+
process.env.START_SNAPSHOT_LOCK ||
|
|
15
|
+
path.join(
|
|
16
|
+
process.platform === 'win32' ? os.tmpdir() : '/tmp',
|
|
17
|
+
'start-command-docker-snapshot.lock'
|
|
18
|
+
);
|
|
19
|
+
const lock = new LockManager(lockPath);
|
|
20
|
+
// The ordinary store lock is held only for a brief write. A snapshot can
|
|
21
|
+
// legitimately take hours, so never reclaim this lock merely for its age.
|
|
22
|
+
lock.isLockStale = (data) => {
|
|
23
|
+
if (
|
|
24
|
+
!data ||
|
|
25
|
+
data.hostname !== os.hostname() ||
|
|
26
|
+
!Number.isInteger(data.pid) ||
|
|
27
|
+
data.pid <= 0
|
|
28
|
+
) {
|
|
29
|
+
return false;
|
|
30
|
+
}
|
|
31
|
+
try {
|
|
32
|
+
process.kill(data.pid, 0);
|
|
33
|
+
return false;
|
|
34
|
+
} catch (error) {
|
|
35
|
+
return error.code === 'ESRCH';
|
|
36
|
+
}
|
|
37
|
+
};
|
|
38
|
+
if (!lock.acquire(1000)) {
|
|
39
|
+
throw new Error(
|
|
40
|
+
'A Docker snapshot resume is in progress on this host. Retry after it finishes.'
|
|
41
|
+
);
|
|
42
|
+
}
|
|
43
|
+
return lock;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
function containerdRoot(config = '') {
|
|
47
|
+
// Only the root-level TOML key configures containerd's storage root.
|
|
48
|
+
const topLevel = config.split(/^\s*\[/m)[0];
|
|
49
|
+
return (
|
|
50
|
+
topLevel.match(/^\s*root\s*=\s*["']([^"']+)["']/m)?.[1] ||
|
|
51
|
+
'/var/lib/containerd'
|
|
52
|
+
);
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
function defaultContainerdRoot() {
|
|
56
|
+
if (process.env.START_CONTAINERD_ROOT) {
|
|
57
|
+
return process.env.START_CONTAINERD_ROOT;
|
|
58
|
+
}
|
|
59
|
+
try {
|
|
60
|
+
return containerdRoot(
|
|
61
|
+
fs.readFileSync('/etc/containerd/config.toml', 'utf8')
|
|
62
|
+
);
|
|
63
|
+
} catch (error) {
|
|
64
|
+
if (error.code === 'ENOENT') {
|
|
65
|
+
return containerdRoot();
|
|
66
|
+
}
|
|
67
|
+
throw error;
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
function readFreeBytes(root) {
|
|
72
|
+
const stats = fs.statfsSync(root, { bigint: true });
|
|
73
|
+
return Number(stats.bavail * stats.bsize);
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
function preflightSnapshot(containerName, runner, options = {}) {
|
|
77
|
+
const docker = require('./docker-cleanup').getDockerCommand();
|
|
78
|
+
const sizeResult = runner(docker, [
|
|
79
|
+
'inspect',
|
|
80
|
+
'--size',
|
|
81
|
+
'--format',
|
|
82
|
+
'{{.SizeRw}}',
|
|
83
|
+
containerName,
|
|
84
|
+
]);
|
|
85
|
+
const size = Number(String(sizeResult.stdout || '').trim());
|
|
86
|
+
if (
|
|
87
|
+
!sizeResult.success ||
|
|
88
|
+
!/^\d+$/.test(String(sizeResult.stdout || '').trim()) ||
|
|
89
|
+
!Number.isSafeInteger(size) ||
|
|
90
|
+
size < 0
|
|
91
|
+
) {
|
|
92
|
+
throw new Error(
|
|
93
|
+
'Cannot determine the writable layer size; refusing Docker snapshot resume.'
|
|
94
|
+
);
|
|
95
|
+
}
|
|
96
|
+
const infoResult = runner(docker, ['info', '--format', '{{json .}}']);
|
|
97
|
+
let info;
|
|
98
|
+
try {
|
|
99
|
+
info = JSON.parse(infoResult.stdout);
|
|
100
|
+
} catch {
|
|
101
|
+
/* Refuse unknown storage. */
|
|
102
|
+
}
|
|
103
|
+
if (!infoResult.success || !info?.DockerRootDir) {
|
|
104
|
+
throw new Error(
|
|
105
|
+
'Cannot determine Docker data root; refusing Docker snapshot resume.'
|
|
106
|
+
);
|
|
107
|
+
}
|
|
108
|
+
const roots = [info.DockerRootDir];
|
|
109
|
+
if (
|
|
110
|
+
JSON.stringify(info.DriverStatus || []).includes(
|
|
111
|
+
'io.containerd.snapshotter'
|
|
112
|
+
)
|
|
113
|
+
) {
|
|
114
|
+
roots.push(options.containerdRoot || defaultContainerdRoot());
|
|
115
|
+
}
|
|
116
|
+
const required = 2 * size + (options.reserveBytes ?? SNAPSHOT_RESERVE_BYTES);
|
|
117
|
+
const free = options.freeBytes || readFreeBytes;
|
|
118
|
+
for (const root of [...new Set(roots)]) {
|
|
119
|
+
let available;
|
|
120
|
+
try {
|
|
121
|
+
available = free(root);
|
|
122
|
+
} catch (error) {
|
|
123
|
+
throw new Error(
|
|
124
|
+
`Cannot check free disk at ${root}: ${error.message}; refusing Docker snapshot resume.`,
|
|
125
|
+
{ cause: error }
|
|
126
|
+
);
|
|
127
|
+
}
|
|
128
|
+
if (!Number.isFinite(available) || available < required) {
|
|
129
|
+
throw new Error(
|
|
130
|
+
`Insufficient disk at ${root}: need ${(required / GiB).toFixed(2)} GiB (2 × writable layer plus reserve), available ${(available / GiB).toFixed(2)} GiB. Docker snapshot resume refused.`
|
|
131
|
+
);
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
return {
|
|
135
|
+
size,
|
|
136
|
+
required,
|
|
137
|
+
roots,
|
|
138
|
+
message: `[Resume] snapshotting ${(size / GiB).toFixed(2)} GiB; disk preflight passed (${(required / GiB).toFixed(2)} GiB required).`,
|
|
139
|
+
};
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
function snapshotImageCleanupSnippet(
|
|
143
|
+
containerName,
|
|
144
|
+
redirection = '>/dev/null 2>&1'
|
|
145
|
+
) {
|
|
146
|
+
const { shellQuote } = require('./isolation-log-utils');
|
|
147
|
+
const name = shellQuote(containerName);
|
|
148
|
+
const format = shellQuote(
|
|
149
|
+
`{{index .Config.Labels "${SNAPSHOT_IMAGE_LABEL}"}}`
|
|
150
|
+
);
|
|
151
|
+
return {
|
|
152
|
+
capture: `__start_snapshot_image=$(docker inspect -f ${format} ${name} 2>/dev/null || true)`,
|
|
153
|
+
remove: `case "$__start_snapshot_image" in start-command-resume/*) docker rmi "$__start_snapshot_image" ${redirection} || true;; esac`,
|
|
154
|
+
};
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
module.exports = {
|
|
158
|
+
acquireSnapshotLock,
|
|
159
|
+
containerdRoot,
|
|
160
|
+
preflightSnapshot,
|
|
161
|
+
SNAPSHOT_RESERVE_BYTES,
|
|
162
|
+
SNAPSHOT_IMAGE_LABEL,
|
|
163
|
+
snapshotImageCleanupSnippet,
|
|
164
|
+
};
|
|
@@ -357,7 +357,14 @@ function recoverKilledExecution(params = {}) {
|
|
|
357
357
|
if (opts.recoveryCommand) {
|
|
358
358
|
let copied;
|
|
359
359
|
try {
|
|
360
|
-
copied =
|
|
360
|
+
copied = opts.commandHandoff
|
|
361
|
+
? require('./docker-command-handoff').writeCommandHandoff(
|
|
362
|
+
containerName,
|
|
363
|
+
`export ${RECOVERY_ATTEMPT_ENV}=${attempt}; ${opts.recoveryCommand}`,
|
|
364
|
+
runner,
|
|
365
|
+
opts
|
|
366
|
+
)
|
|
367
|
+
: writeRecoveryMarker(containerName, attempt, runner);
|
|
361
368
|
} catch (err) {
|
|
362
369
|
copied = { success: false, error: err.message };
|
|
363
370
|
}
|
|
@@ -7,13 +7,13 @@
|
|
|
7
7
|
* addressing one logical session across restarts.
|
|
8
8
|
*
|
|
9
9
|
* Three strategies, chosen from the probed session state:
|
|
10
|
-
* - DOCKER_START: the container still exists
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
16
|
-
*
|
|
10
|
+
* - DOCKER_START: the container still exists. Newly managed containers
|
|
11
|
+
* accept replacement commands through a small handoff
|
|
12
|
+
* script, then restart in place without a snapshot.
|
|
13
|
+
* - DOCKER_SNAPSHOT: a legacy container needs a replacement command, or
|
|
14
|
+
* immutable labels need changing. After disk preflight,
|
|
15
|
+
* its filesystem is committed under a host-wide lock and
|
|
16
|
+
* a derived container runs the command.
|
|
17
17
|
* - RELAUNCH: nothing is left of the session, so the command is
|
|
18
18
|
* launched again through the stored isolation options.
|
|
19
19
|
*/
|
|
@@ -44,6 +44,16 @@ const {
|
|
|
44
44
|
} = require('./launch-persistence');
|
|
45
45
|
const { SessionState, probeSession } = require('./session-probe');
|
|
46
46
|
const { getDockerNetworks } = require('./docker-network-lifecycle');
|
|
47
|
+
const {
|
|
48
|
+
writeCommandHandoff,
|
|
49
|
+
buildCommandHandoffArgs,
|
|
50
|
+
commandHandoffPath,
|
|
51
|
+
} = require('./docker-command-handoff');
|
|
52
|
+
const {
|
|
53
|
+
acquireSnapshotLock,
|
|
54
|
+
preflightSnapshot,
|
|
55
|
+
SNAPSHOT_IMAGE_LABEL,
|
|
56
|
+
} = require('./docker-snapshot-safety');
|
|
47
57
|
const {
|
|
48
58
|
ResumeAllAction,
|
|
49
59
|
resumeAllExecutions,
|
|
@@ -94,6 +104,11 @@ function buildLaunchOptions(record) {
|
|
|
94
104
|
const opts = record.options || {};
|
|
95
105
|
return {
|
|
96
106
|
...opts,
|
|
107
|
+
labels: opts.labels || [],
|
|
108
|
+
sessionId: record.uuid,
|
|
109
|
+
rootSession:
|
|
110
|
+
opts.rootSession || opts.sessionNameHistory?.[0] || opts.sessionName,
|
|
111
|
+
resumeCount: (Number(opts.resumeCount) || 0) + 1,
|
|
97
112
|
image: opts.image || null,
|
|
98
113
|
session: opts.sessionName,
|
|
99
114
|
detached: true,
|
|
@@ -214,12 +229,13 @@ function buildResumePlan(
|
|
|
214
229
|
const attempt = (Number(opts.resumeCount) || 0) + 1;
|
|
215
230
|
|
|
216
231
|
if (backend === 'docker' && probe.state === SessionState.STOPPED) {
|
|
217
|
-
if (!newCommand) {
|
|
232
|
+
if (!opts.forceSnapshot && (!newCommand || opts.commandHandoff)) {
|
|
218
233
|
return {
|
|
219
234
|
mode: ResumeMode.DOCKER_START,
|
|
220
235
|
backend,
|
|
221
236
|
sessionName,
|
|
222
237
|
command,
|
|
238
|
+
...(newCommand ? { handoffCommand: command } : {}),
|
|
223
239
|
attempt,
|
|
224
240
|
steps: [
|
|
225
241
|
{
|
|
@@ -228,7 +244,9 @@ function buildResumePlan(
|
|
|
228
244
|
description: `Start stopped container ${sessionName}`,
|
|
229
245
|
},
|
|
230
246
|
],
|
|
231
|
-
message:
|
|
247
|
+
message: newCommand
|
|
248
|
+
? `Resumed new command in the same detached docker container: ${sessionName} (no filesystem snapshot)`
|
|
249
|
+
: `Resumed detached docker container: ${sessionName}`,
|
|
232
250
|
};
|
|
233
251
|
}
|
|
234
252
|
|
|
@@ -261,12 +279,24 @@ function buildResumePlan(
|
|
|
261
279
|
containerArgs: [
|
|
262
280
|
'--name',
|
|
263
281
|
newSessionName,
|
|
282
|
+
'--label',
|
|
283
|
+
`${SNAPSHOT_IMAGE_LABEL}=${snapshotImage}`,
|
|
264
284
|
...(opts.user ? ['--user', opts.user] : []),
|
|
265
|
-
...buildDockerRuntimeArgs({
|
|
285
|
+
...buildDockerRuntimeArgs({
|
|
286
|
+
...opts,
|
|
287
|
+
resourceLimits,
|
|
288
|
+
session: newSessionName,
|
|
289
|
+
containerName: newSessionName,
|
|
290
|
+
sessionId: record.uuid,
|
|
291
|
+
rootSession:
|
|
292
|
+
opts.rootSession || opts.sessionNameHistory?.[0] || sessionName,
|
|
293
|
+
resumeCount: attempt,
|
|
294
|
+
}),
|
|
266
295
|
snapshotImage,
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
296
|
+
...buildCommandHandoffArgs(
|
|
297
|
+
['sh', '-c', command],
|
|
298
|
+
commandHandoffPath(newSessionName)
|
|
299
|
+
),
|
|
270
300
|
],
|
|
271
301
|
extraNetworks: getDockerNetworks(opts).slice(1),
|
|
272
302
|
newSessionName,
|
|
@@ -360,6 +390,10 @@ function applyResumeToRecord(
|
|
|
360
390
|
archiveAttempt(record);
|
|
361
391
|
record.attempt = attempt;
|
|
362
392
|
const options = { ...(record.options || {}) };
|
|
393
|
+
options.rootSession =
|
|
394
|
+
options.rootSession ||
|
|
395
|
+
options.sessionNameHistory?.[0] ||
|
|
396
|
+
options.sessionName;
|
|
363
397
|
options.resumeCount = plan.attempt;
|
|
364
398
|
options.resumedAt = attempt.startedAt;
|
|
365
399
|
// A resume is a new deliberate start: launch-time recovery applies again.
|
|
@@ -378,6 +412,12 @@ function applyResumeToRecord(
|
|
|
378
412
|
}
|
|
379
413
|
if (plan.snapshotImage) {
|
|
380
414
|
options.image = plan.snapshotImage;
|
|
415
|
+
options.snapshotImage = plan.snapshotImage;
|
|
416
|
+
// Every new detached container, including a snapshot successor, supports handoff.
|
|
417
|
+
options.commandHandoff = true;
|
|
418
|
+
}
|
|
419
|
+
if (plan.mode === ResumeMode.RELAUNCH && plan.backend === 'docker') {
|
|
420
|
+
options.commandHandoff = true;
|
|
381
421
|
}
|
|
382
422
|
if (containerId) {
|
|
383
423
|
options.containerId = containerId;
|
|
@@ -445,12 +485,34 @@ async function resumeExecution(store, identifier, deps = {}) {
|
|
|
445
485
|
} catch (error) {
|
|
446
486
|
return { success: false, error: error.message };
|
|
447
487
|
}
|
|
448
|
-
const
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
488
|
+
const requestedLabels = deps.resourceOptions?.labels || [];
|
|
489
|
+
const labels = [...(opts.labels || [])];
|
|
490
|
+
for (const label of requestedLabels) {
|
|
491
|
+
const key = label.split('=')[0];
|
|
492
|
+
const existing = labels.findIndex((value) => value.split('=')[0] === key);
|
|
493
|
+
if (existing >= 0) {
|
|
494
|
+
labels[existing] = label;
|
|
495
|
+
} else {
|
|
496
|
+
labels.push(label);
|
|
497
|
+
}
|
|
498
|
+
}
|
|
499
|
+
if (requestedLabels.length && opts.isolated !== 'docker') {
|
|
500
|
+
return {
|
|
501
|
+
success: false,
|
|
502
|
+
error: 'Docker labels require a Docker execution when resuming.',
|
|
503
|
+
};
|
|
504
|
+
}
|
|
505
|
+
const labelsChanged =
|
|
506
|
+
JSON.stringify(labels) !== JSON.stringify(opts.labels || []);
|
|
507
|
+
const planningRecord = {
|
|
508
|
+
...record,
|
|
509
|
+
options: {
|
|
510
|
+
...opts,
|
|
511
|
+
...(resources ? { resourceLimits: resources.resourceLimits } : {}),
|
|
512
|
+
...(requestedLabels.length ? { labels } : {}),
|
|
513
|
+
...(labelsChanged ? { forceSnapshot: true } : {}),
|
|
514
|
+
},
|
|
515
|
+
};
|
|
454
516
|
const plan = buildResumePlan(
|
|
455
517
|
planningRecord,
|
|
456
518
|
deps.command || null,
|
|
@@ -485,6 +547,9 @@ async function resumeExecution(store, identifier, deps = {}) {
|
|
|
485
547
|
};
|
|
486
548
|
appendLifecycle(lifecycleRecord, 'resume-started');
|
|
487
549
|
const updated = applyResumeToRecord(record, plan, null, attempt);
|
|
550
|
+
if (requestedLabels.length) {
|
|
551
|
+
updated.options.labels = labels;
|
|
552
|
+
}
|
|
488
553
|
if (resources) {
|
|
489
554
|
updated.options.resolvedLimits = resources.resolvedLimits;
|
|
490
555
|
updated.options.resourceLimitSpecs = resources.resourceLimitSpecs;
|
|
@@ -540,26 +605,108 @@ async function resumeExecution(store, identifier, deps = {}) {
|
|
|
540
605
|
}
|
|
541
606
|
containerId = launchResult.containerId || null;
|
|
542
607
|
} else {
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
608
|
+
let snapshotLock;
|
|
609
|
+
let snapshotCreated = false;
|
|
610
|
+
let derivedCreated = false;
|
|
611
|
+
try {
|
|
612
|
+
if (plan.mode === ResumeMode.DOCKER_SNAPSHOT) {
|
|
613
|
+
snapshotLock = acquireSnapshotLock(deps.snapshotOptions);
|
|
614
|
+
const preflight = preflightSnapshot(
|
|
615
|
+
plan.sessionName,
|
|
616
|
+
runner,
|
|
617
|
+
deps.snapshotOptions
|
|
618
|
+
);
|
|
619
|
+
plan.message += `\n${preflight.message}`;
|
|
620
|
+
if (record.logPath) {
|
|
621
|
+
appendLogFile(record.logPath, `${preflight.message}\n`);
|
|
622
|
+
}
|
|
549
623
|
}
|
|
550
|
-
if (
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
624
|
+
if (plan.handoffCommand) {
|
|
625
|
+
const copied = writeCommandHandoff(
|
|
626
|
+
plan.sessionName,
|
|
627
|
+
plan.handoffCommand,
|
|
628
|
+
runner,
|
|
629
|
+
opts
|
|
630
|
+
);
|
|
631
|
+
if (!copied.success) {
|
|
632
|
+
throw new Error(
|
|
633
|
+
(copied.stderr || '').trim() ||
|
|
634
|
+
copied.error ||
|
|
635
|
+
'Could not copy replacement command into the stopped container'
|
|
636
|
+
);
|
|
637
|
+
}
|
|
561
638
|
}
|
|
562
|
-
|
|
639
|
+
for (const step of plan.steps) {
|
|
640
|
+
let result;
|
|
641
|
+
try {
|
|
642
|
+
result = runner(step.command, step.args);
|
|
643
|
+
} catch (error) {
|
|
644
|
+
result = { success: false, error: error.message };
|
|
645
|
+
}
|
|
646
|
+
if (!result.success) {
|
|
647
|
+
const detail =
|
|
648
|
+
(result.stderr || '').trim() ||
|
|
649
|
+
result.error ||
|
|
650
|
+
`exit code ${result.status}`;
|
|
651
|
+
throw new Error(detail);
|
|
652
|
+
}
|
|
653
|
+
if (step.args[0] === 'commit') {
|
|
654
|
+
snapshotCreated = true;
|
|
655
|
+
}
|
|
656
|
+
if (step.args[0] === 'create') {
|
|
657
|
+
derivedCreated = true;
|
|
658
|
+
}
|
|
659
|
+
// commit returns an image ID; cp/update/network commands are not IDs.
|
|
660
|
+
if (['run', 'create'].includes(step.args[0])) {
|
|
661
|
+
containerId = (result.stdout || '').trim() || containerId;
|
|
662
|
+
}
|
|
663
|
+
}
|
|
664
|
+
if (plan.mode === ResumeMode.DOCKER_SNAPSHOT && deps.removeOriginal) {
|
|
665
|
+
const running = runner(getDockerCommand(), [
|
|
666
|
+
'inspect',
|
|
667
|
+
'-f',
|
|
668
|
+
'{{.State.Running}}',
|
|
669
|
+
plan.newSessionName,
|
|
670
|
+
]);
|
|
671
|
+
if (running.success && String(running.stdout).trim() === 'true') {
|
|
672
|
+
const removed = runner(getDockerCommand(), ['rm', plan.sessionName]);
|
|
673
|
+
const note = removed.success
|
|
674
|
+
? `Stopped original container removed: ${plan.sessionName}`
|
|
675
|
+
: `Original container retained: ${(removed.stderr || removed.error || 'docker rm failed').trim()}`;
|
|
676
|
+
plan.message += `\n${note}`;
|
|
677
|
+
if (record.logPath) {
|
|
678
|
+
appendLogFile(record.logPath, `${note}\n`);
|
|
679
|
+
}
|
|
680
|
+
} else {
|
|
681
|
+
plan.message +=
|
|
682
|
+
'\nOriginal container retained: the snapshot container is not running.';
|
|
683
|
+
}
|
|
684
|
+
}
|
|
685
|
+
} catch (error) {
|
|
686
|
+
if (derivedCreated) {
|
|
687
|
+
runner(getDockerCommand(), ['rm', plan.newSessionName]);
|
|
688
|
+
}
|
|
689
|
+
if (snapshotCreated) {
|
|
690
|
+
runner(getDockerCommand(), ['rmi', plan.snapshotImage]);
|
|
691
|
+
}
|
|
692
|
+
if (plan.handoffCommand) {
|
|
693
|
+
// A failed start must not leave a replacement hidden behind the old record.
|
|
694
|
+
try {
|
|
695
|
+
writeCommandHandoff(plan.sessionName, previous.command, runner, opts);
|
|
696
|
+
} catch {
|
|
697
|
+
/* Next explicit command can still repair the handoff. */
|
|
698
|
+
}
|
|
699
|
+
}
|
|
700
|
+
rollbackLaunch(store, previous);
|
|
701
|
+
appendLifecycle(lifecycleRecord, 'launch-failed', {
|
|
702
|
+
error: error.message,
|
|
703
|
+
});
|
|
704
|
+
return {
|
|
705
|
+
success: false,
|
|
706
|
+
error: `Failed to resume ${plan.backend} session "${plan.sessionName}": ${error.message}`,
|
|
707
|
+
};
|
|
708
|
+
} finally {
|
|
709
|
+
snapshotLock?.release();
|
|
563
710
|
}
|
|
564
711
|
|
|
565
712
|
const limitsLine = buildResourceLimitsStatusLine(plan.resourceLimits);
|
|
@@ -9,6 +9,7 @@ const os = require('os');
|
|
|
9
9
|
const fs = require('fs');
|
|
10
10
|
const path = require('path');
|
|
11
11
|
const { getTimestamp } = require('./isolation');
|
|
12
|
+
const { sanitizeLogToTemp, StreamingSanitizer } = require('./log-sanitizer');
|
|
12
13
|
|
|
13
14
|
/**
|
|
14
15
|
* Handle command failure - detect repository, upload log, create issue
|
|
@@ -55,7 +56,7 @@ function handleFailure(config, cmdName, fullCommand, exitCode, logPath) {
|
|
|
55
56
|
console.log('Log upload disabled via START_DISABLE_LOG_UPLOAD');
|
|
56
57
|
}
|
|
57
58
|
} else if (isGhUploadLogAvailable()) {
|
|
58
|
-
logUrl = uploadLog(logPath);
|
|
59
|
+
logUrl = uploadLog(logPath, { verbose: config.verbose });
|
|
59
60
|
if (logUrl) {
|
|
60
61
|
console.log(`Log uploaded: ${logUrl}`);
|
|
61
62
|
}
|
|
@@ -273,9 +274,11 @@ function isGhUploadLogAvailable() {
|
|
|
273
274
|
* @param {string} logPath - Path to the log file
|
|
274
275
|
* @returns {string|null} URL of the uploaded log or null
|
|
275
276
|
*/
|
|
276
|
-
function uploadLog(logPath) {
|
|
277
|
+
function uploadLog(logPath, options = {}) {
|
|
278
|
+
let prepared;
|
|
277
279
|
try {
|
|
278
|
-
|
|
280
|
+
prepared = sanitizeLogToTemp(logPath, { verbose: options.verbose });
|
|
281
|
+
const result = execFileSync('gh-upload-log', [prepared.path, '--private'], {
|
|
279
282
|
encoding: 'utf8',
|
|
280
283
|
stdio: ['pipe', 'pipe', 'pipe'],
|
|
281
284
|
});
|
|
@@ -293,9 +296,13 @@ function uploadLog(logPath) {
|
|
|
293
296
|
}
|
|
294
297
|
|
|
295
298
|
return null;
|
|
296
|
-
} catch
|
|
297
|
-
console.log(
|
|
299
|
+
} catch {
|
|
300
|
+
console.log('Warning: Log preparation or upload failed; upload blocked.');
|
|
298
301
|
return null;
|
|
302
|
+
} finally {
|
|
303
|
+
if (prepared) {
|
|
304
|
+
prepared.cleanup();
|
|
305
|
+
}
|
|
299
306
|
}
|
|
300
307
|
}
|
|
301
308
|
|
|
@@ -327,6 +334,9 @@ function canCreateIssue(owner, repo) {
|
|
|
327
334
|
*/
|
|
328
335
|
function createIssue(repoInfo, fullCommand, exitCode, logUrl) {
|
|
329
336
|
try {
|
|
337
|
+
fullCommand = new StreamingSanitizer()
|
|
338
|
+
.push(Buffer.from(fullCommand), true)
|
|
339
|
+
.toString();
|
|
330
340
|
const title = `Command failed with exit code ${exitCode}: ${fullCommand.substring(0, 50)}${fullCommand.length > 50 ? '...' : ''}`;
|
|
331
341
|
|
|
332
342
|
// Get runtime information
|