start-command 0.35.3 → 0.35.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/package.json +1 -1
- package/src/lib/detached-finalize.js +31 -7
- package/src/lib/detached-output.js +61 -0
- package/src/lib/docker-cleanup.js +20 -2
- package/src/lib/execution-attempt.js +153 -0
- package/src/lib/execution-recovery.js +61 -1
- package/src/lib/execution-resume-all.js +12 -1
- package/src/lib/execution-resume.js +70 -20
- package/src/lib/execution-store.js +31 -0
- package/src/lib/isolation-log-utils.js +3 -3
- package/src/lib/isolation.js +9 -7
- package/src/lib/status-formatter.js +38 -6
- package/test/execution-resume.js +2 -1
- package/test/regression-176.js +5 -1
- package/test/regression-187.js +546 -0
|
@@ -240,16 +240,16 @@ const FATAL_MARKER_TAIL_BYTES = 64 * 1024;
|
|
|
240
240
|
* @param {number} [bytes] - Maximum number of trailing bytes to read
|
|
241
241
|
* @returns {string|null} Tail content, or null when the file cannot be read
|
|
242
242
|
*/
|
|
243
|
-
function readLogTail(logPath, bytes = LOG_TAIL_BYTES) {
|
|
243
|
+
function readLogTail(logPath, bytes = LOG_TAIL_BYTES, offset = 0) {
|
|
244
244
|
let fd;
|
|
245
245
|
try {
|
|
246
246
|
fd = fs.openSync(logPath, 'r');
|
|
247
247
|
const size = fs.fstatSync(fd).size;
|
|
248
|
-
const length = Math.min(size, bytes);
|
|
248
|
+
const length = Math.min(Math.max(0, size - offset), bytes);
|
|
249
249
|
const buffer = Buffer.alloc(length);
|
|
250
250
|
fs.readSync(fd, buffer, 0, length, size - length);
|
|
251
251
|
const tail = buffer.toString('utf8');
|
|
252
|
-
if (size
|
|
252
|
+
if (size - length === offset || length === 0) {
|
|
253
253
|
return tail;
|
|
254
254
|
}
|
|
255
255
|
const firstNewline = tail.indexOf('\n');
|
package/src/lib/isolation.js
CHANGED
|
@@ -659,13 +659,15 @@ function runInDocker(command, options = {}) {
|
|
|
659
659
|
}
|
|
660
660
|
}
|
|
661
661
|
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
662
|
+
if (!options.deferCompletionWatcher) {
|
|
663
|
+
startDetachedDockerCompletionWatcher(
|
|
664
|
+
containerName,
|
|
665
|
+
cleanupPolicy,
|
|
666
|
+
options.logPath,
|
|
667
|
+
options.executionId || null,
|
|
668
|
+
{ recoverOnKill: Boolean(options.onKillResume) }
|
|
669
|
+
);
|
|
670
|
+
}
|
|
669
671
|
|
|
670
672
|
let message = `Command started in detached docker container: ${containerName}`;
|
|
671
673
|
message += `\nContainer ID: ${containerId.substring(0, 12)}`;
|
|
@@ -94,7 +94,15 @@ function readDockerState(record) {
|
|
|
94
94
|
if (opts.isolated !== 'docker' || !opts.sessionName) {
|
|
95
95
|
return null;
|
|
96
96
|
}
|
|
97
|
-
|
|
97
|
+
const state = inspectDockerState(opts.sessionName);
|
|
98
|
+
if (
|
|
99
|
+
record.attempt &&
|
|
100
|
+
state?.startedAt &&
|
|
101
|
+
Date.parse(state.startedAt) < Date.parse(record.attempt.startedAt)
|
|
102
|
+
) {
|
|
103
|
+
return null;
|
|
104
|
+
}
|
|
105
|
+
return state;
|
|
98
106
|
}
|
|
99
107
|
|
|
100
108
|
/**
|
|
@@ -506,12 +514,26 @@ function attachMemoryExhaustion(record, logTail) {
|
|
|
506
514
|
* @returns {Object} Possibly updated execution record
|
|
507
515
|
*/
|
|
508
516
|
function enrichDetachedStatus(record) {
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
517
|
+
if (!record) {
|
|
518
|
+
return record;
|
|
519
|
+
}
|
|
520
|
+
const {
|
|
521
|
+
readAttemptActivity,
|
|
522
|
+
readAttemptLogTail,
|
|
523
|
+
} = require('./execution-attempt');
|
|
524
|
+
const logTail = record
|
|
525
|
+
? readAttemptLogTail(record, FATAL_MARKER_TAIL_BYTES)
|
|
526
|
+
: null;
|
|
513
527
|
const resolved = resolveDetachedStatus(record, logTail);
|
|
514
|
-
|
|
528
|
+
const enriched = attachMemoryExhaustion(
|
|
529
|
+
attachExitReason(resolved, logTail),
|
|
530
|
+
logTail
|
|
531
|
+
);
|
|
532
|
+
const copy = Object.assign(
|
|
533
|
+
Object.create(Object.getPrototypeOf(enriched)),
|
|
534
|
+
enriched
|
|
535
|
+
);
|
|
536
|
+
return readAttemptActivity(copy);
|
|
515
537
|
}
|
|
516
538
|
|
|
517
539
|
/**
|
|
@@ -693,6 +715,16 @@ function formatRecordAsText(record) {
|
|
|
693
715
|
lines.push(`Stale Detected At: ${obj.staleDetectedAt}`);
|
|
694
716
|
}
|
|
695
717
|
lines.push(`Log Path: ${obj.logPath}`);
|
|
718
|
+
if (obj.attempt) {
|
|
719
|
+
lines.push(
|
|
720
|
+
`Attempt: ${obj.attempt.number}`,
|
|
721
|
+
`Attempt Started: ${obj.attempt.startedAt}`,
|
|
722
|
+
`Attempt Log Offset: ${obj.attempt.logOffset ?? 'N/A'}`,
|
|
723
|
+
`Launch Accepted: ${obj.attempt.launchAcceptedAt || 'N/A'}`,
|
|
724
|
+
`Watcher Attached: ${obj.attempt.watcherAttachedAt || 'N/A'}`,
|
|
725
|
+
`Last Output: ${obj.attempt.lastOutputAt || 'N/A'}`
|
|
726
|
+
);
|
|
727
|
+
}
|
|
696
728
|
|
|
697
729
|
// Format options as nested list instead of JSON
|
|
698
730
|
const optionEntries = Object.entries(obj.options || {}).filter(
|
package/test/execution-resume.js
CHANGED
|
@@ -274,7 +274,7 @@ describe('resumeExecution', () => {
|
|
|
274
274
|
assert.strictEqual(watchers.length, 1);
|
|
275
275
|
assert.strictEqual(watchers[0][0], 'box');
|
|
276
276
|
assert.strictEqual(watchers[0][2], '/tmp/session.log');
|
|
277
|
-
assert.strictEqual(store.saved.length,
|
|
277
|
+
assert.strictEqual(store.saved.length, 2);
|
|
278
278
|
assert.strictEqual(store.saved[0].uuid, record.uuid);
|
|
279
279
|
assert.strictEqual(store.saved[0].status, 'executing');
|
|
280
280
|
assert.strictEqual(store.saved[0].options.resumeCount, 1);
|
|
@@ -329,6 +329,7 @@ describe('resumeExecution', () => {
|
|
|
329
329
|
const launches = [];
|
|
330
330
|
const result = await resumeExecution(store, 'box', {
|
|
331
331
|
probe: () => probeWith(SessionState.MISSING),
|
|
332
|
+
startWatcher: () => {},
|
|
332
333
|
runIsolated: (backend, command, options) => {
|
|
333
334
|
launches.push({ backend, command, options });
|
|
334
335
|
return Promise.resolve({ success: true, containerId: 'newid' });
|
package/test/regression-176.js
CHANGED
|
@@ -313,7 +313,11 @@ describe('issue #176: snapshot resume keeps the resource limits', () => {
|
|
|
313
313
|
expect(saved[0].options.sessionName).toBe('box-resume-1');
|
|
314
314
|
expect(result.output).toContain('--pids-limit=64');
|
|
315
315
|
// A snapshot container has no recovery selector.
|
|
316
|
-
expect(watchers[0][4]).toEqual({
|
|
316
|
+
expect(watchers[0][4]).toEqual({
|
|
317
|
+
recoverOnKill: false,
|
|
318
|
+
since: saved[0].attempt.startedAt,
|
|
319
|
+
attemptNumber: saved[0].attempt.number,
|
|
320
|
+
});
|
|
317
321
|
});
|
|
318
322
|
|
|
319
323
|
it('shows the limits in [Isolation] lines and stores them as metadata', () => {
|
|
@@ -0,0 +1,546 @@
|
|
|
1
|
+
const { describe, it, expect, afterEach } = require('bun:test');
|
|
2
|
+
const fs = require('fs');
|
|
3
|
+
const os = require('os');
|
|
4
|
+
const path = require('path');
|
|
5
|
+
const { spawnSync } = require('child_process');
|
|
6
|
+
const { recoverKilledExecution } = require('../src/lib/execution-recovery');
|
|
7
|
+
const {
|
|
8
|
+
ExecutionRecord,
|
|
9
|
+
ExecutionStore,
|
|
10
|
+
} = require('../src/lib/execution-store');
|
|
11
|
+
const { resumeExecution } = require('../src/lib/execution-resume');
|
|
12
|
+
const { resumeAllExecutions } = require('../src/lib/execution-resume-all');
|
|
13
|
+
const { SessionState } = require('../src/lib/session-probe');
|
|
14
|
+
const { OutputObserver } = require('../src/lib/detached-output');
|
|
15
|
+
const {
|
|
16
|
+
activityPath,
|
|
17
|
+
readAttemptLogTail,
|
|
18
|
+
} = require('../src/lib/execution-attempt');
|
|
19
|
+
const {
|
|
20
|
+
buildDetachedDockerCompletionScript,
|
|
21
|
+
DOCKER_CONTAINER_CLEANUP_POLICY,
|
|
22
|
+
} = require('../src/lib/docker-cleanup');
|
|
23
|
+
const { finalizeDetachedExecution } = require('../src/lib/detached-finalize');
|
|
24
|
+
const {
|
|
25
|
+
enrichDetachedStatus,
|
|
26
|
+
formatRecord,
|
|
27
|
+
formatRecordList,
|
|
28
|
+
} = require('../src/lib/status-formatter');
|
|
29
|
+
|
|
30
|
+
const directories = [];
|
|
31
|
+
afterEach(() => {
|
|
32
|
+
for (const directory of directories.splice(0)) {
|
|
33
|
+
fs.rmSync(directory, { recursive: true, force: true });
|
|
34
|
+
}
|
|
35
|
+
});
|
|
36
|
+
|
|
37
|
+
function fixture() {
|
|
38
|
+
const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'resume-187-'));
|
|
39
|
+
directories.push(directory);
|
|
40
|
+
const logPath = path.join(directory, 'execution.log');
|
|
41
|
+
const oldLog =
|
|
42
|
+
'previous output: 🐝\nFATAL ERROR: Reached heap limit Allocation failed - JavaScript heap out of memory\n' +
|
|
43
|
+
'\n==================================================\nFinished: 2026-01-01 00:01:00.000\nExit Code: 137\n';
|
|
44
|
+
fs.writeFileSync(logPath, oldLog);
|
|
45
|
+
const store = new ExecutionStore({ appFolder: directory, useLinks: false });
|
|
46
|
+
const record = new ExecutionRecord({
|
|
47
|
+
command: 'worker',
|
|
48
|
+
logPath,
|
|
49
|
+
startTime: '2026-01-01T00:00:00.000Z',
|
|
50
|
+
status: 'executed',
|
|
51
|
+
exitCode: 137,
|
|
52
|
+
endTime: '2026-01-01T00:01:00.000Z',
|
|
53
|
+
endTimeSource: 'docker-finished-at',
|
|
54
|
+
observedAt: '2026-01-01T00:01:01.000Z',
|
|
55
|
+
staleDetectedAt: '2026-01-01T00:01:02.000Z',
|
|
56
|
+
containerStartedAt: '2026-01-01T00:00:00.000Z',
|
|
57
|
+
exitReason: 'cgroup-oom-killer',
|
|
58
|
+
oomKilled: true,
|
|
59
|
+
memoryExhausted: true,
|
|
60
|
+
memoryExhaustedReason: 'cgroup-oom-killer',
|
|
61
|
+
cgroupMemory: {
|
|
62
|
+
limitBytes: 128,
|
|
63
|
+
peakBytes: 128,
|
|
64
|
+
oomEvents: 1,
|
|
65
|
+
oomKills: 1,
|
|
66
|
+
},
|
|
67
|
+
options: {
|
|
68
|
+
isolated: 'docker',
|
|
69
|
+
isolationMode: 'detached',
|
|
70
|
+
sessionName: 'issue-187-container-that-does-not-exist',
|
|
71
|
+
containerError: 'previous error',
|
|
72
|
+
},
|
|
73
|
+
});
|
|
74
|
+
store.save(record);
|
|
75
|
+
return { store, record, logPath, oldLog };
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
const stopped = () => ({ alive: false, state: SessionState.STOPPED });
|
|
79
|
+
const accepted = () => ({ success: true, stdout: 'container-id\n', status: 0 });
|
|
80
|
+
|
|
81
|
+
describe('issue #187: explicit resume attempts', () => {
|
|
82
|
+
it('reattaches the current attempt without enabling kill recovery for snapshot commands', async () => {
|
|
83
|
+
for (const mode of ['docker-start', 'docker-snapshot', 'relaunch']) {
|
|
84
|
+
const command = mode === 'docker-snapshot' ? 'replacement' : undefined;
|
|
85
|
+
const { store, record } = fixture();
|
|
86
|
+
record.options.onKillResume = 1;
|
|
87
|
+
store.save(record);
|
|
88
|
+
await resumeExecution(store, record.uuid, {
|
|
89
|
+
command,
|
|
90
|
+
probe:
|
|
91
|
+
mode === 'relaunch'
|
|
92
|
+
? () => ({ alive: false, state: SessionState.MISSING })
|
|
93
|
+
: stopped,
|
|
94
|
+
runner: accepted,
|
|
95
|
+
runIsolated: () => ({ success: true, containerId: 'new-container' }),
|
|
96
|
+
startWatcher: (_name, _policy, _log, _uuid, options) => {
|
|
97
|
+
expect(options.recoverOnKill).toBe(mode !== 'docker-snapshot');
|
|
98
|
+
},
|
|
99
|
+
});
|
|
100
|
+
const current = store.get(record.uuid);
|
|
101
|
+
let options;
|
|
102
|
+
resumeAllExecutions(store, {
|
|
103
|
+
probe: () => ({
|
|
104
|
+
alive: true,
|
|
105
|
+
backend: 'docker',
|
|
106
|
+
sessionName: current.options.sessionName,
|
|
107
|
+
}),
|
|
108
|
+
startWatcher: (_name, _policy, _log, _uuid, watcherOptions) => {
|
|
109
|
+
options = watcherOptions;
|
|
110
|
+
},
|
|
111
|
+
});
|
|
112
|
+
expect(options.attemptNumber).toBe(2);
|
|
113
|
+
expect(options.since).toBe(current.attempt.startedAt);
|
|
114
|
+
expect(options.recoverOnKill).toBe(mode !== 'docker-snapshot');
|
|
115
|
+
}
|
|
116
|
+
});
|
|
117
|
+
it('merges attachment metadata without reviving a completed or newer attempt', async () => {
|
|
118
|
+
const { store, record } = fixture();
|
|
119
|
+
await resumeExecution(store, record.uuid, {
|
|
120
|
+
probe: stopped,
|
|
121
|
+
runner: accepted,
|
|
122
|
+
startWatcher: () => {},
|
|
123
|
+
});
|
|
124
|
+
const current = store.get(record.uuid);
|
|
125
|
+
current.status = 'executed';
|
|
126
|
+
current.exitCode = 0;
|
|
127
|
+
store.save(current);
|
|
128
|
+
const patched = store.patchAttempt(record.uuid, 2, {
|
|
129
|
+
watcherAttachedAt: '2026-10-06T09:00:00Z',
|
|
130
|
+
});
|
|
131
|
+
expect(patched.status).toBe('executed');
|
|
132
|
+
expect(patched.exitCode).toBe(0);
|
|
133
|
+
expect(
|
|
134
|
+
store.patchAttempt(record.uuid, 1, { watcherError: 'stale watcher' })
|
|
135
|
+
).toBeNull();
|
|
136
|
+
expect(store.get(record.uuid).attempt.watcherError).toBeUndefined();
|
|
137
|
+
expect(enrichDetachedStatus(null)).toBeNull();
|
|
138
|
+
});
|
|
139
|
+
it('keeps boundaries and evidence when an explicit resume is automatically recovered', async () => {
|
|
140
|
+
const { store, record, logPath } = fixture();
|
|
141
|
+
record.options.onKillResume = 1;
|
|
142
|
+
store.save(record);
|
|
143
|
+
await resumeExecution(store, record.uuid, {
|
|
144
|
+
probe: stopped,
|
|
145
|
+
runner: accepted,
|
|
146
|
+
startWatcher: () => {},
|
|
147
|
+
});
|
|
148
|
+
const current = store.get(record.uuid);
|
|
149
|
+
const outcome = recoverKilledExecution({
|
|
150
|
+
store,
|
|
151
|
+
executionId: record.uuid,
|
|
152
|
+
exitCode: 137,
|
|
153
|
+
oomKilled: true,
|
|
154
|
+
startedAt: current.attempt.startedAt,
|
|
155
|
+
finishedAt: new Date().toISOString(),
|
|
156
|
+
cgroupMemory: '128 128 1 1',
|
|
157
|
+
runner: accepted,
|
|
158
|
+
startWatcher: (_name, _policy, _log, _uuid, options) => {
|
|
159
|
+
expect(options.attemptNumber).toBe(3);
|
|
160
|
+
expect(store.get(record.uuid).attempt.number).toBe(3);
|
|
161
|
+
},
|
|
162
|
+
});
|
|
163
|
+
expect(outcome.recovered).toBe(true);
|
|
164
|
+
const saved = store.get(record.uuid);
|
|
165
|
+
expect(saved.attemptHistory.map((a) => a.exitCode)).toEqual([137, 137]);
|
|
166
|
+
expect(saved.attemptHistory[1].cgroupMemory.oomKills).toBe(1);
|
|
167
|
+
expect(saved.cgroupMemory).toBeUndefined();
|
|
168
|
+
expect(saved.attempt.lastOutputAt).toBeNull();
|
|
169
|
+
expect(saved.attempt.logOffset).toBeGreaterThan(current.attempt.logOffset);
|
|
170
|
+
expect(fs.readFileSync(logPath, 'utf8')).toContain('automatic-recovery');
|
|
171
|
+
});
|
|
172
|
+
|
|
173
|
+
it('runs the timestamped output capture and finalizer through the real watcher shell', async () => {
|
|
174
|
+
if (process.platform === 'win32') {
|
|
175
|
+
return;
|
|
176
|
+
}
|
|
177
|
+
const { store, record, logPath } = fixture();
|
|
178
|
+
await resumeExecution(store, record.uuid, {
|
|
179
|
+
probe: stopped,
|
|
180
|
+
runner: accepted,
|
|
181
|
+
startWatcher: () => {},
|
|
182
|
+
});
|
|
183
|
+
const current = store.get(record.uuid);
|
|
184
|
+
const directory = path.dirname(logPath);
|
|
185
|
+
const bin = path.join(directory, 'bin');
|
|
186
|
+
fs.mkdirSync(bin);
|
|
187
|
+
const at = new Date().toISOString();
|
|
188
|
+
fs.writeFileSync(
|
|
189
|
+
path.join(bin, 'docker'),
|
|
190
|
+
`#!/bin/sh
|
|
191
|
+
case "$1" in
|
|
192
|
+
logs) printf '%s command output 🐝\\n' "$TEST_OUTPUT_AT" ;;
|
|
193
|
+
wait) echo 0 ;;
|
|
194
|
+
inspect) case "$3" in
|
|
195
|
+
*State.Running*) echo false ;;
|
|
196
|
+
*State.ExitCode*) echo "0 false $TEST_OUTPUT_AT $TEST_OUTPUT_AT" ;;
|
|
197
|
+
*State.Error*) echo '' ;;
|
|
198
|
+
*) exit 1 ;;
|
|
199
|
+
esac ;;
|
|
200
|
+
esac
|
|
201
|
+
`,
|
|
202
|
+
{ mode: 0o755 }
|
|
203
|
+
);
|
|
204
|
+
const script = buildDetachedDockerCompletionScript(
|
|
205
|
+
current.options.sessionName,
|
|
206
|
+
DOCKER_CONTAINER_CLEANUP_POLICY.KEEP,
|
|
207
|
+
logPath,
|
|
208
|
+
record.uuid,
|
|
209
|
+
{ since: current.attempt.startedAt, attemptNumber: 2 }
|
|
210
|
+
);
|
|
211
|
+
const output = spawnSync('/bin/sh', ['-c', script], {
|
|
212
|
+
env: {
|
|
213
|
+
...process.env,
|
|
214
|
+
PATH: `${bin}${path.delimiter}${process.env.PATH}`,
|
|
215
|
+
START_APP_FOLDER: directory,
|
|
216
|
+
TEST_OUTPUT_AT: at,
|
|
217
|
+
START_COMMAND_CGROUP_ROOT: path.join(directory, 'no-cgroup'),
|
|
218
|
+
START_COMMAND_PROC_ROOT: path.join(directory, 'no-proc'),
|
|
219
|
+
},
|
|
220
|
+
encoding: 'utf8',
|
|
221
|
+
maxBuffer: 1024 * 1024,
|
|
222
|
+
});
|
|
223
|
+
expect(output.status).toBe(0);
|
|
224
|
+
const saved = store.get(record.uuid);
|
|
225
|
+
expect(saved.status).toBe('executed');
|
|
226
|
+
expect(saved.exitCode).toBe(0);
|
|
227
|
+
expect(saved.attempt.lastOutputAt).toBe(at);
|
|
228
|
+
expect(saved.cgroupMemory).toBeUndefined();
|
|
229
|
+
expect(fs.readFileSync(logPath, 'utf8')).toContain('command output 🐝');
|
|
230
|
+
}, 10000);
|
|
231
|
+
|
|
232
|
+
it('archives terminal evidence and resets every attempt-local field', async () => {
|
|
233
|
+
const { store, record, oldLog } = fixture();
|
|
234
|
+
const outcome = await resumeExecution(store, record.uuid, {
|
|
235
|
+
probe: stopped,
|
|
236
|
+
runner: accepted,
|
|
237
|
+
startWatcher: () => {},
|
|
238
|
+
});
|
|
239
|
+
expect(outcome.success).toBe(true);
|
|
240
|
+
const saved = store.get(record.uuid);
|
|
241
|
+
expect(saved.status).toBe('executing');
|
|
242
|
+
expect(saved.exitCode).toBeNull();
|
|
243
|
+
for (const field of [
|
|
244
|
+
'endTimeSource',
|
|
245
|
+
'observedAt',
|
|
246
|
+
'staleDetectedAt',
|
|
247
|
+
'containerStartedAt',
|
|
248
|
+
'exitReason',
|
|
249
|
+
'oomKilled',
|
|
250
|
+
'memoryExhausted',
|
|
251
|
+
'memoryExhaustedReason',
|
|
252
|
+
'cgroupMemory',
|
|
253
|
+
]) {
|
|
254
|
+
expect(saved[field]).toBeUndefined();
|
|
255
|
+
}
|
|
256
|
+
expect(saved.options.containerError).toBeUndefined();
|
|
257
|
+
expect(saved.attempt.number).toBe(2);
|
|
258
|
+
expect(saved.attempt.logOffset).toBe(Buffer.byteLength(oldLog));
|
|
259
|
+
expect(saved.attempt.startedAt).not.toBe(record.startTime);
|
|
260
|
+
expect(saved.attempt.launchAcceptedAt).toBeDefined();
|
|
261
|
+
expect(saved.attempt.lastOutputAt).toBeNull();
|
|
262
|
+
expect(saved.attemptHistory[0].exitCode).toBe(137);
|
|
263
|
+
expect(saved.attemptHistory[0].memoryExhausted).toBe(true);
|
|
264
|
+
expect(saved.attemptHistory[0].cgroupMemory.oomKills).toBe(1);
|
|
265
|
+
});
|
|
266
|
+
|
|
267
|
+
it('does not resolve the previous footer or memory marker as current evidence', async () => {
|
|
268
|
+
const { store, record } = fixture();
|
|
269
|
+
await resumeExecution(store, record.uuid, {
|
|
270
|
+
probe: stopped,
|
|
271
|
+
runner: accepted,
|
|
272
|
+
startWatcher: () => {},
|
|
273
|
+
});
|
|
274
|
+
const current = store.get(record.uuid);
|
|
275
|
+
const enriched = enrichDetachedStatus(current);
|
|
276
|
+
expect(enriched.status).toBe('executing');
|
|
277
|
+
expect(enriched.exitCode).toBeNull();
|
|
278
|
+
expect(enriched.memoryExhausted).toBeUndefined();
|
|
279
|
+
const status = JSON.parse(formatRecord(enriched, 'json'));
|
|
280
|
+
const list = JSON.parse(formatRecordList([enriched], 'json'));
|
|
281
|
+
expect(status.attempt.logOffset).toBe(current.attempt.logOffset);
|
|
282
|
+
expect(list.executions[0].attempt.startedAt).toBe(
|
|
283
|
+
current.attempt.startedAt
|
|
284
|
+
);
|
|
285
|
+
});
|
|
286
|
+
|
|
287
|
+
it('persists the new attempt before a watcher can finalize an immediate exit', async () => {
|
|
288
|
+
const { store, record } = fixture();
|
|
289
|
+
await resumeExecution(store, record.uuid, {
|
|
290
|
+
probe: stopped,
|
|
291
|
+
runner: accepted,
|
|
292
|
+
startWatcher: (name, policy, logPath, uuid, options) => {
|
|
293
|
+
expect(store.get(uuid).status).toBe('executing');
|
|
294
|
+
expect(options.since).toBe(store.get(uuid).attempt.startedAt);
|
|
295
|
+
const outcome = finalizeDetachedExecution({
|
|
296
|
+
store,
|
|
297
|
+
executionId: uuid,
|
|
298
|
+
exitCode: 0,
|
|
299
|
+
oomKilled: false,
|
|
300
|
+
startedAt: new Date().toISOString(),
|
|
301
|
+
finishedAt: new Date().toISOString(),
|
|
302
|
+
attemptNumber: store.get(uuid).attempt.number,
|
|
303
|
+
});
|
|
304
|
+
expect(outcome.updated).toBe(true);
|
|
305
|
+
},
|
|
306
|
+
});
|
|
307
|
+
expect(store.get(record.uuid).status).toBe('executed');
|
|
308
|
+
expect(store.get(record.uuid).exitCode).toBe(0);
|
|
309
|
+
});
|
|
310
|
+
|
|
311
|
+
it('records a structured boundary before launch and accepted launch separately from output', async () => {
|
|
312
|
+
const { store, record, logPath, oldLog } = fixture();
|
|
313
|
+
await resumeExecution(store, record.uuid, {
|
|
314
|
+
probe: stopped,
|
|
315
|
+
runner: () => {
|
|
316
|
+
expect(fs.readFileSync(logPath, 'utf8').slice(oldLog.length)).toContain(
|
|
317
|
+
'resume-started'
|
|
318
|
+
);
|
|
319
|
+
return accepted();
|
|
320
|
+
},
|
|
321
|
+
startWatcher: () => {},
|
|
322
|
+
});
|
|
323
|
+
const log = fs.readFileSync(logPath, 'utf8');
|
|
324
|
+
expect(log).toContain('launch-accepted');
|
|
325
|
+
expect(log).toContain('watcher-attached');
|
|
326
|
+
expect(store.get(record.uuid).attempt.lastOutputAt).toBeNull();
|
|
327
|
+
});
|
|
328
|
+
|
|
329
|
+
it('preserves the stopped record when launch fails and logs that failure', async () => {
|
|
330
|
+
const { store, record, logPath } = fixture();
|
|
331
|
+
const outcome = await resumeExecution(store, record.uuid, {
|
|
332
|
+
probe: stopped,
|
|
333
|
+
runner: () => ({ success: false, stderr: 'launch rejected', status: 1 }),
|
|
334
|
+
startWatcher: () => {
|
|
335
|
+
throw new Error('must not attach');
|
|
336
|
+
},
|
|
337
|
+
});
|
|
338
|
+
expect(outcome.success).toBe(false);
|
|
339
|
+
expect(store.get(record.uuid).exitCode).toBe(137);
|
|
340
|
+
expect(store.get(record.uuid).memoryExhausted).toBe(true);
|
|
341
|
+
expect(fs.readFileSync(logPath, 'utf8')).toContain('launch-failed');
|
|
342
|
+
});
|
|
343
|
+
|
|
344
|
+
it('records previous and new container names for a snapshot resume', async () => {
|
|
345
|
+
const { store, record, logPath } = fixture();
|
|
346
|
+
const result = await resumeExecution(store, record.uuid, {
|
|
347
|
+
command: 'replacement',
|
|
348
|
+
probe: stopped,
|
|
349
|
+
runner: accepted,
|
|
350
|
+
startWatcher: () => {},
|
|
351
|
+
});
|
|
352
|
+
expect(result.success).toBe(true);
|
|
353
|
+
const saved = store.get(record.uuid);
|
|
354
|
+
expect(saved.attempt.sessionName).toBe(
|
|
355
|
+
`${record.options.sessionName}-resume-1`
|
|
356
|
+
);
|
|
357
|
+
expect(saved.attempt.previousSessionName).toBe(record.options.sessionName);
|
|
358
|
+
expect(saved.attemptHistory[0].command).toBe('worker');
|
|
359
|
+
expect(fs.readFileSync(logPath, 'utf8')).toContain(
|
|
360
|
+
saved.attempt.sessionName
|
|
361
|
+
);
|
|
362
|
+
});
|
|
363
|
+
|
|
364
|
+
it('defers the relaunch watcher until the accepted attempt is saved', async () => {
|
|
365
|
+
const { store, record } = fixture();
|
|
366
|
+
let attached = false;
|
|
367
|
+
const result = await resumeExecution(store, record.uuid, {
|
|
368
|
+
probe: () => ({ alive: false, state: SessionState.MISSING }),
|
|
369
|
+
runIsolated: (backend, command, options) => {
|
|
370
|
+
expect(options.deferCompletionWatcher).toBe(true);
|
|
371
|
+
expect(store.get(record.uuid).status).toBe('executed');
|
|
372
|
+
return { success: true, containerId: 'replacement-container' };
|
|
373
|
+
},
|
|
374
|
+
startWatcher: () => {
|
|
375
|
+
attached = true;
|
|
376
|
+
expect(store.get(record.uuid).status).toBe('executing');
|
|
377
|
+
},
|
|
378
|
+
});
|
|
379
|
+
expect(result.success).toBe(true);
|
|
380
|
+
expect(attached).toBe(true);
|
|
381
|
+
expect(store.get(record.uuid).attempt.mode).toBe('relaunch');
|
|
382
|
+
});
|
|
383
|
+
|
|
384
|
+
it('reports watcher failure while preserving the accepted launch', async () => {
|
|
385
|
+
const { store, record, logPath } = fixture();
|
|
386
|
+
const result = await resumeExecution(store, record.uuid, {
|
|
387
|
+
probe: stopped,
|
|
388
|
+
runner: accepted,
|
|
389
|
+
startWatcher: () => {
|
|
390
|
+
throw new Error('cannot spawn watcher');
|
|
391
|
+
},
|
|
392
|
+
});
|
|
393
|
+
expect(result.success).toBe(false);
|
|
394
|
+
expect(result.error).toContain('Launch accepted');
|
|
395
|
+
const saved = store.get(record.uuid);
|
|
396
|
+
expect(saved.status).toBe('executing');
|
|
397
|
+
expect(saved.attempt.launchAcceptedAt).toBeDefined();
|
|
398
|
+
expect(saved.attempt.watcherAttachedAt).toBeNull();
|
|
399
|
+
expect(saved.attempt.watcherError).toBe('cannot spawn watcher');
|
|
400
|
+
expect(fs.readFileSync(logPath, 'utf8')).toContain(
|
|
401
|
+
'watcher-attachment-failed'
|
|
402
|
+
);
|
|
403
|
+
});
|
|
404
|
+
|
|
405
|
+
it('rejects a finalizer carrying evidence from a previous attempt', async () => {
|
|
406
|
+
const { store, record } = fixture();
|
|
407
|
+
await resumeExecution(store, record.uuid, {
|
|
408
|
+
probe: stopped,
|
|
409
|
+
runner: accepted,
|
|
410
|
+
startWatcher: () => {},
|
|
411
|
+
});
|
|
412
|
+
const result = finalizeDetachedExecution({
|
|
413
|
+
store,
|
|
414
|
+
executionId: record.uuid,
|
|
415
|
+
attemptNumber: 1,
|
|
416
|
+
exitCode: 137,
|
|
417
|
+
oomKilled: true,
|
|
418
|
+
startedAt: record.startTime,
|
|
419
|
+
finishedAt: record.endTime,
|
|
420
|
+
});
|
|
421
|
+
expect(result.reason).toBe('stale-attempt');
|
|
422
|
+
expect(store.get(record.uuid).exitCode).toBeNull();
|
|
423
|
+
});
|
|
424
|
+
|
|
425
|
+
it('does not reuse a previous fatal marker when the new run exits 139', async () => {
|
|
426
|
+
const { store, record, logPath } = fixture();
|
|
427
|
+
await resumeExecution(store, record.uuid, {
|
|
428
|
+
probe: stopped,
|
|
429
|
+
runner: accepted,
|
|
430
|
+
startWatcher: () => {},
|
|
431
|
+
});
|
|
432
|
+
const current = store.get(record.uuid);
|
|
433
|
+
const at = new Date().toISOString();
|
|
434
|
+
finalizeDetachedExecution({
|
|
435
|
+
store,
|
|
436
|
+
executionId: record.uuid,
|
|
437
|
+
attemptNumber: current.attempt.number,
|
|
438
|
+
exitCode: 139,
|
|
439
|
+
oomKilled: false,
|
|
440
|
+
startedAt: at,
|
|
441
|
+
finishedAt: at,
|
|
442
|
+
});
|
|
443
|
+
const terminal = enrichDetachedStatus(store.get(record.uuid));
|
|
444
|
+
expect(terminal.memoryExhausted).toBeUndefined();
|
|
445
|
+
expect(terminal.cgroupMemory).toBeUndefined();
|
|
446
|
+
expect(terminal.exitReason).not.toBe('cgroup-oom-killer');
|
|
447
|
+
expect(fs.readFileSync(logPath, 'utf8')).toContain('"event":"terminal"');
|
|
448
|
+
});
|
|
449
|
+
|
|
450
|
+
it('keeps successive attempts and their terminal evidence through store round trips', async () => {
|
|
451
|
+
const { store, record } = fixture();
|
|
452
|
+
for (let i = 0; i < 2; i++) {
|
|
453
|
+
await resumeExecution(store, record.uuid, {
|
|
454
|
+
probe: stopped,
|
|
455
|
+
runner: accepted,
|
|
456
|
+
startWatcher: () => {},
|
|
457
|
+
});
|
|
458
|
+
const current = store.get(record.uuid);
|
|
459
|
+
const at = new Date().toISOString();
|
|
460
|
+
finalizeDetachedExecution({
|
|
461
|
+
store,
|
|
462
|
+
executionId: record.uuid,
|
|
463
|
+
attemptNumber: current.attempt.number,
|
|
464
|
+
exitCode: i,
|
|
465
|
+
oomKilled: false,
|
|
466
|
+
startedAt: at,
|
|
467
|
+
finishedAt: at,
|
|
468
|
+
});
|
|
469
|
+
}
|
|
470
|
+
const saved = store.get(record.uuid);
|
|
471
|
+
expect(saved.attempt.number).toBe(3);
|
|
472
|
+
expect(saved.attemptHistory.map((a) => a.exitCode)).toEqual([137, 0]);
|
|
473
|
+
expect(saved.attemptHistory[0].cgroupMemory.oomKills).toBe(1);
|
|
474
|
+
});
|
|
475
|
+
|
|
476
|
+
it('reports output time only after fresh timestamped command output, even without a newline', async () => {
|
|
477
|
+
const { store, record, logPath } = fixture();
|
|
478
|
+
await resumeExecution(store, record.uuid, {
|
|
479
|
+
probe: stopped,
|
|
480
|
+
runner: accepted,
|
|
481
|
+
startWatcher: () => {},
|
|
482
|
+
});
|
|
483
|
+
const current = store.get(record.uuid);
|
|
484
|
+
const observer = new OutputObserver(
|
|
485
|
+
logPath,
|
|
486
|
+
current.attempt.number,
|
|
487
|
+
current.attempt.startedAt
|
|
488
|
+
);
|
|
489
|
+
observer.write(
|
|
490
|
+
Buffer.from(
|
|
491
|
+
'Docker administrative error\n2026-01-01T00:00:00Z old output\n'
|
|
492
|
+
)
|
|
493
|
+
);
|
|
494
|
+
expect(
|
|
495
|
+
enrichDetachedStatus(store.get(record.uuid)).attempt.lastOutputAt
|
|
496
|
+
).toBeNull();
|
|
497
|
+
const at = new Date().toISOString();
|
|
498
|
+
const output = Buffer.from(`${at} quiet run emits 🐝 without a newline`);
|
|
499
|
+
observer.write(output.subarray(0, 10));
|
|
500
|
+
observer.write(output.subarray(10));
|
|
501
|
+
expect(
|
|
502
|
+
enrichDetachedStatus(store.get(record.uuid)).attempt.lastOutputAt
|
|
503
|
+
).toBe(at);
|
|
504
|
+
expect(fs.readFileSync(logPath).includes(output)).toBe(true);
|
|
505
|
+
fs.appendFileSync(
|
|
506
|
+
logPath,
|
|
507
|
+
'\n[Start Command Lifecycle] administrative activity\n'
|
|
508
|
+
);
|
|
509
|
+
expect(
|
|
510
|
+
enrichDetachedStatus(store.get(record.uuid)).attempt.lastOutputAt
|
|
511
|
+
).toBe(at);
|
|
512
|
+
fs.writeFileSync(activityPath(current), '2026-01-01T00:00:00Z');
|
|
513
|
+
expect(
|
|
514
|
+
enrichDetachedStatus(store.get(record.uuid)).attempt.lastOutputAt
|
|
515
|
+
).toBeNull();
|
|
516
|
+
});
|
|
517
|
+
|
|
518
|
+
it('bounds reads and never scans prior evidence after log truncation', async () => {
|
|
519
|
+
const { store, record, logPath } = fixture();
|
|
520
|
+
await resumeExecution(store, record.uuid, {
|
|
521
|
+
probe: stopped,
|
|
522
|
+
runner: accepted,
|
|
523
|
+
startWatcher: () => {},
|
|
524
|
+
});
|
|
525
|
+
const current = store.get(record.uuid);
|
|
526
|
+
fs.appendFileSync(logPath, `\n${'x'.repeat(70 * 1024)}\nfresh\n`);
|
|
527
|
+
expect(readAttemptLogTail(current, 64 * 1024).length).toBeLessThanOrEqual(
|
|
528
|
+
64 * 1024
|
|
529
|
+
);
|
|
530
|
+
fs.writeFileSync(logPath, 'old footer truncated');
|
|
531
|
+
expect(readAttemptLogTail(current, 64 * 1024)).toBe('');
|
|
532
|
+
});
|
|
533
|
+
|
|
534
|
+
it('builds a watcher that filters historical output and carries the attempt identity', () => {
|
|
535
|
+
const script = buildDetachedDockerCompletionScript(
|
|
536
|
+
'box',
|
|
537
|
+
DOCKER_CONTAINER_CLEANUP_POLICY.KEEP,
|
|
538
|
+
'/tmp/log with spaces',
|
|
539
|
+
'uuid',
|
|
540
|
+
{ since: '2026-10-06T08:00:00Z', attemptNumber: 2 }
|
|
541
|
+
);
|
|
542
|
+
expect(script).toContain('docker logs -f --timestamps --since');
|
|
543
|
+
expect(script).toContain('detached-output.js');
|
|
544
|
+
expect(script).toContain("'2' >/dev/null");
|
|
545
|
+
});
|
|
546
|
+
});
|