start-command 0.35.3 → 0.35.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -240,16 +240,16 @@ const FATAL_MARKER_TAIL_BYTES = 64 * 1024;
240
240
  * @param {number} [bytes] - Maximum number of trailing bytes to read
241
241
  * @returns {string|null} Tail content, or null when the file cannot be read
242
242
  */
243
- function readLogTail(logPath, bytes = LOG_TAIL_BYTES) {
243
+ function readLogTail(logPath, bytes = LOG_TAIL_BYTES, offset = 0) {
244
244
  let fd;
245
245
  try {
246
246
  fd = fs.openSync(logPath, 'r');
247
247
  const size = fs.fstatSync(fd).size;
248
- const length = Math.min(size, bytes);
248
+ const length = Math.min(Math.max(0, size - offset), bytes);
249
249
  const buffer = Buffer.alloc(length);
250
250
  fs.readSync(fd, buffer, 0, length, size - length);
251
251
  const tail = buffer.toString('utf8');
252
- if (size <= bytes) {
252
+ if (size - length === offset || length === 0) {
253
253
  return tail;
254
254
  }
255
255
  const firstNewline = tail.indexOf('\n');
@@ -659,13 +659,15 @@ function runInDocker(command, options = {}) {
659
659
  }
660
660
  }
661
661
 
662
- startDetachedDockerCompletionWatcher(
663
- containerName,
664
- cleanupPolicy,
665
- options.logPath,
666
- options.executionId || null,
667
- { recoverOnKill: Boolean(options.onKillResume) }
668
- );
662
+ if (!options.deferCompletionWatcher) {
663
+ startDetachedDockerCompletionWatcher(
664
+ containerName,
665
+ cleanupPolicy,
666
+ options.logPath,
667
+ options.executionId || null,
668
+ { recoverOnKill: Boolean(options.onKillResume) }
669
+ );
670
+ }
669
671
 
670
672
  let message = `Command started in detached docker container: ${containerName}`;
671
673
  message += `\nContainer ID: ${containerId.substring(0, 12)}`;
@@ -94,7 +94,15 @@ function readDockerState(record) {
94
94
  if (opts.isolated !== 'docker' || !opts.sessionName) {
95
95
  return null;
96
96
  }
97
- return inspectDockerState(opts.sessionName);
97
+ const state = inspectDockerState(opts.sessionName);
98
+ if (
99
+ record.attempt &&
100
+ state?.startedAt &&
101
+ Date.parse(state.startedAt) < Date.parse(record.attempt.startedAt)
102
+ ) {
103
+ return null;
104
+ }
105
+ return state;
98
106
  }
99
107
 
100
108
  /**
@@ -506,12 +514,26 @@ function attachMemoryExhaustion(record, logTail) {
506
514
  * @returns {Object} Possibly updated execution record
507
515
  */
508
516
  function enrichDetachedStatus(record) {
509
- const logTail =
510
- record && record.logPath
511
- ? readLogTail(record.logPath, FATAL_MARKER_TAIL_BYTES)
512
- : null;
517
+ if (!record) {
518
+ return record;
519
+ }
520
+ const {
521
+ readAttemptActivity,
522
+ readAttemptLogTail,
523
+ } = require('./execution-attempt');
524
+ const logTail = record
525
+ ? readAttemptLogTail(record, FATAL_MARKER_TAIL_BYTES)
526
+ : null;
513
527
  const resolved = resolveDetachedStatus(record, logTail);
514
- return attachMemoryExhaustion(attachExitReason(resolved, logTail), logTail);
528
+ const enriched = attachMemoryExhaustion(
529
+ attachExitReason(resolved, logTail),
530
+ logTail
531
+ );
532
+ const copy = Object.assign(
533
+ Object.create(Object.getPrototypeOf(enriched)),
534
+ enriched
535
+ );
536
+ return readAttemptActivity(copy);
515
537
  }
516
538
 
517
539
  /**
@@ -693,6 +715,16 @@ function formatRecordAsText(record) {
693
715
  lines.push(`Stale Detected At: ${obj.staleDetectedAt}`);
694
716
  }
695
717
  lines.push(`Log Path: ${obj.logPath}`);
718
+ if (obj.attempt) {
719
+ lines.push(
720
+ `Attempt: ${obj.attempt.number}`,
721
+ `Attempt Started: ${obj.attempt.startedAt}`,
722
+ `Attempt Log Offset: ${obj.attempt.logOffset ?? 'N/A'}`,
723
+ `Launch Accepted: ${obj.attempt.launchAcceptedAt || 'N/A'}`,
724
+ `Watcher Attached: ${obj.attempt.watcherAttachedAt || 'N/A'}`,
725
+ `Last Output: ${obj.attempt.lastOutputAt || 'N/A'}`
726
+ );
727
+ }
696
728
 
697
729
  // Format options as nested list instead of JSON
698
730
  const optionEntries = Object.entries(obj.options || {}).filter(
@@ -274,7 +274,7 @@ describe('resumeExecution', () => {
274
274
  assert.strictEqual(watchers.length, 1);
275
275
  assert.strictEqual(watchers[0][0], 'box');
276
276
  assert.strictEqual(watchers[0][2], '/tmp/session.log');
277
- assert.strictEqual(store.saved.length, 1);
277
+ assert.strictEqual(store.saved.length, 2);
278
278
  assert.strictEqual(store.saved[0].uuid, record.uuid);
279
279
  assert.strictEqual(store.saved[0].status, 'executing');
280
280
  assert.strictEqual(store.saved[0].options.resumeCount, 1);
@@ -329,6 +329,7 @@ describe('resumeExecution', () => {
329
329
  const launches = [];
330
330
  const result = await resumeExecution(store, 'box', {
331
331
  probe: () => probeWith(SessionState.MISSING),
332
+ startWatcher: () => {},
332
333
  runIsolated: (backend, command, options) => {
333
334
  launches.push({ backend, command, options });
334
335
  return Promise.resolve({ success: true, containerId: 'newid' });
@@ -313,7 +313,11 @@ describe('issue #176: snapshot resume keeps the resource limits', () => {
313
313
  expect(saved[0].options.sessionName).toBe('box-resume-1');
314
314
  expect(result.output).toContain('--pids-limit=64');
315
315
  // A snapshot container has no recovery selector.
316
- expect(watchers[0][4]).toEqual({ recoverOnKill: false });
316
+ expect(watchers[0][4]).toEqual({
317
+ recoverOnKill: false,
318
+ since: saved[0].attempt.startedAt,
319
+ attemptNumber: saved[0].attempt.number,
320
+ });
317
321
  });
318
322
 
319
323
  it('shows the limits in [Isolation] lines and stores them as metadata', () => {
@@ -0,0 +1,546 @@
1
+ const { describe, it, expect, afterEach } = require('bun:test');
2
+ const fs = require('fs');
3
+ const os = require('os');
4
+ const path = require('path');
5
+ const { spawnSync } = require('child_process');
6
+ const { recoverKilledExecution } = require('../src/lib/execution-recovery');
7
+ const {
8
+ ExecutionRecord,
9
+ ExecutionStore,
10
+ } = require('../src/lib/execution-store');
11
+ const { resumeExecution } = require('../src/lib/execution-resume');
12
+ const { resumeAllExecutions } = require('../src/lib/execution-resume-all');
13
+ const { SessionState } = require('../src/lib/session-probe');
14
+ const { OutputObserver } = require('../src/lib/detached-output');
15
+ const {
16
+ activityPath,
17
+ readAttemptLogTail,
18
+ } = require('../src/lib/execution-attempt');
19
+ const {
20
+ buildDetachedDockerCompletionScript,
21
+ DOCKER_CONTAINER_CLEANUP_POLICY,
22
+ } = require('../src/lib/docker-cleanup');
23
+ const { finalizeDetachedExecution } = require('../src/lib/detached-finalize');
24
+ const {
25
+ enrichDetachedStatus,
26
+ formatRecord,
27
+ formatRecordList,
28
+ } = require('../src/lib/status-formatter');
29
+
30
+ const directories = [];
31
+ afterEach(() => {
32
+ for (const directory of directories.splice(0)) {
33
+ fs.rmSync(directory, { recursive: true, force: true });
34
+ }
35
+ });
36
+
37
+ function fixture() {
38
+ const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'resume-187-'));
39
+ directories.push(directory);
40
+ const logPath = path.join(directory, 'execution.log');
41
+ const oldLog =
42
+ 'previous output: 🐝\nFATAL ERROR: Reached heap limit Allocation failed - JavaScript heap out of memory\n' +
43
+ '\n==================================================\nFinished: 2026-01-01 00:01:00.000\nExit Code: 137\n';
44
+ fs.writeFileSync(logPath, oldLog);
45
+ const store = new ExecutionStore({ appFolder: directory, useLinks: false });
46
+ const record = new ExecutionRecord({
47
+ command: 'worker',
48
+ logPath,
49
+ startTime: '2026-01-01T00:00:00.000Z',
50
+ status: 'executed',
51
+ exitCode: 137,
52
+ endTime: '2026-01-01T00:01:00.000Z',
53
+ endTimeSource: 'docker-finished-at',
54
+ observedAt: '2026-01-01T00:01:01.000Z',
55
+ staleDetectedAt: '2026-01-01T00:01:02.000Z',
56
+ containerStartedAt: '2026-01-01T00:00:00.000Z',
57
+ exitReason: 'cgroup-oom-killer',
58
+ oomKilled: true,
59
+ memoryExhausted: true,
60
+ memoryExhaustedReason: 'cgroup-oom-killer',
61
+ cgroupMemory: {
62
+ limitBytes: 128,
63
+ peakBytes: 128,
64
+ oomEvents: 1,
65
+ oomKills: 1,
66
+ },
67
+ options: {
68
+ isolated: 'docker',
69
+ isolationMode: 'detached',
70
+ sessionName: 'issue-187-container-that-does-not-exist',
71
+ containerError: 'previous error',
72
+ },
73
+ });
74
+ store.save(record);
75
+ return { store, record, logPath, oldLog };
76
+ }
77
+
78
+ const stopped = () => ({ alive: false, state: SessionState.STOPPED });
79
+ const accepted = () => ({ success: true, stdout: 'container-id\n', status: 0 });
80
+
81
+ describe('issue #187: explicit resume attempts', () => {
82
+ it('reattaches the current attempt without enabling kill recovery for snapshot commands', async () => {
83
+ for (const mode of ['docker-start', 'docker-snapshot', 'relaunch']) {
84
+ const command = mode === 'docker-snapshot' ? 'replacement' : undefined;
85
+ const { store, record } = fixture();
86
+ record.options.onKillResume = 1;
87
+ store.save(record);
88
+ await resumeExecution(store, record.uuid, {
89
+ command,
90
+ probe:
91
+ mode === 'relaunch'
92
+ ? () => ({ alive: false, state: SessionState.MISSING })
93
+ : stopped,
94
+ runner: accepted,
95
+ runIsolated: () => ({ success: true, containerId: 'new-container' }),
96
+ startWatcher: (_name, _policy, _log, _uuid, options) => {
97
+ expect(options.recoverOnKill).toBe(mode !== 'docker-snapshot');
98
+ },
99
+ });
100
+ const current = store.get(record.uuid);
101
+ let options;
102
+ resumeAllExecutions(store, {
103
+ probe: () => ({
104
+ alive: true,
105
+ backend: 'docker',
106
+ sessionName: current.options.sessionName,
107
+ }),
108
+ startWatcher: (_name, _policy, _log, _uuid, watcherOptions) => {
109
+ options = watcherOptions;
110
+ },
111
+ });
112
+ expect(options.attemptNumber).toBe(2);
113
+ expect(options.since).toBe(current.attempt.startedAt);
114
+ expect(options.recoverOnKill).toBe(mode !== 'docker-snapshot');
115
+ }
116
+ });
117
+ it('merges attachment metadata without reviving a completed or newer attempt', async () => {
118
+ const { store, record } = fixture();
119
+ await resumeExecution(store, record.uuid, {
120
+ probe: stopped,
121
+ runner: accepted,
122
+ startWatcher: () => {},
123
+ });
124
+ const current = store.get(record.uuid);
125
+ current.status = 'executed';
126
+ current.exitCode = 0;
127
+ store.save(current);
128
+ const patched = store.patchAttempt(record.uuid, 2, {
129
+ watcherAttachedAt: '2026-10-06T09:00:00Z',
130
+ });
131
+ expect(patched.status).toBe('executed');
132
+ expect(patched.exitCode).toBe(0);
133
+ expect(
134
+ store.patchAttempt(record.uuid, 1, { watcherError: 'stale watcher' })
135
+ ).toBeNull();
136
+ expect(store.get(record.uuid).attempt.watcherError).toBeUndefined();
137
+ expect(enrichDetachedStatus(null)).toBeNull();
138
+ });
139
+ it('keeps boundaries and evidence when an explicit resume is automatically recovered', async () => {
140
+ const { store, record, logPath } = fixture();
141
+ record.options.onKillResume = 1;
142
+ store.save(record);
143
+ await resumeExecution(store, record.uuid, {
144
+ probe: stopped,
145
+ runner: accepted,
146
+ startWatcher: () => {},
147
+ });
148
+ const current = store.get(record.uuid);
149
+ const outcome = recoverKilledExecution({
150
+ store,
151
+ executionId: record.uuid,
152
+ exitCode: 137,
153
+ oomKilled: true,
154
+ startedAt: current.attempt.startedAt,
155
+ finishedAt: new Date().toISOString(),
156
+ cgroupMemory: '128 128 1 1',
157
+ runner: accepted,
158
+ startWatcher: (_name, _policy, _log, _uuid, options) => {
159
+ expect(options.attemptNumber).toBe(3);
160
+ expect(store.get(record.uuid).attempt.number).toBe(3);
161
+ },
162
+ });
163
+ expect(outcome.recovered).toBe(true);
164
+ const saved = store.get(record.uuid);
165
+ expect(saved.attemptHistory.map((a) => a.exitCode)).toEqual([137, 137]);
166
+ expect(saved.attemptHistory[1].cgroupMemory.oomKills).toBe(1);
167
+ expect(saved.cgroupMemory).toBeUndefined();
168
+ expect(saved.attempt.lastOutputAt).toBeNull();
169
+ expect(saved.attempt.logOffset).toBeGreaterThan(current.attempt.logOffset);
170
+ expect(fs.readFileSync(logPath, 'utf8')).toContain('automatic-recovery');
171
+ });
172
+
173
+ it('runs the timestamped output capture and finalizer through the real watcher shell', async () => {
174
+ if (process.platform === 'win32') {
175
+ return;
176
+ }
177
+ const { store, record, logPath } = fixture();
178
+ await resumeExecution(store, record.uuid, {
179
+ probe: stopped,
180
+ runner: accepted,
181
+ startWatcher: () => {},
182
+ });
183
+ const current = store.get(record.uuid);
184
+ const directory = path.dirname(logPath);
185
+ const bin = path.join(directory, 'bin');
186
+ fs.mkdirSync(bin);
187
+ const at = new Date().toISOString();
188
+ fs.writeFileSync(
189
+ path.join(bin, 'docker'),
190
+ `#!/bin/sh
191
+ case "$1" in
192
+ logs) printf '%s command output 🐝\\n' "$TEST_OUTPUT_AT" ;;
193
+ wait) echo 0 ;;
194
+ inspect) case "$3" in
195
+ *State.Running*) echo false ;;
196
+ *State.ExitCode*) echo "0 false $TEST_OUTPUT_AT $TEST_OUTPUT_AT" ;;
197
+ *State.Error*) echo '' ;;
198
+ *) exit 1 ;;
199
+ esac ;;
200
+ esac
201
+ `,
202
+ { mode: 0o755 }
203
+ );
204
+ const script = buildDetachedDockerCompletionScript(
205
+ current.options.sessionName,
206
+ DOCKER_CONTAINER_CLEANUP_POLICY.KEEP,
207
+ logPath,
208
+ record.uuid,
209
+ { since: current.attempt.startedAt, attemptNumber: 2 }
210
+ );
211
+ const output = spawnSync('/bin/sh', ['-c', script], {
212
+ env: {
213
+ ...process.env,
214
+ PATH: `${bin}${path.delimiter}${process.env.PATH}`,
215
+ START_APP_FOLDER: directory,
216
+ TEST_OUTPUT_AT: at,
217
+ START_COMMAND_CGROUP_ROOT: path.join(directory, 'no-cgroup'),
218
+ START_COMMAND_PROC_ROOT: path.join(directory, 'no-proc'),
219
+ },
220
+ encoding: 'utf8',
221
+ maxBuffer: 1024 * 1024,
222
+ });
223
+ expect(output.status).toBe(0);
224
+ const saved = store.get(record.uuid);
225
+ expect(saved.status).toBe('executed');
226
+ expect(saved.exitCode).toBe(0);
227
+ expect(saved.attempt.lastOutputAt).toBe(at);
228
+ expect(saved.cgroupMemory).toBeUndefined();
229
+ expect(fs.readFileSync(logPath, 'utf8')).toContain('command output 🐝');
230
+ }, 10000);
231
+
232
+ it('archives terminal evidence and resets every attempt-local field', async () => {
233
+ const { store, record, oldLog } = fixture();
234
+ const outcome = await resumeExecution(store, record.uuid, {
235
+ probe: stopped,
236
+ runner: accepted,
237
+ startWatcher: () => {},
238
+ });
239
+ expect(outcome.success).toBe(true);
240
+ const saved = store.get(record.uuid);
241
+ expect(saved.status).toBe('executing');
242
+ expect(saved.exitCode).toBeNull();
243
+ for (const field of [
244
+ 'endTimeSource',
245
+ 'observedAt',
246
+ 'staleDetectedAt',
247
+ 'containerStartedAt',
248
+ 'exitReason',
249
+ 'oomKilled',
250
+ 'memoryExhausted',
251
+ 'memoryExhaustedReason',
252
+ 'cgroupMemory',
253
+ ]) {
254
+ expect(saved[field]).toBeUndefined();
255
+ }
256
+ expect(saved.options.containerError).toBeUndefined();
257
+ expect(saved.attempt.number).toBe(2);
258
+ expect(saved.attempt.logOffset).toBe(Buffer.byteLength(oldLog));
259
+ expect(saved.attempt.startedAt).not.toBe(record.startTime);
260
+ expect(saved.attempt.launchAcceptedAt).toBeDefined();
261
+ expect(saved.attempt.lastOutputAt).toBeNull();
262
+ expect(saved.attemptHistory[0].exitCode).toBe(137);
263
+ expect(saved.attemptHistory[0].memoryExhausted).toBe(true);
264
+ expect(saved.attemptHistory[0].cgroupMemory.oomKills).toBe(1);
265
+ });
266
+
267
+ it('does not resolve the previous footer or memory marker as current evidence', async () => {
268
+ const { store, record } = fixture();
269
+ await resumeExecution(store, record.uuid, {
270
+ probe: stopped,
271
+ runner: accepted,
272
+ startWatcher: () => {},
273
+ });
274
+ const current = store.get(record.uuid);
275
+ const enriched = enrichDetachedStatus(current);
276
+ expect(enriched.status).toBe('executing');
277
+ expect(enriched.exitCode).toBeNull();
278
+ expect(enriched.memoryExhausted).toBeUndefined();
279
+ const status = JSON.parse(formatRecord(enriched, 'json'));
280
+ const list = JSON.parse(formatRecordList([enriched], 'json'));
281
+ expect(status.attempt.logOffset).toBe(current.attempt.logOffset);
282
+ expect(list.executions[0].attempt.startedAt).toBe(
283
+ current.attempt.startedAt
284
+ );
285
+ });
286
+
287
+ it('persists the new attempt before a watcher can finalize an immediate exit', async () => {
288
+ const { store, record } = fixture();
289
+ await resumeExecution(store, record.uuid, {
290
+ probe: stopped,
291
+ runner: accepted,
292
+ startWatcher: (name, policy, logPath, uuid, options) => {
293
+ expect(store.get(uuid).status).toBe('executing');
294
+ expect(options.since).toBe(store.get(uuid).attempt.startedAt);
295
+ const outcome = finalizeDetachedExecution({
296
+ store,
297
+ executionId: uuid,
298
+ exitCode: 0,
299
+ oomKilled: false,
300
+ startedAt: new Date().toISOString(),
301
+ finishedAt: new Date().toISOString(),
302
+ attemptNumber: store.get(uuid).attempt.number,
303
+ });
304
+ expect(outcome.updated).toBe(true);
305
+ },
306
+ });
307
+ expect(store.get(record.uuid).status).toBe('executed');
308
+ expect(store.get(record.uuid).exitCode).toBe(0);
309
+ });
310
+
311
+ it('records a structured boundary before launch and accepted launch separately from output', async () => {
312
+ const { store, record, logPath, oldLog } = fixture();
313
+ await resumeExecution(store, record.uuid, {
314
+ probe: stopped,
315
+ runner: () => {
316
+ expect(fs.readFileSync(logPath, 'utf8').slice(oldLog.length)).toContain(
317
+ 'resume-started'
318
+ );
319
+ return accepted();
320
+ },
321
+ startWatcher: () => {},
322
+ });
323
+ const log = fs.readFileSync(logPath, 'utf8');
324
+ expect(log).toContain('launch-accepted');
325
+ expect(log).toContain('watcher-attached');
326
+ expect(store.get(record.uuid).attempt.lastOutputAt).toBeNull();
327
+ });
328
+
329
+ it('preserves the stopped record when launch fails and logs that failure', async () => {
330
+ const { store, record, logPath } = fixture();
331
+ const outcome = await resumeExecution(store, record.uuid, {
332
+ probe: stopped,
333
+ runner: () => ({ success: false, stderr: 'launch rejected', status: 1 }),
334
+ startWatcher: () => {
335
+ throw new Error('must not attach');
336
+ },
337
+ });
338
+ expect(outcome.success).toBe(false);
339
+ expect(store.get(record.uuid).exitCode).toBe(137);
340
+ expect(store.get(record.uuid).memoryExhausted).toBe(true);
341
+ expect(fs.readFileSync(logPath, 'utf8')).toContain('launch-failed');
342
+ });
343
+
344
+ it('records previous and new container names for a snapshot resume', async () => {
345
+ const { store, record, logPath } = fixture();
346
+ const result = await resumeExecution(store, record.uuid, {
347
+ command: 'replacement',
348
+ probe: stopped,
349
+ runner: accepted,
350
+ startWatcher: () => {},
351
+ });
352
+ expect(result.success).toBe(true);
353
+ const saved = store.get(record.uuid);
354
+ expect(saved.attempt.sessionName).toBe(
355
+ `${record.options.sessionName}-resume-1`
356
+ );
357
+ expect(saved.attempt.previousSessionName).toBe(record.options.sessionName);
358
+ expect(saved.attemptHistory[0].command).toBe('worker');
359
+ expect(fs.readFileSync(logPath, 'utf8')).toContain(
360
+ saved.attempt.sessionName
361
+ );
362
+ });
363
+
364
+ it('defers the relaunch watcher until the accepted attempt is saved', async () => {
365
+ const { store, record } = fixture();
366
+ let attached = false;
367
+ const result = await resumeExecution(store, record.uuid, {
368
+ probe: () => ({ alive: false, state: SessionState.MISSING }),
369
+ runIsolated: (backend, command, options) => {
370
+ expect(options.deferCompletionWatcher).toBe(true);
371
+ expect(store.get(record.uuid).status).toBe('executed');
372
+ return { success: true, containerId: 'replacement-container' };
373
+ },
374
+ startWatcher: () => {
375
+ attached = true;
376
+ expect(store.get(record.uuid).status).toBe('executing');
377
+ },
378
+ });
379
+ expect(result.success).toBe(true);
380
+ expect(attached).toBe(true);
381
+ expect(store.get(record.uuid).attempt.mode).toBe('relaunch');
382
+ });
383
+
384
+ it('reports watcher failure while preserving the accepted launch', async () => {
385
+ const { store, record, logPath } = fixture();
386
+ const result = await resumeExecution(store, record.uuid, {
387
+ probe: stopped,
388
+ runner: accepted,
389
+ startWatcher: () => {
390
+ throw new Error('cannot spawn watcher');
391
+ },
392
+ });
393
+ expect(result.success).toBe(false);
394
+ expect(result.error).toContain('Launch accepted');
395
+ const saved = store.get(record.uuid);
396
+ expect(saved.status).toBe('executing');
397
+ expect(saved.attempt.launchAcceptedAt).toBeDefined();
398
+ expect(saved.attempt.watcherAttachedAt).toBeNull();
399
+ expect(saved.attempt.watcherError).toBe('cannot spawn watcher');
400
+ expect(fs.readFileSync(logPath, 'utf8')).toContain(
401
+ 'watcher-attachment-failed'
402
+ );
403
+ });
404
+
405
+ it('rejects a finalizer carrying evidence from a previous attempt', async () => {
406
+ const { store, record } = fixture();
407
+ await resumeExecution(store, record.uuid, {
408
+ probe: stopped,
409
+ runner: accepted,
410
+ startWatcher: () => {},
411
+ });
412
+ const result = finalizeDetachedExecution({
413
+ store,
414
+ executionId: record.uuid,
415
+ attemptNumber: 1,
416
+ exitCode: 137,
417
+ oomKilled: true,
418
+ startedAt: record.startTime,
419
+ finishedAt: record.endTime,
420
+ });
421
+ expect(result.reason).toBe('stale-attempt');
422
+ expect(store.get(record.uuid).exitCode).toBeNull();
423
+ });
424
+
425
+ it('does not reuse a previous fatal marker when the new run exits 139', async () => {
426
+ const { store, record, logPath } = fixture();
427
+ await resumeExecution(store, record.uuid, {
428
+ probe: stopped,
429
+ runner: accepted,
430
+ startWatcher: () => {},
431
+ });
432
+ const current = store.get(record.uuid);
433
+ const at = new Date().toISOString();
434
+ finalizeDetachedExecution({
435
+ store,
436
+ executionId: record.uuid,
437
+ attemptNumber: current.attempt.number,
438
+ exitCode: 139,
439
+ oomKilled: false,
440
+ startedAt: at,
441
+ finishedAt: at,
442
+ });
443
+ const terminal = enrichDetachedStatus(store.get(record.uuid));
444
+ expect(terminal.memoryExhausted).toBeUndefined();
445
+ expect(terminal.cgroupMemory).toBeUndefined();
446
+ expect(terminal.exitReason).not.toBe('cgroup-oom-killer');
447
+ expect(fs.readFileSync(logPath, 'utf8')).toContain('"event":"terminal"');
448
+ });
449
+
450
+ it('keeps successive attempts and their terminal evidence through store round trips', async () => {
451
+ const { store, record } = fixture();
452
+ for (let i = 0; i < 2; i++) {
453
+ await resumeExecution(store, record.uuid, {
454
+ probe: stopped,
455
+ runner: accepted,
456
+ startWatcher: () => {},
457
+ });
458
+ const current = store.get(record.uuid);
459
+ const at = new Date().toISOString();
460
+ finalizeDetachedExecution({
461
+ store,
462
+ executionId: record.uuid,
463
+ attemptNumber: current.attempt.number,
464
+ exitCode: i,
465
+ oomKilled: false,
466
+ startedAt: at,
467
+ finishedAt: at,
468
+ });
469
+ }
470
+ const saved = store.get(record.uuid);
471
+ expect(saved.attempt.number).toBe(3);
472
+ expect(saved.attemptHistory.map((a) => a.exitCode)).toEqual([137, 0]);
473
+ expect(saved.attemptHistory[0].cgroupMemory.oomKills).toBe(1);
474
+ });
475
+
476
+ it('reports output time only after fresh timestamped command output, even without a newline', async () => {
477
+ const { store, record, logPath } = fixture();
478
+ await resumeExecution(store, record.uuid, {
479
+ probe: stopped,
480
+ runner: accepted,
481
+ startWatcher: () => {},
482
+ });
483
+ const current = store.get(record.uuid);
484
+ const observer = new OutputObserver(
485
+ logPath,
486
+ current.attempt.number,
487
+ current.attempt.startedAt
488
+ );
489
+ observer.write(
490
+ Buffer.from(
491
+ 'Docker administrative error\n2026-01-01T00:00:00Z old output\n'
492
+ )
493
+ );
494
+ expect(
495
+ enrichDetachedStatus(store.get(record.uuid)).attempt.lastOutputAt
496
+ ).toBeNull();
497
+ const at = new Date().toISOString();
498
+ const output = Buffer.from(`${at} quiet run emits 🐝 without a newline`);
499
+ observer.write(output.subarray(0, 10));
500
+ observer.write(output.subarray(10));
501
+ expect(
502
+ enrichDetachedStatus(store.get(record.uuid)).attempt.lastOutputAt
503
+ ).toBe(at);
504
+ expect(fs.readFileSync(logPath).includes(output)).toBe(true);
505
+ fs.appendFileSync(
506
+ logPath,
507
+ '\n[Start Command Lifecycle] administrative activity\n'
508
+ );
509
+ expect(
510
+ enrichDetachedStatus(store.get(record.uuid)).attempt.lastOutputAt
511
+ ).toBe(at);
512
+ fs.writeFileSync(activityPath(current), '2026-01-01T00:00:00Z');
513
+ expect(
514
+ enrichDetachedStatus(store.get(record.uuid)).attempt.lastOutputAt
515
+ ).toBeNull();
516
+ });
517
+
518
+ it('bounds reads and never scans prior evidence after log truncation', async () => {
519
+ const { store, record, logPath } = fixture();
520
+ await resumeExecution(store, record.uuid, {
521
+ probe: stopped,
522
+ runner: accepted,
523
+ startWatcher: () => {},
524
+ });
525
+ const current = store.get(record.uuid);
526
+ fs.appendFileSync(logPath, `\n${'x'.repeat(70 * 1024)}\nfresh\n`);
527
+ expect(readAttemptLogTail(current, 64 * 1024).length).toBeLessThanOrEqual(
528
+ 64 * 1024
529
+ );
530
+ fs.writeFileSync(logPath, 'old footer truncated');
531
+ expect(readAttemptLogTail(current, 64 * 1024)).toBe('');
532
+ });
533
+
534
+ it('builds a watcher that filters historical output and carries the attempt identity', () => {
535
+ const script = buildDetachedDockerCompletionScript(
536
+ 'box',
537
+ DOCKER_CONTAINER_CLEANUP_POLICY.KEEP,
538
+ '/tmp/log with spaces',
539
+ 'uuid',
540
+ { since: '2026-10-06T08:00:00Z', attemptNumber: 2 }
541
+ );
542
+ expect(script).toContain('docker logs -f --timestamps --since');
543
+ expect(script).toContain('detached-output.js');
544
+ expect(script).toContain("'2' >/dev/null");
545
+ });
546
+ });