rollbridge 0.1.49 → 0.1.55
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +5 -0
- package/README.md +5 -0
- package/changelog.d/20260909120000-velocious-testing.md +1 -0
- package/docs/cli.md +7 -1
- package/docs/generation-deployment-contract.md +9 -0
- package/eslint.config.js +8 -0
- package/package.json +3 -2
- package/src/cli.js +10 -2
- package/src/daemon.js +102 -12
- package/src/process-guardian.js +5 -1
- package/src/release-group.js +48 -1
- package/test/completion.test.js +18 -16
- package/test/config-examples.test.js +16 -17
- package/test/config-path.test.js +10 -11
- package/test/config-validation.test.js +163 -167
- package/test/control-protocol.test.js +75 -14
- package/test/daemon-bootstrap.test.js +104 -104
- package/test/daemon-runtime.test.js +17 -26
- package/test/doctor.test.js +51 -49
- package/test/event-log.test.js +13 -11
- package/test/guardian-client.test.js +160 -145
- package/test/health.test.js +6 -4
- package/test/logs.test.js +23 -17
- package/test/managed-process.test.js +96 -91
- package/test/owner-recovery.test.js +254 -239
- package/test/owner-replacement.test.js +228 -223
- package/test/package-metadata.test.js +48 -39
- package/test/port-allocator.test.js +13 -16
- package/test/predeploy-cleanup.test.js +12 -10
- package/test/process-memory.test.js +17 -15
- package/test/proxy.test.js +10 -8
- package/test/recover.test.js +30 -23
- package/test/release-group.test.js +16 -17
- package/test/release-retention.test.js +10 -8
- package/test/release-runtime-retention.test.js +31 -39
- package/test/rollbridge.test.js +388 -395
- package/test/shutdown-completion.test.js +51 -51
- package/test/state-store.test.js +10 -8
- package/test/system-ids.test.js +15 -13
package/test/rollbridge.test.js
CHANGED
|
@@ -1,13 +1,12 @@
|
|
|
1
1
|
// @ts-check
|
|
2
2
|
|
|
3
|
-
import assert from "node:assert/strict"
|
|
4
3
|
import {spawn} from "node:child_process"
|
|
5
4
|
import {once} from "node:events"
|
|
6
5
|
import fs from "node:fs/promises"
|
|
7
6
|
import net from "node:net"
|
|
8
7
|
import os from "node:os"
|
|
9
8
|
import path from "node:path"
|
|
10
|
-
import test from "
|
|
9
|
+
import {describe, expect, test} from "@velocious/testing"
|
|
11
10
|
import {fileURLToPath, pathToFileURL} from "node:url"
|
|
12
11
|
import RollbridgeDaemon from "../src/daemon.js"
|
|
13
12
|
import {normalizeConfig} from "../src/config.js"
|
|
@@ -15,6 +14,8 @@ import {sendControlCommand} from "../src/control-client.js"
|
|
|
15
14
|
import {liveProcesses, readState, writeState} from "../src/state-store.js"
|
|
16
15
|
import {runCli} from "../src/cli.js"
|
|
17
16
|
|
|
17
|
+
describe("rollbridge", () => {
|
|
18
|
+
|
|
18
19
|
const currentDir = path.dirname(fileURLToPath(import.meta.url))
|
|
19
20
|
const binPath = path.join(currentDir, "..", "bin", "rollbridge")
|
|
20
21
|
const dependentAppPath = path.join(currentDir, "fixtures", "dependent-app.js")
|
|
@@ -22,6 +23,7 @@ const dummyAppPath = path.join(currentDir, "fixtures", "dummy-app.js")
|
|
|
22
23
|
const memoryHogPath = path.join(currentDir, "fixtures", "memory-hog.js")
|
|
23
24
|
const serviceAppPath = path.join(currentDir, "fixtures", "service-app.js")
|
|
24
25
|
const singletonAppPath = path.join(currentDir, "fixtures", "singleton-app.js")
|
|
26
|
+
const linuxTest = process.platform === "linux" ? test : test.skip
|
|
25
27
|
|
|
26
28
|
test("a nonBlockingDrain worker stops immediately while its release is still draining", async () => {
|
|
27
29
|
const fixture = await createFixture({nonBlockingDrainWorker: true})
|
|
@@ -47,9 +49,9 @@ test("a nonBlockingDrain worker stops immediately while its release is still dra
|
|
|
47
49
|
|
|
48
50
|
// The release is still draining (the WebSocket is held) and its proxied process is still
|
|
49
51
|
// serving, but the worker has already drained.
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
52
|
+
expect(v1.state).toBe("draining")
|
|
53
|
+
expect(v1.processes.find((processStatus) => processStatus.id === "web")?.state).toBe("running")
|
|
54
|
+
expect(v1.processes.find((processStatus) => processStatus.id === "worker")?.state).toBe("stopped")
|
|
53
55
|
} finally {
|
|
54
56
|
if (socket) socket.close()
|
|
55
57
|
await daemon.shutdown()
|
|
@@ -63,16 +65,16 @@ test("deploy switches new HTTP traffic while old WebSockets drain", async () =>
|
|
|
63
65
|
|
|
64
66
|
try {
|
|
65
67
|
await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
|
|
66
|
-
|
|
68
|
+
expect(await fetchText(daemon, "/release")).toBe("v1")
|
|
67
69
|
|
|
68
70
|
const websocket = await openWebSocket(daemon)
|
|
69
71
|
|
|
70
72
|
await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
|
|
71
|
-
|
|
73
|
+
expect(await fetchText(daemon, "/release")).toBe("v2")
|
|
72
74
|
|
|
73
75
|
const drainingRelease = statusRelease(daemon, "v1")
|
|
74
|
-
|
|
75
|
-
|
|
76
|
+
expect(drainingRelease.state).toBe("draining")
|
|
77
|
+
expect(drainingRelease.connections.websocket).toBe(1)
|
|
76
78
|
|
|
77
79
|
websocket.close()
|
|
78
80
|
await waitFor(async () => statusRelease(daemon, "v1").state === "stopped")
|
|
@@ -89,13 +91,10 @@ test("failed health check leaves the previous release active", async () => {
|
|
|
89
91
|
try {
|
|
90
92
|
await daemon.deploy({releaseId: "good", releasePath: fixture.root, revision: "good"})
|
|
91
93
|
|
|
92
|
-
await
|
|
93
|
-
() => daemon.deploy({releaseId: "bad", releasePath: fixture.root, revision: "bad"}),
|
|
94
|
-
/Health check failed/
|
|
95
|
-
)
|
|
94
|
+
await expect(daemon.deploy({releaseId: "bad", releasePath: fixture.root, revision: "bad"})).rejects.toThrow(/Health check failed/)
|
|
96
95
|
|
|
97
|
-
|
|
98
|
-
|
|
96
|
+
expect(await fetchText(daemon, "/release")).toBe("good")
|
|
97
|
+
expect(daemon.status().activeReleaseId).toBe("good")
|
|
99
98
|
} finally {
|
|
100
99
|
await daemon.shutdown()
|
|
101
100
|
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
@@ -128,8 +127,8 @@ test("deploy reloads process config and retires the previous worker with the ref
|
|
|
128
127
|
await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
|
|
129
128
|
await waitFor(() => statusRelease(daemon, "v1").processes.find((processStatus) => processStatus.id === "worker")?.state === "stopped", 1000)
|
|
130
129
|
|
|
131
|
-
|
|
132
|
-
|
|
130
|
+
expect(daemon.config.processes.find((processConfig) => processConfig.id === "worker")?.gracefulStopMs).toBe(50)
|
|
131
|
+
expect(await fetchText(daemon, "/release")).toBe("v2")
|
|
133
132
|
} finally {
|
|
134
133
|
await daemon.shutdown()
|
|
135
134
|
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
@@ -149,12 +148,9 @@ test("deploy rejects a reloaded config that changes the running proxy", async ()
|
|
|
149
148
|
proxy: {...fixture.config.proxy, host: "0.0.0.0"}
|
|
150
149
|
}), fixture.root)
|
|
151
150
|
|
|
152
|
-
await
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
)
|
|
156
|
-
assert.equal(daemon.status().activeReleaseId, "v1")
|
|
157
|
-
assert.equal(await fetchText(daemon, "/release"), "v1")
|
|
151
|
+
await expect(daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})).rejects.toThrow(/proxy\.host.*restart the Rollbridge daemon/)
|
|
152
|
+
expect(daemon.status().activeReleaseId).toBe("v1")
|
|
153
|
+
expect(await fetchText(daemon, "/release")).toBe("v1")
|
|
158
154
|
} finally {
|
|
159
155
|
await daemon.shutdown()
|
|
160
156
|
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
@@ -178,14 +174,11 @@ test("a failed deploy does not adopt reloaded process config", async () => {
|
|
|
178
174
|
})
|
|
179
175
|
|
|
180
176
|
await writeConfigFile(failingConfig, fixture.root)
|
|
181
|
-
await
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
)
|
|
185
|
-
|
|
186
|
-
assert.equal(daemon.config.processes.find((processConfig) => processConfig.id === "web")?.health?.path, "/ping")
|
|
187
|
-
assert.equal(daemon.status().activeReleaseId, "v1")
|
|
188
|
-
assert.equal(await fetchText(daemon, "/release"), "v1")
|
|
177
|
+
await expect(daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})).rejects.toThrow(/Health check failed/)
|
|
178
|
+
|
|
179
|
+
expect(daemon.config.processes.find((processConfig) => processConfig.id === "web")?.health?.path).toBe("/ping")
|
|
180
|
+
expect(daemon.status().activeReleaseId).toBe("v1")
|
|
181
|
+
expect(await fetchText(daemon, "/release")).toBe("v1")
|
|
189
182
|
} finally {
|
|
190
183
|
await daemon.shutdown()
|
|
191
184
|
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
@@ -202,11 +195,11 @@ test("wildcard proxy bind host targets release processes through loopback", asyn
|
|
|
202
195
|
const status = daemon.status()
|
|
203
196
|
const release = statusRelease(daemon, "v1")
|
|
204
197
|
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
198
|
+
if (!daemon.activeRelease) throw new Error("expected active release")
|
|
199
|
+
expect(status.proxy.host).toBe("0.0.0.0")
|
|
200
|
+
expect(status.proxy.upstreamHost).toBe("127.0.0.1")
|
|
201
|
+
expect(daemon.activeRelease.proxyTarget().target).toBe(`http://127.0.0.1:${release.ports.web}`)
|
|
202
|
+
expect(await fetchText(daemon, "/release")).toBe("v1")
|
|
210
203
|
} finally {
|
|
211
204
|
await daemon.shutdown()
|
|
212
205
|
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
@@ -225,27 +218,24 @@ test("failed release startup logs process output and cleanup status", async () =
|
|
|
225
218
|
await daemon.start()
|
|
226
219
|
|
|
227
220
|
try {
|
|
228
|
-
await
|
|
229
|
-
() => daemon.deploy({releaseId: "bad", releasePath: fixture.root, revision: "bad"}),
|
|
230
|
-
/Health check failed/
|
|
231
|
-
)
|
|
221
|
+
await expect(daemon.deploy({releaseId: "bad", releasePath: fixture.root, revision: "bad"})).rejects.toThrow(/Health check failed/)
|
|
232
222
|
|
|
233
223
|
const processStatusLog = logs.find((entry) => entry.message === "release startup process status" && entry.data?.phase === "before cleanup" && entry.data?.processId === "web")
|
|
234
224
|
const cleanupProcessStatusLog = logs.find((entry) => entry.message === "release startup process status" && entry.data?.phase === "after cleanup" && entry.data?.processId === "web")
|
|
235
225
|
const handoffServiceStatusLog = logs.find((entry) => entry.message === "release startup process status" && entry.data?.phase === "after cleanup" && entry.data?.processId === "beacon")
|
|
236
226
|
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
227
|
+
if (!processStatusLog) throw new Error("expected failed web process diagnostics to be logged")
|
|
228
|
+
if (!processStatusLog.data) throw new Error("expected diagnostic data")
|
|
229
|
+
if (!Array.isArray(processStatusLog.data.logs)) throw new Error("expected retained process output in diagnostics")
|
|
230
|
+
expect(processStatusLog.data.logs.some((entry) => typeof entry === "object" && entry && "line" in entry && entry.line === "startup stdout")).toBeTruthy()
|
|
231
|
+
expect(processStatusLog.data.logs.some((entry) => typeof entry === "object" && entry && "line" in entry && entry.line === "startup stderr")).toBeTruthy()
|
|
232
|
+
expect(processStatusLog.data.state).toBe("running")
|
|
233
|
+
if (!cleanupProcessStatusLog) throw new Error("expected failed web cleanup diagnostics to be logged")
|
|
234
|
+
expect(cleanupProcessStatusLog.data?.state).toBe("stopped")
|
|
235
|
+
expect(cleanupProcessStatusLog.data?.exitSignal).toBe("SIGTERM")
|
|
236
|
+
if (!handoffServiceStatusLog) throw new Error("expected handoff service cleanup diagnostics to be logged")
|
|
237
|
+
expect(handoffServiceStatusLog.data?.state).toBe("stopped")
|
|
238
|
+
expect(handoffServiceStatusLog.data?.exitSignal).toBe("SIGTERM")
|
|
249
239
|
} finally {
|
|
250
240
|
await daemon.shutdown()
|
|
251
241
|
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
@@ -270,8 +260,8 @@ test("singleton processes restart without overlap during deploy", async () => {
|
|
|
270
260
|
|
|
271
261
|
const status = daemon.status()
|
|
272
262
|
|
|
273
|
-
|
|
274
|
-
|
|
263
|
+
expect(status.singletons.length).toBe(1)
|
|
264
|
+
expect(status.singletons[0].process.state).toBe("running")
|
|
275
265
|
} finally {
|
|
276
266
|
await daemon.shutdown()
|
|
277
267
|
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
@@ -319,9 +309,12 @@ test("candidate activation quiesces the old jobs generation before a blocked sin
|
|
|
319
309
|
await singletonReplacementBlocked
|
|
320
310
|
await Promise.resolve()
|
|
321
311
|
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
312
|
+
// Candidate traffic must already be active.
|
|
313
|
+
expect(daemon.status().activeReleaseId).toBe("v2")
|
|
314
|
+
// Deploy must remain pending on singleton replacement.
|
|
315
|
+
expect(deploySettled).toBe(false)
|
|
316
|
+
// Old jobs-main must quiesce before singleton replacement completes.
|
|
317
|
+
expect(await fs.readFile(fixture.serviceQuietPath, "utf8")).toBe("v1\n")
|
|
325
318
|
|
|
326
319
|
await fs.writeFile(singletonGatePath, "continue\n")
|
|
327
320
|
singletonGateReleased = true
|
|
@@ -350,7 +343,7 @@ test("a failed singleton replacement surfaces the error after stopping the old s
|
|
|
350
343
|
await waitFor(async () => (await processEvents(fixture.singletonLogPath)).some((event) => event.event === "start" && event.releaseId === "v1"))
|
|
351
344
|
|
|
352
345
|
// The new release's singleton fails to start, so the deploy surfaces the error.
|
|
353
|
-
await
|
|
346
|
+
await expect(daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})).rejects.toThrow()
|
|
354
347
|
|
|
355
348
|
// The old singleton is stopped before the new one is started, so two copies never
|
|
356
349
|
// overlap — even when the replacement then fails.
|
|
@@ -360,9 +353,9 @@ test("a failed singleton replacement surfaces the error after stopping the old s
|
|
|
360
353
|
|
|
361
354
|
// Traffic switches before singletons are replaced, so the new release is already active,
|
|
362
355
|
// but its singleton is left failed with no replacement running.
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
356
|
+
expect(status.activeReleaseId).toBe("v2")
|
|
357
|
+
expect(status.singletons.length).toBe(1)
|
|
358
|
+
expect(status.singletons[0].process.state).toBe("failed")
|
|
366
359
|
} finally {
|
|
367
360
|
await daemon.shutdown()
|
|
368
361
|
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
@@ -379,15 +372,15 @@ test("service processes start before releases and restart with the latest deploy
|
|
|
379
372
|
|
|
380
373
|
const firstServiceStatus = daemon.status().services[0].process
|
|
381
374
|
|
|
382
|
-
|
|
383
|
-
|
|
375
|
+
expect(firstServiceStatus.pid).toBeTruthy()
|
|
376
|
+
expect(firstServiceStatus.command).toMatch(/v1/)
|
|
384
377
|
|
|
385
378
|
await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
|
|
386
379
|
|
|
387
380
|
const secondServiceStatus = daemon.status().services[0].process
|
|
388
381
|
|
|
389
|
-
|
|
390
|
-
|
|
382
|
+
expect(secondServiceStatus.pid).toBe(firstServiceStatus.pid)
|
|
383
|
+
expect(secondServiceStatus.command).toMatch(/v2/)
|
|
391
384
|
|
|
392
385
|
process.kill(-Number(secondServiceStatus.pid), "SIGTERM")
|
|
393
386
|
await waitFor(async () => {
|
|
@@ -413,28 +406,28 @@ test("handoff services start per release and drain with their release", async ()
|
|
|
413
406
|
const v1 = statusRelease(daemon, "v1")
|
|
414
407
|
const v1Service = v1.processes.find((processStatus) => processStatus.id === "beacon")
|
|
415
408
|
|
|
416
|
-
|
|
417
|
-
|
|
409
|
+
expect(v1Service?.pid).toBeTruthy()
|
|
410
|
+
expect(v1.ports.beacon > 0).toBe(true)
|
|
418
411
|
|
|
419
412
|
await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
|
|
420
413
|
const v2 = statusRelease(daemon, "v2")
|
|
421
414
|
const v2Service = v2.processes.find((processStatus) => processStatus.id === "beacon")
|
|
422
415
|
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
416
|
+
expect(v2Service?.pid).toBeTruthy()
|
|
417
|
+
expect(v2.ports.beacon).not.toBe(v1.ports.beacon)
|
|
418
|
+
expect(statusRelease(daemon, "v1").processes.find((processStatus) => processStatus.id === "beacon")?.state).toBe("quiesced")
|
|
426
419
|
|
|
427
420
|
socket.close()
|
|
428
421
|
socket = undefined
|
|
429
422
|
|
|
430
423
|
await waitFor(() => statusRelease(daemon, "v1").state === "stopped")
|
|
431
|
-
|
|
424
|
+
expect(statusRelease(daemon, "v1").processes.find((processStatus) => processStatus.id === "beacon")?.state).toBe("stopped")
|
|
432
425
|
|
|
433
426
|
const events = await processEvents(fixture.serviceLogPath)
|
|
434
427
|
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
428
|
+
expect(events.some((event) => event.event === "start" && event.releaseId === "v1")).toBe(true)
|
|
429
|
+
expect(events.some((event) => event.event === "start" && event.releaseId === "v2")).toBe(true)
|
|
430
|
+
expect(events.some((event) => event.event === "stop" && event.releaseId === "v1")).toBe(true)
|
|
438
431
|
} finally {
|
|
439
432
|
if (socket) socket.close()
|
|
440
433
|
await daemon.shutdown()
|
|
@@ -457,8 +450,8 @@ test("handoff services stop after release-local dependents finish draining", asy
|
|
|
457
450
|
|
|
458
451
|
const drainingRelease = statusRelease(daemon, "v1")
|
|
459
452
|
|
|
460
|
-
|
|
461
|
-
|
|
453
|
+
expect(drainingRelease.state).toBe("draining")
|
|
454
|
+
expect(drainingRelease.processes.find((processStatus) => processStatus.id === "beacon")?.state).toBe("quiesced")
|
|
462
455
|
|
|
463
456
|
socket.close()
|
|
464
457
|
socket = undefined
|
|
@@ -467,7 +460,8 @@ test("handoff services stop after release-local dependents finish draining", asy
|
|
|
467
460
|
const events = await processEvents(fixture.serviceLogPath)
|
|
468
461
|
const v1ServiceStop = events.find((event) => event.event === "stop" && event.releaseId === "v1")
|
|
469
462
|
|
|
470
|
-
|
|
463
|
+
// V1 handoff service should stop after release drain.
|
|
464
|
+
expect(v1ServiceStop).toBeTruthy()
|
|
471
465
|
} finally {
|
|
472
466
|
if (socket) socket.close()
|
|
473
467
|
await daemon.shutdown()
|
|
@@ -485,40 +479,46 @@ test("candidate activation retires jobs-main with its workers without waiting fo
|
|
|
485
479
|
const oldService = oldRelease.processes.find((processStatus) => processStatus.id === "beacon")
|
|
486
480
|
const oldWorker = oldRelease.processes.find((processStatus) => processStatus.id === "worker")
|
|
487
481
|
|
|
488
|
-
|
|
489
|
-
|
|
482
|
+
expect(oldService?.pid).toBeTruthy()
|
|
483
|
+
expect(oldWorker?.pid).toBeTruthy()
|
|
490
484
|
|
|
491
485
|
await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
|
|
492
486
|
|
|
493
|
-
|
|
494
|
-
|
|
487
|
+
// Traffic must switch only after the complete candidate is healthy.
|
|
488
|
+
expect(daemon.status().activeReleaseId).toBe("v2")
|
|
489
|
+
// Old jobs-main must quiesce immediately after candidate activation.
|
|
490
|
+
expect(await fs.readFile(fixture.serviceQuietPath, "utf8")).toBe("v1\n")
|
|
495
491
|
|
|
496
492
|
const retired = statusRelease(daemon, "v1")
|
|
497
493
|
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
494
|
+
// Deployment completion must not wait for the old jobs generation.
|
|
495
|
+
expect(retired.state).toBe("draining")
|
|
496
|
+
// Old jobs-main must remain alive and quiesced with its draining workers.
|
|
497
|
+
expect(retired.processes.find((processStatus) => processStatus.id === "beacon")?.state).toBe("quiesced")
|
|
498
|
+
// Old worker must remain in its original generation until accepted work settles.
|
|
499
|
+
expect(retired.processes.find((processStatus) => processStatus.id === "worker")?.state).not.toBe("stopped")
|
|
500
|
+
// Old and new workers must retain distinct jobs-main endpoints.
|
|
501
|
+
expect(statusRelease(daemon, "v2").ports.beacon).not.toBe(retired.ports.beacon)
|
|
502
502
|
} finally {
|
|
503
503
|
await daemon.shutdown()
|
|
504
504
|
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
505
505
|
}
|
|
506
506
|
})
|
|
507
507
|
|
|
508
|
-
test("opt-in generation lifecycle
|
|
508
|
+
test("opt-in generation lifecycle acknowledges old retirement before activating the candidate", async () => {
|
|
509
509
|
const fixture = await createFixture({handoffService: true, handoffServiceActivate: true, nonBlockingDrainWorker: true, webDependsOnService: true})
|
|
510
510
|
const daemon = await startDaemon(fixture.config)
|
|
511
511
|
|
|
512
512
|
try {
|
|
513
513
|
await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
|
|
514
|
-
|
|
515
|
-
|
|
514
|
+
expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1"])
|
|
515
|
+
expect(daemon.status().activeReleaseId).toBe("v1")
|
|
516
516
|
|
|
517
517
|
await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
|
|
518
518
|
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
519
|
+
expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "retire:v1", "activate:v2"])
|
|
520
|
+
expect(daemon.status().activeReleaseId).toBe("v2")
|
|
521
|
+
expect(daemon.status().generationTransition?.phase).toBe("committed")
|
|
522
522
|
} finally {
|
|
523
523
|
await daemon.shutdown()
|
|
524
524
|
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
@@ -535,11 +535,11 @@ test("manual restart reaches the active handoff coordinator and restores its lif
|
|
|
535
535
|
const result = await daemon.restartProcesses({processId: "beacon"})
|
|
536
536
|
const after = statusRelease(daemon, "v1").processes.find((processStatus) => processStatus.id === "beacon")?.pid
|
|
537
537
|
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
538
|
+
expect(result).toEqual({restarted: ["beacon"]})
|
|
539
|
+
expect(before).toBeTruthy()
|
|
540
|
+
expect(after).toBeTruthy()
|
|
541
|
+
expect(after).not.toBe(before)
|
|
542
|
+
expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "retire:v1", "activate:v1"])
|
|
543
543
|
} finally {
|
|
544
544
|
await daemon.shutdown()
|
|
545
545
|
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
@@ -574,14 +574,14 @@ test("generation commit is durable before awaited post-transition work", async (
|
|
|
574
574
|
|
|
575
575
|
const persisted = /** @type {{activeReleaseId?: string, generationTransition?: {phase?: string}, singletonReleaseIds?: Record<string, string>} | undefined} */ (await readState(fixture.statePath))
|
|
576
576
|
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
577
|
+
expect(daemon.status().activeReleaseId).toBe("v2")
|
|
578
|
+
expect(persisted?.activeReleaseId).toBe("v2")
|
|
579
|
+
expect(persisted?.generationTransition?.phase).toBe("committed_pending")
|
|
580
|
+
expect(persisted?.singletonReleaseIds?.["jobs-main"]).toBe("v1")
|
|
581
581
|
|
|
582
582
|
releaseReplacement()
|
|
583
583
|
await deployPromise
|
|
584
|
-
|
|
584
|
+
expect(daemon.status().generationTransition?.phase).toBe("committed")
|
|
585
585
|
} finally {
|
|
586
586
|
releaseReplacement()
|
|
587
587
|
await deployPromise?.catch(() => {})
|
|
@@ -598,18 +598,19 @@ test("exact committed retry finishes pending singleton replacement before succes
|
|
|
598
598
|
|
|
599
599
|
try {
|
|
600
600
|
await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
|
|
601
|
-
await
|
|
601
|
+
await expect(daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})).rejects.toThrow(/ENOENT/)
|
|
602
602
|
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
603
|
+
// Traffic remains durably committed.
|
|
604
|
+
expect(daemon.status().activeReleaseId).toBe("v2")
|
|
605
|
+
expect(daemon.status().generationTransition?.phase).toBe("committed_pending")
|
|
606
|
+
expect(daemon.status().singletons[0]?.process.state).not.toBe("running")
|
|
606
607
|
|
|
607
608
|
await fs.mkdir(path.join(fixture.root, "v2"))
|
|
608
609
|
await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
|
|
609
610
|
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
611
|
+
expect(daemon.status().generationTransition?.phase).toBe("committed")
|
|
612
|
+
expect(daemon.status().singletons[0]?.process.state).toBe("running")
|
|
613
|
+
expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "retire:v1", "activate:v2"])
|
|
613
614
|
} finally {
|
|
614
615
|
await daemon.shutdown()
|
|
615
616
|
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
@@ -621,39 +622,39 @@ test("first generation is not committed when its activation acknowledgement fail
|
|
|
621
622
|
const daemon = await startDaemon(fixture.config)
|
|
622
623
|
|
|
623
624
|
try {
|
|
624
|
-
await
|
|
625
|
+
await expect(daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})).rejects.toThrow(/activate command exited non-zero/)
|
|
625
626
|
const status = daemon.status()
|
|
626
627
|
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
628
|
+
expect(status.activeReleaseId).toBe(null)
|
|
629
|
+
expect(status.generationTransition?.phase).toBe("activating_candidate")
|
|
630
|
+
expect(status.releaseReferences.map((reference) => reference.releaseId)).toEqual(["v1"])
|
|
630
631
|
} finally {
|
|
631
632
|
await daemon.shutdown()
|
|
632
633
|
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
633
634
|
}
|
|
634
635
|
})
|
|
635
636
|
|
|
636
|
-
test("retirement failure retains the exact transition, blocks other deploys, and exact resume continues it", async () => {
|
|
637
|
+
test("retirement acknowledgement failure retains the exact transition, blocks other deploys, and exact resume continues it", async () => {
|
|
637
638
|
const fixture = await createFixture({handoffService: true, handoffServiceActivate: true, handoffServiceQuietFailure: true, nonBlockingDrainWorker: true, webDependsOnService: true})
|
|
638
639
|
const daemon = await startDaemon(fixture.config)
|
|
639
640
|
|
|
640
641
|
try {
|
|
641
642
|
await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
|
|
642
|
-
await
|
|
643
|
+
await expect(daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})).rejects.toThrow(/retirement quiescence failed/)
|
|
643
644
|
|
|
644
645
|
const failed = daemon.status()
|
|
645
646
|
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
await
|
|
647
|
+
expect(failed.activeReleaseId).toBe("v1")
|
|
648
|
+
expect(failed.generationTransition?.phase).toBe("retiring_previous")
|
|
649
|
+
expect(String(failed.generationTransition?.error)).toMatch(/quiet command exited non-zero/)
|
|
650
|
+
expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1"])
|
|
651
|
+
await expect(daemon.deploy({releaseId: "v3", releasePath: fixture.root, revision: "v3"})).rejects.toThrow(/transition.*v2.*unresolved/i)
|
|
651
652
|
|
|
652
653
|
await fs.writeFile(fixture.retirementGatePath, "allow\n")
|
|
653
654
|
await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
|
|
654
655
|
|
|
655
|
-
|
|
656
|
-
|
|
656
|
+
expect(daemon.status().activeReleaseId).toBe("v2")
|
|
657
|
+
expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "retire:v1", "activate:v2"])
|
|
657
658
|
} finally {
|
|
658
659
|
await daemon.shutdown()
|
|
659
660
|
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
@@ -669,41 +670,37 @@ test("candidate activation failure reports restoration failure and exact recover
|
|
|
669
670
|
const incumbentCoordinator = daemon.releases.get("v1")?.getProcess("beacon")
|
|
670
671
|
const reactivate = incumbentCoordinator?.reactivateStrict.bind(incumbentCoordinator)
|
|
671
672
|
|
|
672
|
-
|
|
673
|
+
if (!(incumbentCoordinator && reactivate)) throw new Error("Missing required fixture: incumbentCoordinator && reactivate")
|
|
673
674
|
incumbentCoordinator.reactivateStrict = async () => { throw new Error("incumbent restoration rejected") }
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
assert.match(error.message, /incumbent v1 restoration failed: incumbent restoration rejected/i)
|
|
680
|
-
return true
|
|
681
|
-
}
|
|
682
|
-
)
|
|
675
|
+
const deployment = daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
|
|
676
|
+
|
|
677
|
+
await expect(deployment).rejects.toBeInstanceOf(AggregateError)
|
|
678
|
+
await expect(deployment).rejects.toMatchObject({message: expect.stringMatching(/activate command exited non-zero/)})
|
|
679
|
+
await expect(deployment).rejects.toMatchObject({message: expect.stringMatching(/incumbent v1 restoration failed: incumbent restoration rejected/i)})
|
|
683
680
|
|
|
684
681
|
const failed = daemon.status()
|
|
685
682
|
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
683
|
+
expect(failed.activeReleaseId).toBe("v1")
|
|
684
|
+
expect(failed.generationTransition?.phase).toBe("restoring_previous")
|
|
685
|
+
expect(String(failed.generationTransition?.activationError)).toMatch(/activate command exited non-zero/)
|
|
686
|
+
expect(String(failed.generationTransition?.compensationError)).toMatch(/incumbent restoration rejected/)
|
|
690
687
|
const failedEvents = daemon.eventLog.recent()
|
|
691
688
|
const activationEvent = failedEvents.find((event) => event.message === "release generation activation failed")
|
|
692
689
|
const restorationEvent = failedEvents.find((event) => event.message === "release generation compensation restoration failed")
|
|
693
690
|
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
await
|
|
691
|
+
expect(String(activationEvent?.data.error)).toMatch(/activate command exited non-zero/)
|
|
692
|
+
expect(String(restorationEvent?.data.activationError)).toMatch(/activate command exited non-zero/)
|
|
693
|
+
expect(String(restorationEvent?.data.error)).toMatch(/incumbent restoration rejected/)
|
|
694
|
+
expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "retire:v1", "retire:v2"])
|
|
695
|
+
await expect(daemon.deploy({releaseId: "v3", releasePath: fixture.root, revision: "v3"})).rejects.toThrow(/transition.*v2.*unresolved/i)
|
|
699
696
|
const failedCandidate = daemon.releases.get("v2")
|
|
700
697
|
|
|
701
|
-
|
|
698
|
+
if (!failedCandidate) throw new Error("Missing required fixture: failedCandidate")
|
|
702
699
|
await failedCandidate.stop()
|
|
703
700
|
incumbentCoordinator.reactivateStrict = reactivate
|
|
704
701
|
await incumbentCoordinator.stop()
|
|
705
|
-
|
|
706
|
-
|
|
702
|
+
expect(failedCandidate.state).toBe("stopped")
|
|
703
|
+
expect(incumbentCoordinator.status().state).toBe("stopped")
|
|
707
704
|
daemon.config = structuredClone(daemon.config)
|
|
708
705
|
daemon.config.processes[0].lifecycle.activateTimeoutMs = (daemon.config.processes[0].lifecycle.activateTimeoutMs ?? 30000) + 1
|
|
709
706
|
const recovery = await sendControlCommand({
|
|
@@ -717,13 +714,13 @@ test("candidate activation failure reports restoration failure and exact recover
|
|
|
717
714
|
path: fixture.config.control.path
|
|
718
715
|
})
|
|
719
716
|
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
717
|
+
expect(recovery.recoveryStatus).toBe("recovered")
|
|
718
|
+
expect(daemon.status().activeReleaseId).toBe("v1")
|
|
719
|
+
expect(daemon.status().generationTransition).toBe(undefined)
|
|
723
720
|
const persisted = /** @type {{generationTransition?: import("../src/json.js").JsonValue} | undefined} */ (await readState(fixture.statePath))
|
|
724
721
|
|
|
725
|
-
|
|
726
|
-
|
|
722
|
+
expect(persisted?.generationTransition).toBe(undefined)
|
|
723
|
+
expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "retire:v1", "retire:v2", "activate:v1"])
|
|
727
724
|
|
|
728
725
|
const idempotent = await sendControlCommand({
|
|
729
726
|
command: {
|
|
@@ -736,10 +733,9 @@ test("candidate activation failure reports restoration failure and exact recover
|
|
|
736
733
|
path: fixture.config.control.path
|
|
737
734
|
})
|
|
738
735
|
|
|
739
|
-
|
|
736
|
+
expect(idempotent.recoveryStatus).toBe("already_recovered")
|
|
740
737
|
await daemon.deploy({releaseId: "v3", releasePath: fixture.root, revision: "v3"})
|
|
741
|
-
await
|
|
742
|
-
() => sendControlCommand({
|
|
738
|
+
await expect(sendControlCommand({
|
|
743
739
|
command: {
|
|
744
740
|
command: "recover-generation-transition",
|
|
745
741
|
previousReleaseId: "v1",
|
|
@@ -748,9 +744,7 @@ test("candidate activation failure reports restoration failure and exact recover
|
|
|
748
744
|
revision: "v3"
|
|
749
745
|
},
|
|
750
746
|
path: fixture.config.control.path
|
|
751
|
-
})
|
|
752
|
-
/not a safe failed pre-commit transition/i
|
|
753
|
-
)
|
|
747
|
+
})).rejects.toThrow(/not a safe failed pre-commit transition/i)
|
|
754
748
|
} finally {
|
|
755
749
|
await daemon.shutdown()
|
|
756
750
|
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
@@ -765,12 +759,9 @@ test("explicit recovery stops the exact failed candidate and fences degraded inc
|
|
|
765
759
|
await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
|
|
766
760
|
const incumbentCoordinator = daemon.releases.get("v1")?.getProcess("beacon")
|
|
767
761
|
|
|
768
|
-
|
|
762
|
+
if (!incumbentCoordinator) throw new Error("Missing required fixture: incumbentCoordinator")
|
|
769
763
|
incumbentCoordinator.reactivateStrict = async () => { throw new Error("Cannot activate background jobs generation from retired") }
|
|
770
|
-
await
|
|
771
|
-
() => daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"}),
|
|
772
|
-
/Cannot activate background jobs generation from retired/i
|
|
773
|
-
)
|
|
764
|
+
await expect(daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})).rejects.toThrow(/Cannot activate background jobs generation from retired/i)
|
|
774
765
|
|
|
775
766
|
const candidate = daemon.releases.get("v2")
|
|
776
767
|
const transition = daemon.generationTransition
|
|
@@ -788,44 +779,49 @@ test("explicit recovery stops the exact failed candidate and fences degraded inc
|
|
|
788
779
|
path: fixture.config.control.path
|
|
789
780
|
})
|
|
790
781
|
|
|
791
|
-
|
|
792
|
-
|
|
782
|
+
if (!(candidate && transition && incumbentWebPid)) throw new Error("Missing required fixture: candidate && transition && incumbentWebPid")
|
|
783
|
+
// Ordinary failed compensation leaves the candidate draining.
|
|
784
|
+
expect(candidate.state).toBe("draining")
|
|
793
785
|
|
|
794
786
|
const retainedCandidateConfig = candidate.config
|
|
795
787
|
|
|
796
788
|
candidate.config = {...candidate.config, releaseRetention: {...candidate.config.releaseRetention, keep: candidate.config.releaseRetention.keep + 1}}
|
|
797
|
-
await
|
|
789
|
+
await expect(exactRecovery()).rejects.toThrow(/does not retain its exact path, revision, and config authority/i)
|
|
798
790
|
candidate.config = retainedCandidateConfig
|
|
799
|
-
await
|
|
800
|
-
await
|
|
791
|
+
await expect(exactRecovery({previousReleaseId: "wrong-v1"})).rejects.toThrow(/refusing stale recovery/i)
|
|
792
|
+
await expect(exactRecovery({revision: "wrong-v2"})).rejects.toThrow(/exact same release, path, revision, and config authority/i)
|
|
801
793
|
transition.phase = "retiring_failed_candidate"
|
|
802
|
-
await
|
|
794
|
+
await expect(exactRecovery()).rejects.toThrow(/requires retiring_previous or restoring_previous/i)
|
|
803
795
|
transition.phase = "restoring_previous"
|
|
804
796
|
const terminalFailure = transition.compensationError
|
|
805
797
|
|
|
806
798
|
transition.compensationError = "incumbent activation was temporarily unavailable"
|
|
807
|
-
await
|
|
799
|
+
await expect(exactRecovery()).rejects.toThrow(/terminal retirement/i)
|
|
808
800
|
transition.compensationError = terminalFailure
|
|
801
|
+
await incumbentCoordinator.setLifecycleRole("retired")
|
|
802
|
+
expect(incumbentCoordinator.status().lifecycleRole).toBe("retired")
|
|
809
803
|
const checkpoint = daemon.checkpointGenerationTransition.bind(daemon)
|
|
810
804
|
|
|
811
805
|
daemon.checkpointGenerationTransition = async () => { throw new Error("injected checkpoint failure") }
|
|
812
|
-
await
|
|
813
|
-
|
|
806
|
+
await expect(exactRecovery()).rejects.toThrow(/checkpoint failed: injected checkpoint failure/i)
|
|
807
|
+
expect(daemon.generationTransition).toBe(transition)
|
|
814
808
|
daemon.checkpointGenerationTransition = checkpoint
|
|
815
809
|
|
|
816
810
|
const eventsBeforeRecovery = await lifecycleEvents(fixture.lifecycleLogPath)
|
|
817
811
|
const recovery = await exactRecovery()
|
|
818
812
|
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
813
|
+
expect(recovery.recoveryStatus).toBe("retired_incumbent_accepted")
|
|
814
|
+
expect(recovery.jobsStatus).toBe("degraded")
|
|
815
|
+
expect(daemon.status().generationTransition?.phase).toBe("degraded_active")
|
|
816
|
+
expect(statusRelease(daemon, "v1").processes.find(({id}) => id === "web")?.pid).toBe(incumbentWebPid)
|
|
817
|
+
expect(await fetchText(daemon, "/release")).toBe("v1")
|
|
818
|
+
// Guarded recovery returns before failed-candidate drain completion.
|
|
819
|
+
expect(["draining", "stopped"].includes(candidate.state)).toBe(true)
|
|
820
|
+
// Recovery must not activate either retained generation.
|
|
821
|
+
expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(eventsBeforeRecovery)
|
|
826
822
|
const persisted = /** @type {{generationTransition?: import("../src/json.js").JsonValue} | undefined} */ (await readState(fixture.statePath))
|
|
827
823
|
|
|
828
|
-
|
|
824
|
+
expect(/** @type {{phase?: string} | undefined} */ (persisted?.generationTransition)?.phase).toBe("degraded_active")
|
|
829
825
|
|
|
830
826
|
transition.phase = "retiring_previous"
|
|
831
827
|
transition.error = "Release v1 retirement quiescence failed: quiet command exited non-zero with status 1"
|
|
@@ -834,17 +830,29 @@ test("explicit recovery stops the exact failed candidate and fences degraded inc
|
|
|
834
830
|
await daemon.checkpointGenerationTransition()
|
|
835
831
|
const legacyRecovery = await exactRecovery()
|
|
836
832
|
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
833
|
+
expect(legacyRecovery.recoveryStatus).toBe("retired_incumbent_accepted")
|
|
834
|
+
// Guarded recovery migrates a legacy terminal retirement fence.
|
|
835
|
+
expect(daemon.status().generationTransition?.phase).toBe("degraded_active")
|
|
836
|
+
|
|
837
|
+
transition.phase = "restoring_previous"
|
|
838
|
+
transition.compensationError = "Process background-jobs-main is not retained for reactivation"
|
|
839
|
+
daemon.releases.get("v1")?.processes.delete("beacon")
|
|
840
|
+
await daemon.checkpointGenerationTransition()
|
|
841
|
+
const absentCoordinatorRecovery = await exactRecovery()
|
|
842
|
+
|
|
843
|
+
expect(absentCoordinatorRecovery.jobsStatus).toBe("degraded")
|
|
844
|
+
// Terminally absent incumbent coordinator remains guarded jobs-degraded authority.
|
|
845
|
+
expect(daemon.status().generationTransition?.phase).toBe("degraded_active")
|
|
846
|
+
await expect(daemon.deploy({releaseId: "bad-v3", releasePath: fixture.root, revision: "bad-v3"})).rejects.toThrow(/health check failed/i)
|
|
847
|
+
expect(daemon.status().generationTransition?.phase).toBe("degraded_active")
|
|
848
|
+
expect(statusRelease(daemon, "v1").processes.find(({id}) => id === "web")?.pid).toBe(incumbentWebPid)
|
|
849
|
+
expect(await fetchText(daemon, "/release")).toBe("v1")
|
|
843
850
|
await daemon.deploy({releaseId: "v3", releasePath: fixture.root, revision: "v3"})
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
851
|
+
expect(daemon.status().activeReleaseId).toBe("v3")
|
|
852
|
+
expect(daemon.status().generationTransition?.phase).toBe("committed")
|
|
853
|
+
expect(await fetchText(daemon, "/release")).toBe("v3")
|
|
854
|
+
// Fresh deployment must not re-retire a degraded incumbent generation.
|
|
855
|
+
expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "retire:v1", "retire:v2", "retire:bad-v3", "activate:v3"])
|
|
848
856
|
} finally {
|
|
849
857
|
await daemon.shutdown()
|
|
850
858
|
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
@@ -857,23 +865,20 @@ test("candidate activation failure compensates to the incumbent and admits a dif
|
|
|
857
865
|
|
|
858
866
|
try {
|
|
859
867
|
await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
|
|
860
|
-
await
|
|
861
|
-
() => daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"}),
|
|
862
|
-
/activate command exited non-zero.*compensation restored incumbent v1 as authoritative and retired failed candidate v2/i
|
|
863
|
-
)
|
|
868
|
+
await expect(daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})).rejects.toThrow(/activate command exited non-zero.*compensation restored incumbent v1 as authoritative and retired failed candidate v2/i)
|
|
864
869
|
|
|
865
870
|
const compensated = daemon.status()
|
|
866
871
|
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
+
expect(compensated.activeReleaseId).toBe("v1")
|
|
873
|
+
expect(compensated.generationTransition).toBe(undefined)
|
|
874
|
+
expect(await fetchText(daemon, "/release")).toBe("v1")
|
|
875
|
+
expect(statusRelease(daemon, "v1").processes.find((processStatus) => processStatus.id === "worker")?.state).toBe("running")
|
|
876
|
+
expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "retire:v1", "retire:v2", "activate:v1"])
|
|
872
877
|
|
|
873
878
|
await daemon.deploy({releaseId: "v3", releasePath: fixture.root, revision: "v3"})
|
|
874
879
|
|
|
875
|
-
|
|
876
|
-
|
|
880
|
+
expect(daemon.status().activeReleaseId).toBe("v3")
|
|
881
|
+
expect(await fetchText(daemon, "/release")).toBe("v3")
|
|
877
882
|
} finally {
|
|
878
883
|
await daemon.shutdown()
|
|
879
884
|
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
@@ -886,20 +891,17 @@ test("ambiguous candidate activation retires the candidate before reactivating t
|
|
|
886
891
|
|
|
887
892
|
try {
|
|
888
893
|
await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
|
|
889
|
-
await
|
|
890
|
-
() => daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"}),
|
|
891
|
-
/activate command exited non-zero.*compensation restored incumbent v1 as authoritative and retired failed candidate v2/i
|
|
892
|
-
)
|
|
894
|
+
await expect(daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})).rejects.toThrow(/activate command exited non-zero.*compensation restored incumbent v1 as authoritative and retired failed candidate v2/i)
|
|
893
895
|
|
|
894
|
-
|
|
896
|
+
expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual([
|
|
895
897
|
"activate:v1",
|
|
896
898
|
"retire:v1",
|
|
897
899
|
"activate:v2",
|
|
898
900
|
"retire:v2",
|
|
899
901
|
"activate:v1"
|
|
900
902
|
])
|
|
901
|
-
|
|
902
|
-
|
|
903
|
+
expect(daemon.status().activeReleaseId).toBe("v1")
|
|
904
|
+
expect(daemon.status().generationTransition).toBe(undefined)
|
|
903
905
|
} finally {
|
|
904
906
|
await daemon.shutdown()
|
|
905
907
|
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
@@ -912,20 +914,17 @@ test("candidate activation recovery reverses a worker-specific quiet hook before
|
|
|
912
914
|
|
|
913
915
|
try {
|
|
914
916
|
await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
|
|
915
|
-
await
|
|
916
|
-
() => daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"}),
|
|
917
|
-
/compensation restored incumbent v1 as authoritative/i
|
|
918
|
-
)
|
|
917
|
+
await expect(daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})).rejects.toThrow(/compensation restored incumbent v1 as authoritative/i)
|
|
919
918
|
|
|
920
919
|
const events = await lifecycleEvents(fixture.lifecycleLogPath)
|
|
921
920
|
const candidateRetired = events.indexOf("worker-retire:v2")
|
|
922
921
|
const workerReactivated = events.indexOf("worker-reactivate:v1")
|
|
923
922
|
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
923
|
+
expect({value: Boolean(candidateRetired >= 0), context: JSON.stringify(events)}).toMatchObject({value: true})
|
|
924
|
+
expect({value: Boolean(workerReactivated > candidateRetired), context: JSON.stringify(events)}).toMatchObject({value: true})
|
|
925
|
+
expect(statusRelease(daemon, "v1").processes.find((processStatus) => processStatus.id === "worker")?.state).toBe("running")
|
|
926
|
+
expect(daemon.status().activeReleaseId).toBe("v1")
|
|
927
|
+
expect(daemon.status().generationTransition).toBe(undefined)
|
|
929
928
|
} finally {
|
|
930
929
|
await daemon.shutdown()
|
|
931
930
|
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
@@ -938,28 +937,22 @@ test("candidate activation recovery keeps the fence when a worker-specific resum
|
|
|
938
937
|
|
|
939
938
|
try {
|
|
940
939
|
await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
assert.match(failure.message, /activate command exited non-zero/)
|
|
947
|
-
assert.match(failure.message, /reactivate command exited non-zero/)
|
|
948
|
-
return true
|
|
949
|
-
}
|
|
950
|
-
)
|
|
940
|
+
const deployment = daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
|
|
941
|
+
|
|
942
|
+
await expect(deployment).rejects.toMatchObject({message: expect.stringMatching(/activate command exited non-zero/)})
|
|
943
|
+
await expect(deployment).rejects.toMatchObject({message: expect.stringMatching(/reactivate command exited non-zero/)})
|
|
951
944
|
|
|
952
945
|
const status = daemon.status()
|
|
953
946
|
const restorationEvent = daemon.eventLog.recent().find((event) => event.message === "release generation compensation restoration failed")
|
|
954
947
|
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
await
|
|
948
|
+
expect(status.activeReleaseId).toBe("v1")
|
|
949
|
+
expect(status.generationTransition?.phase).toBe("restoring_previous")
|
|
950
|
+
expect(String(status.generationTransition?.activationError)).toMatch(/activate command exited non-zero/)
|
|
951
|
+
expect(String(status.generationTransition?.compensationError)).toMatch(/reactivate command exited non-zero/)
|
|
952
|
+
expect(statusRelease(daemon, "v1").processes.find((processStatus) => processStatus.id === "worker")?.state).toBe("quiesced")
|
|
953
|
+
expect(String(restorationEvent?.data.activationError)).toMatch(/activate command exited non-zero/)
|
|
954
|
+
expect(String(restorationEvent?.data.error)).toMatch(/reactivate command exited non-zero/)
|
|
955
|
+
await expect(daemon.deploy({releaseId: "v3", releasePath: fixture.root, revision: "v3"})).rejects.toThrow(/transition.*v2.*unresolved/i)
|
|
963
956
|
} finally {
|
|
964
957
|
await daemon.shutdown()
|
|
965
958
|
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
@@ -978,17 +971,14 @@ test("compensation keeps the fence when the cleared checkpoint cannot be persist
|
|
|
978
971
|
if (!daemon.generationTransition) throw new Error("cleared checkpoint unavailable")
|
|
979
972
|
await checkpoint()
|
|
980
973
|
}
|
|
981
|
-
await
|
|
982
|
-
() => daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"}),
|
|
983
|
-
/activate command exited non-zero.*compensation checkpoint clear failed: cleared checkpoint unavailable/i
|
|
984
|
-
)
|
|
974
|
+
await expect(daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})).rejects.toThrow(/activate command exited non-zero.*compensation checkpoint clear failed: cleared checkpoint unavailable/i)
|
|
985
975
|
|
|
986
|
-
|
|
987
|
-
|
|
976
|
+
expect(daemon.status().activeReleaseId).toBe("v1")
|
|
977
|
+
expect(daemon.status().generationTransition?.phase).toBe("restoring_previous")
|
|
988
978
|
const persisted = /** @type {{generationTransition?: {phase?: string}} | undefined} */ (await readState(fixture.statePath))
|
|
989
979
|
|
|
990
|
-
|
|
991
|
-
await
|
|
980
|
+
expect(persisted?.generationTransition?.phase).toBe("restoring_previous")
|
|
981
|
+
await expect(daemon.deploy({releaseId: "v3", releasePath: fixture.root, revision: "v3"})).rejects.toThrow(/transition.*v2.*unresolved/i)
|
|
992
982
|
|
|
993
983
|
daemon.checkpointGenerationTransition = checkpoint
|
|
994
984
|
const recovery = await sendControlCommand({
|
|
@@ -1002,8 +992,8 @@ test("compensation keeps the fence when the cleared checkpoint cannot be persist
|
|
|
1002
992
|
path: fixture.config.control.path
|
|
1003
993
|
})
|
|
1004
994
|
|
|
1005
|
-
|
|
1006
|
-
|
|
995
|
+
expect(recovery.recoveryStatus).toBe("recovered")
|
|
996
|
+
expect(daemon.status().generationTransition).toBe(undefined)
|
|
1007
997
|
} finally {
|
|
1008
998
|
await daemon.shutdown()
|
|
1009
999
|
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
@@ -1018,16 +1008,16 @@ test("unresolved generation transition fences stop, restart, and rollback mutati
|
|
|
1018
1008
|
await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
|
|
1019
1009
|
const incumbentCoordinator = daemon.releases.get("v1")?.getProcess("beacon")
|
|
1020
1010
|
|
|
1021
|
-
|
|
1011
|
+
if (!incumbentCoordinator) throw new Error("Missing required fixture: incumbentCoordinator")
|
|
1022
1012
|
incumbentCoordinator.reactivateStrict = async () => { throw new Error("incumbent restoration rejected") }
|
|
1023
|
-
await
|
|
1013
|
+
await expect(daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})).rejects.toThrow(/activate command exited non-zero/)
|
|
1024
1014
|
|
|
1025
|
-
await
|
|
1026
|
-
await
|
|
1027
|
-
await
|
|
1015
|
+
await expect(daemon.stopRelease("v2")).rejects.toThrow(/cannot stop.*generation transition.*unresolved/i)
|
|
1016
|
+
await expect(daemon.restartProcesses({processId: "beacon"})).rejects.toThrow(/cannot restart.*generation transition.*unresolved/i)
|
|
1017
|
+
await expect(daemon.rollback({releaseId: "v2"})).rejects.toThrow(/cannot rollback.*generation transition.*unresolved/i)
|
|
1028
1018
|
|
|
1029
|
-
|
|
1030
|
-
|
|
1019
|
+
expect(statusRelease(daemon, "v2").processes.find((entry) => entry.id === "web")?.state).not.toBe("stopped")
|
|
1020
|
+
expect(await fetchText(daemon, "/release")).toBe("v1")
|
|
1031
1021
|
} finally {
|
|
1032
1022
|
await daemon.shutdown()
|
|
1033
1023
|
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
@@ -1042,13 +1032,13 @@ test("active generation restores its exact lifecycle role after coordinator auto
|
|
|
1042
1032
|
await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
|
|
1043
1033
|
const coordinator = statusRelease(daemon, "v1").processes.find((entry) => entry.id === "beacon")
|
|
1044
1034
|
|
|
1045
|
-
|
|
1035
|
+
if (!coordinator?.pid) throw new Error("Missing required fixture: coordinator?.pid")
|
|
1046
1036
|
process.kill(-coordinator.pid, "SIGKILL")
|
|
1047
1037
|
await waitFor(async () => (await lifecycleEvents(fixture.lifecycleLogPath)).length === 2 && statusRelease(daemon, "v1").processes.find((entry) => entry.id === "beacon")?.state === "running", 3000)
|
|
1048
1038
|
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1039
|
+
expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "activate:v1"])
|
|
1040
|
+
expect(statusRelease(daemon, "v1").processes.find((entry) => entry.id === "beacon")?.state).toBe("running")
|
|
1041
|
+
expect(await fetchText(daemon, "/release")).toBe("v1")
|
|
1052
1042
|
} finally {
|
|
1053
1043
|
await daemon.shutdown()
|
|
1054
1044
|
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
@@ -1066,7 +1056,7 @@ test("failed active-role restoration is loud and never reports the restarted coo
|
|
|
1066
1056
|
await fs.rm(fixture.activationGatePath)
|
|
1067
1057
|
const coordinator = statusRelease(daemon, "v1").processes.find((entry) => entry.id === "beacon")
|
|
1068
1058
|
|
|
1069
|
-
|
|
1059
|
+
if (!coordinator?.pid) throw new Error("Missing required fixture: coordinator?.pid")
|
|
1070
1060
|
process.kill(-coordinator.pid, "SIGKILL")
|
|
1071
1061
|
await waitFor(() => {
|
|
1072
1062
|
const status = statusRelease(daemon, "v1").processes.find((entry) => entry.id === "beacon")
|
|
@@ -1076,9 +1066,10 @@ test("failed active-role restoration is loud and never reports the restarted coo
|
|
|
1076
1066
|
|
|
1077
1067
|
const failed = statusRelease(daemon, "v1").processes.find((entry) => entry.id === "beacon")
|
|
1078
1068
|
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1069
|
+
expect(failed?.lifecycleRole).toBe("active")
|
|
1070
|
+
// Role restoration failure must not create an internal retry loop.
|
|
1071
|
+
expect(failed?.restarts).toBe(1)
|
|
1072
|
+
expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1"])
|
|
1082
1073
|
} finally {
|
|
1083
1074
|
await daemon.shutdown()
|
|
1084
1075
|
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
@@ -1095,28 +1086,31 @@ test("retired generation coordinator remains fenced after exit", async () => {
|
|
|
1095
1086
|
await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
|
|
1096
1087
|
socket = await openWebSocket(daemon)
|
|
1097
1088
|
await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
|
|
1089
|
+
await waitFor(() => statusRelease(daemon, "v1").processes.find((entry) => entry.id === "beacon")?.lifecycleRole === "retired")
|
|
1098
1090
|
const coordinator = statusRelease(daemon, "v1").processes.find((entry) => entry.id === "beacon")
|
|
1099
1091
|
const coordinatorProcess = daemon.releases.get("v1")?.getProcess("beacon")
|
|
1100
1092
|
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1093
|
+
if (!coordinator?.pid) throw new Error("Missing required fixture: coordinator?.pid")
|
|
1094
|
+
if (!coordinatorProcess) throw new Error("Missing required fixture: coordinatorProcess")
|
|
1095
|
+
expect(coordinator.lifecycleRole).toBe("retired")
|
|
1096
|
+
// Retirement refresh must preserve exact guardian event routing.
|
|
1097
|
+
expect(daemon.guardian?.processes.get("release:v1:beacon")).toBe(coordinatorProcess)
|
|
1105
1098
|
const exited = once(coordinatorProcess, "exit")
|
|
1106
1099
|
|
|
1107
1100
|
process.kill(-coordinator.pid, "SIGKILL")
|
|
1108
1101
|
const [exit] = await exited
|
|
1109
1102
|
const stopped = coordinatorProcess.status()
|
|
1110
1103
|
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
|
|
1104
|
+
expect(exit.code).toBe(null)
|
|
1105
|
+
expect(exit.id).toBe("beacon")
|
|
1106
|
+
expect(exit.signal).toBe("SIGKILL")
|
|
1107
|
+
expect(await lifecycleEvents(fixture.lifecycleLogPath)).toEqual(["activate:v1", "retire:v1", "activate:v2"])
|
|
1108
|
+
expect(stopped.lifecycleRole).toBe("retired")
|
|
1109
|
+
expect(stopped.pid).toBe(undefined)
|
|
1110
|
+
expect(stopped.restarts).toBe(0)
|
|
1111
|
+
expect(stopped.state).toBe("stopped")
|
|
1112
|
+
// Retired process must not queue a restart.
|
|
1113
|
+
expect(coordinatorProcess.restartTimer).toBe(undefined)
|
|
1120
1114
|
} finally {
|
|
1121
1115
|
socket?.close()
|
|
1122
1116
|
await daemon.shutdown()
|
|
@@ -1141,14 +1135,14 @@ test("multiple retired jobs generations keep distinct endpoints and live referen
|
|
|
1141
1135
|
const status = daemon.status()
|
|
1142
1136
|
const generations = ["v1", "v2", "v3"].map((releaseId) => statusRelease(daemon, releaseId))
|
|
1143
1137
|
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1138
|
+
expect(generations.map((release) => release.state)).toEqual(["draining", "draining", "active"])
|
|
1139
|
+
expect(new Set(generations.map((release) => release.ports.beacon)).size).toBe(3)
|
|
1140
|
+
expect(status.releaseReferences.map((reference) => reference.releaseId)).toEqual(["v1", "v2", "v3"])
|
|
1141
|
+
expect(status.releaseReferences.map((reference) => reference.releasePath)).toEqual(["v1", "v2", "v3"].map((releaseId) => path.join(fixture.root, releaseId)))
|
|
1148
1142
|
|
|
1149
1143
|
for (const socket of sockets.splice(0)) socket.close()
|
|
1150
1144
|
await waitFor(() => statusRelease(daemon, "v1").state === "stopped" && statusRelease(daemon, "v2").state === "stopped")
|
|
1151
|
-
|
|
1145
|
+
expect(daemon.status().releaseReferences.map((reference) => reference.releaseId)).toEqual(["v3"])
|
|
1152
1146
|
} finally {
|
|
1153
1147
|
for (const socket of sockets) socket.close()
|
|
1154
1148
|
await daemon.shutdown()
|
|
@@ -1165,16 +1159,17 @@ test("candidate failure preserves old traffic, jobs generation, endpoint, and re
|
|
|
1165
1159
|
const before = statusRelease(daemon, "good")
|
|
1166
1160
|
const beforeService = before.processes.find((processStatus) => processStatus.id === "beacon")
|
|
1167
1161
|
|
|
1168
|
-
await
|
|
1162
|
+
await expect(daemon.deploy({releaseId: "bad", releasePath: fixture.root, revision: "bad"})).rejects.toThrow(/Health check failed/)
|
|
1169
1163
|
|
|
1170
1164
|
const after = statusRelease(daemon, "good")
|
|
1171
1165
|
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1166
|
+
expect(await fetchText(daemon, "/release")).toBe("good")
|
|
1167
|
+
expect(after.state).toBe("active")
|
|
1168
|
+
expect(after.ports.beacon).toBe(before.ports.beacon)
|
|
1169
|
+
expect(after.processes.find((processStatus) => processStatus.id === "beacon")?.pid).toBe(beforeService?.pid)
|
|
1170
|
+
// Candidate cleanup must not quiesce the active generation.
|
|
1171
|
+
expect((await fs.readFile(fixture.serviceQuietPath, "utf8")).includes("good\n")).toBe(false)
|
|
1172
|
+
expect(daemon.status().releaseReferences.map((reference) => reference.releaseId)).toEqual(["good"])
|
|
1178
1173
|
} finally {
|
|
1179
1174
|
await daemon.shutdown()
|
|
1180
1175
|
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
@@ -1194,12 +1189,12 @@ test("handoff-service quiescence failure is visible and leaves the generation al
|
|
|
1194
1189
|
|
|
1195
1190
|
const retired = statusRelease(daemon, "v1")
|
|
1196
1191
|
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1192
|
+
expect(retired.state).toBe("draining")
|
|
1193
|
+
expect(String(retired.retirementError)).toMatch(/quiet command exited non-zero.*23/)
|
|
1194
|
+
expect(retired.processes.find((processStatus) => processStatus.id === "beacon")?.state).toBe("stopping")
|
|
1195
|
+
expect(retired.processes.find((processStatus) => processStatus.id === "worker")?.state).not.toBe("stopped")
|
|
1196
|
+
expect(logs.some((entry) => entry.message === "release retirement quiescence failed" && entry.data?.releaseId === "v1")).toBeTruthy()
|
|
1197
|
+
expect(result.retirement).toEqual({error: retired.retirementError, releaseId: "v1", status: "quiescence_failed"})
|
|
1203
1198
|
} finally {
|
|
1204
1199
|
await daemon.shutdown()
|
|
1205
1200
|
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
@@ -1215,21 +1210,21 @@ test("a replicated companion starts one instance per replica, and restart target
|
|
|
1215
1210
|
|
|
1216
1211
|
const release = daemon.status().releases.find((candidate) => candidate.state === "active")
|
|
1217
1212
|
|
|
1218
|
-
|
|
1213
|
+
if (!release) throw new Error("Missing required fixture: release")
|
|
1219
1214
|
|
|
1220
1215
|
const workerIds = release.processes.filter((processStatus) => processStatus.id.startsWith("worker")).map((processStatus) => processStatus.id).sort()
|
|
1221
1216
|
|
|
1222
|
-
|
|
1217
|
+
expect(workerIds).toEqual(["worker#0", "worker#1", "worker#2"])
|
|
1223
1218
|
|
|
1224
1219
|
// A specific replica id restarts only that replica.
|
|
1225
1220
|
const one = await daemon.restartProcesses({processId: "worker#1"})
|
|
1226
1221
|
|
|
1227
|
-
|
|
1222
|
+
expect(one.restarted).toEqual(["worker#1"])
|
|
1228
1223
|
|
|
1229
1224
|
// The base id restarts every replica.
|
|
1230
1225
|
const all = /** @type {string[]} */ ((await daemon.restartProcesses({processId: "worker"})).restarted)
|
|
1231
1226
|
|
|
1232
|
-
|
|
1227
|
+
expect([...all].sort()).toEqual(["worker#0", "worker#1", "worker#2"])
|
|
1233
1228
|
} finally {
|
|
1234
1229
|
await daemon.shutdown()
|
|
1235
1230
|
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
@@ -1246,12 +1241,12 @@ test("restart bounces a single process by id", async () => {
|
|
|
1246
1241
|
const before = pidsById(daemon.status())
|
|
1247
1242
|
const result = await daemon.restartProcesses({processId: "beacon"})
|
|
1248
1243
|
|
|
1249
|
-
|
|
1244
|
+
expect(result.restarted).toEqual(["beacon"])
|
|
1250
1245
|
|
|
1251
1246
|
const after = pidsById(daemon.status())
|
|
1252
1247
|
|
|
1253
|
-
|
|
1254
|
-
|
|
1248
|
+
expect(before.beacon && after.beacon).toBeTruthy()
|
|
1249
|
+
expect(after.beacon).not.toBe(before.beacon)
|
|
1255
1250
|
} finally {
|
|
1256
1251
|
await daemon.shutdown()
|
|
1257
1252
|
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
@@ -1269,14 +1264,15 @@ test("restart with no selector bounces every non-proxied process but not the pro
|
|
|
1269
1264
|
const result = await daemon.restartProcesses()
|
|
1270
1265
|
const restarted = /** @type {string[]} */ (result.restarted)
|
|
1271
1266
|
|
|
1272
|
-
|
|
1267
|
+
expect([...restarted].sort()).toEqual(["beacon", "jobs-main", "worker"])
|
|
1273
1268
|
|
|
1274
1269
|
const after = pidsById(daemon.status())
|
|
1275
1270
|
|
|
1276
|
-
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1271
|
+
// Proxied process should not be restarted.
|
|
1272
|
+
expect(after.web).toBe(before.web)
|
|
1273
|
+
expect(after.beacon).not.toBe(before.beacon)
|
|
1274
|
+
expect(after["jobs-main"]).not.toBe(before["jobs-main"])
|
|
1275
|
+
expect(after.worker).not.toBe(before.worker)
|
|
1280
1276
|
} finally {
|
|
1281
1277
|
await daemon.shutdown()
|
|
1282
1278
|
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
@@ -1293,12 +1289,13 @@ test("restart --policy targets only processes with that policy", async () => {
|
|
|
1293
1289
|
const before = pidsById(daemon.status())
|
|
1294
1290
|
const result = await daemon.restartProcesses({policy: "companion"})
|
|
1295
1291
|
|
|
1296
|
-
|
|
1292
|
+
expect(result.restarted).toEqual(["worker"])
|
|
1297
1293
|
|
|
1298
1294
|
const after = pidsById(daemon.status())
|
|
1299
1295
|
|
|
1300
|
-
|
|
1301
|
-
|
|
1296
|
+
expect(after.worker).not.toBe(before.worker)
|
|
1297
|
+
// The service should be left running.
|
|
1298
|
+
expect(after.beacon).toBe(before.beacon)
|
|
1302
1299
|
} finally {
|
|
1303
1300
|
await daemon.shutdown()
|
|
1304
1301
|
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
@@ -1312,9 +1309,9 @@ test("restart refuses the proxied process and reports unknown ids", async () =>
|
|
|
1312
1309
|
try {
|
|
1313
1310
|
await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
|
|
1314
1311
|
|
|
1315
|
-
await
|
|
1316
|
-
await
|
|
1317
|
-
await
|
|
1312
|
+
await expect(daemon.restartProcesses({processId: "web"})).rejects.toThrow(/proxied process cannot be restarted/)
|
|
1313
|
+
await expect(daemon.restartProcesses({policy: "proxied"})).rejects.toThrow(/proxied process cannot be restarted/)
|
|
1314
|
+
await expect(daemon.restartProcesses({processId: "missing"})).rejects.toThrow(/No managed process with id "missing"/)
|
|
1318
1315
|
} finally {
|
|
1319
1316
|
await daemon.shutdown()
|
|
1320
1317
|
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
@@ -1331,15 +1328,15 @@ test("restart revives a stopped process instead of erroring", async () => {
|
|
|
1331
1328
|
// Simulate the worker having exited (e.g. crashed and exhausted its restart budget).
|
|
1332
1329
|
const worker = daemon.activeRelease?.getProcess("worker")
|
|
1333
1330
|
|
|
1334
|
-
|
|
1331
|
+
if (!worker) throw new Error("worker process should exist")
|
|
1335
1332
|
await worker.stop()
|
|
1336
|
-
|
|
1333
|
+
expect(worker.status().state).toBe("stopped")
|
|
1337
1334
|
|
|
1338
1335
|
const result = await daemon.restartProcesses({processId: "worker"})
|
|
1339
1336
|
|
|
1340
|
-
|
|
1341
|
-
|
|
1342
|
-
|
|
1337
|
+
expect(result.restarted).toEqual(["worker"])
|
|
1338
|
+
expect(worker.status().state).toBe("running")
|
|
1339
|
+
expect(worker.status().pid).toBeTruthy()
|
|
1343
1340
|
} finally {
|
|
1344
1341
|
await daemon.shutdown()
|
|
1345
1342
|
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
@@ -1359,8 +1356,8 @@ test("the restart control command bounces a process over the socket", async () =
|
|
|
1359
1356
|
path: fixture.config.control.path
|
|
1360
1357
|
})
|
|
1361
1358
|
|
|
1362
|
-
|
|
1363
|
-
|
|
1359
|
+
expect(response.restarted).toEqual(["beacon"])
|
|
1360
|
+
expect(pidsById(daemon.status()).beacon).not.toBe(before.beacon)
|
|
1364
1361
|
} finally {
|
|
1365
1362
|
await daemon.shutdown()
|
|
1366
1363
|
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
@@ -1376,15 +1373,15 @@ test("status and events distinguish deploy starts from manual restarts", async (
|
|
|
1376
1373
|
|
|
1377
1374
|
const afterDeploy = daemon.status().services.find((service) => service.id === "beacon")
|
|
1378
1375
|
|
|
1379
|
-
|
|
1380
|
-
|
|
1376
|
+
if (!afterDeploy) throw new Error("Missing required fixture: afterDeploy")
|
|
1377
|
+
expect(afterDeploy.process.lastStartReason).toBe("deploy")
|
|
1381
1378
|
|
|
1382
1379
|
await daemon.restartProcesses({processId: "beacon"})
|
|
1383
1380
|
|
|
1384
1381
|
const afterRestart = daemon.status().services.find((service) => service.id === "beacon")
|
|
1385
1382
|
|
|
1386
|
-
|
|
1387
|
-
|
|
1383
|
+
if (!afterRestart) throw new Error("Missing required fixture: afterRestart")
|
|
1384
|
+
expect(afterRestart.process.lastStartReason).toBe("manual")
|
|
1388
1385
|
|
|
1389
1386
|
const events = /** @type {import("../src/event-log.js").DaemonEvent[]} */ ((await sendControlCommand({
|
|
1390
1387
|
command: {command: "events"},
|
|
@@ -1392,8 +1389,8 @@ test("status and events distinguish deploy starts from manual restarts", async (
|
|
|
1392
1389
|
})).events)
|
|
1393
1390
|
const startReasons = events.filter((event) => event.message === "process started").map((event) => event.data.reason)
|
|
1394
1391
|
|
|
1395
|
-
|
|
1396
|
-
|
|
1392
|
+
expect({value: Boolean(startReasons.includes("deploy")), context: JSON.stringify(startReasons)}).toMatchObject({value: true})
|
|
1393
|
+
expect({value: Boolean(startReasons.includes("manual")), context: JSON.stringify(startReasons)}).toMatchObject({value: true})
|
|
1397
1394
|
} finally {
|
|
1398
1395
|
await daemon.shutdown()
|
|
1399
1396
|
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
@@ -1422,7 +1419,8 @@ test("persists daemon state to statePath and removes it on a clean shutdown", as
|
|
|
1422
1419
|
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
1423
1420
|
}
|
|
1424
1421
|
|
|
1425
|
-
|
|
1422
|
+
// State file removed on clean shutdown.
|
|
1423
|
+
expect(stateAfterShutdown).toBe(undefined)
|
|
1426
1424
|
})
|
|
1427
1425
|
|
|
1428
1426
|
test("persisted daemon state excludes process commands, environment values, and output", async () => {
|
|
@@ -1430,7 +1428,7 @@ test("persisted daemon state excludes process commands, environment values, and
|
|
|
1430
1428
|
const fixture = await createFixture({persistState: true})
|
|
1431
1429
|
const web = fixture.config.processes.find((processConfig) => processConfig.id === "web")
|
|
1432
1430
|
|
|
1433
|
-
|
|
1431
|
+
if (!web) throw new Error("Missing required fixture: web")
|
|
1434
1432
|
web.env.ROLLBRIDGE_TEST_SECRET = secret
|
|
1435
1433
|
web.command = `${JSON.stringify(process.execPath)} -e ${JSON.stringify(`console.log(process.env.ROLLBRIDGE_TEST_SECRET); import(${JSON.stringify(pathToFileURL(dummyAppPath).href)})`)}`
|
|
1436
1434
|
|
|
@@ -1440,20 +1438,21 @@ test("persisted daemon state excludes process commands, environment values, and
|
|
|
1440
1438
|
await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
|
|
1441
1439
|
const webProcess = daemon.activeRelease?.getProcess("web")
|
|
1442
1440
|
|
|
1443
|
-
|
|
1441
|
+
if (!webProcess) throw new Error("Missing required fixture: webProcess")
|
|
1444
1442
|
await recordedLogLine(webProcess, secret)
|
|
1445
|
-
|
|
1443
|
+
// Secret output must be retained before persistence.
|
|
1444
|
+
expect(webProcess.status().logs.some((entry) => entry.line === secret)).toBe(true)
|
|
1446
1445
|
|
|
1447
1446
|
daemon.persistState()
|
|
1448
1447
|
await waitFor(async () => (await fs.readFile(fixture.statePath, "utf8")).includes('"activeReleaseId": "v1"'))
|
|
1449
1448
|
|
|
1450
1449
|
const persisted = await fs.readFile(fixture.statePath, "utf8")
|
|
1451
1450
|
|
|
1452
|
-
|
|
1453
|
-
|
|
1454
|
-
|
|
1455
|
-
|
|
1456
|
-
|
|
1451
|
+
expect(persisted).not.toMatch(/state-secret-value/)
|
|
1452
|
+
expect(persisted).not.toMatch(/ROLLBRIDGE_TEST_SECRET/)
|
|
1453
|
+
expect(persisted).not.toMatch(/"command"/)
|
|
1454
|
+
expect(persisted).not.toMatch(/"logs"/)
|
|
1455
|
+
expect(liveProcesses(JSON.parse(persisted), () => true).map(({id, releaseId}) => ({id, releaseId}))).toEqual([{id: "web", releaseId: "v1"}])
|
|
1457
1456
|
} finally {
|
|
1458
1457
|
await daemon.shutdown()
|
|
1459
1458
|
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
@@ -1470,7 +1469,8 @@ test("a clean shutdown clears the state file even when a persist write is in fli
|
|
|
1470
1469
|
// Shut down immediately — the deploy's fire-and-forget persist may still be in flight.
|
|
1471
1470
|
await daemon.shutdown()
|
|
1472
1471
|
|
|
1473
|
-
|
|
1472
|
+
// State file must not be recreated by an in-flight write.
|
|
1473
|
+
expect(await readState(fixture.statePath)).toBe(undefined)
|
|
1474
1474
|
} finally {
|
|
1475
1475
|
if (!daemon.stopping) await daemon.shutdown()
|
|
1476
1476
|
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
@@ -1507,7 +1507,7 @@ test("reports orphaned managed processes from a previous daemon's state", async
|
|
|
1507
1507
|
|
|
1508
1508
|
await daemon.reportOrphans()
|
|
1509
1509
|
|
|
1510
|
-
|
|
1510
|
+
expect({value: Boolean(logs.some((entry) => entry.message === "orphaned managed process detected" && entry.data.pid === leftover.pid)), context: JSON.stringify(logs)}).toMatchObject({value: true})
|
|
1511
1511
|
|
|
1512
1512
|
// A dead pid is not reported.
|
|
1513
1513
|
logs.length = 0
|
|
@@ -1519,7 +1519,7 @@ test("reports orphaned managed processes from a previous daemon's state", async
|
|
|
1519
1519
|
})
|
|
1520
1520
|
await daemon.reportOrphans()
|
|
1521
1521
|
|
|
1522
|
-
|
|
1522
|
+
expect(!logs.some((entry) => entry.message === "orphaned managed process detected")).toBeTruthy()
|
|
1523
1523
|
} finally {
|
|
1524
1524
|
leftover.kill("SIGKILL")
|
|
1525
1525
|
await fs.rm(dir, {force: true, recursive: true})
|
|
@@ -1554,16 +1554,16 @@ test("status surfaces still-alive orphaned processes from a previous daemon and
|
|
|
1554
1554
|
await daemon.reportOrphans()
|
|
1555
1555
|
|
|
1556
1556
|
// status reflects the still-running child even though the daemon cannot re-manage it.
|
|
1557
|
-
|
|
1557
|
+
expect(daemon.status().orphans).toEqual([{id: "worker", pid: leftover.pid, releaseId: "v1"}])
|
|
1558
1558
|
|
|
1559
1559
|
// Once the leftover is stopped, status re-checks liveness and drops it.
|
|
1560
1560
|
leftover.kill("SIGKILL")
|
|
1561
1561
|
await waitFor(() => daemon.status().orphans.length === 0)
|
|
1562
|
-
|
|
1562
|
+
expect(daemon.status().orphans).toEqual([])
|
|
1563
1563
|
|
|
1564
1564
|
// The dead entry is pruned from the underlying list, not merely filtered, so a recycled pid
|
|
1565
1565
|
// can't resurrect a cleared orphan.
|
|
1566
|
-
|
|
1566
|
+
expect(daemon.orphans).toEqual([])
|
|
1567
1567
|
} finally {
|
|
1568
1568
|
leftover.kill("SIGKILL")
|
|
1569
1569
|
await fs.rm(dir, {force: true, recursive: true})
|
|
@@ -1584,14 +1584,14 @@ test("the daemon records a structured event history served by the events command
|
|
|
1584
1584
|
const events = /** @type {import("../src/event-log.js").DaemonEvent[]} */ (response.events)
|
|
1585
1585
|
const messages = events.map((event) => event.message)
|
|
1586
1586
|
|
|
1587
|
-
|
|
1588
|
-
|
|
1587
|
+
expect({value: Boolean(messages.includes("deploy starting")), context: JSON.stringify(messages)}).toMatchObject({value: true})
|
|
1588
|
+
expect({value: Boolean(messages.includes("traffic switched")), context: JSON.stringify(messages)}).toMatchObject({value: true})
|
|
1589
1589
|
|
|
1590
1590
|
const switched = events.find((event) => event.message === "traffic switched")
|
|
1591
1591
|
|
|
1592
|
-
|
|
1593
|
-
|
|
1594
|
-
|
|
1592
|
+
if (!switched) throw new Error("Missing required fixture: switched")
|
|
1593
|
+
expect(switched.data.releaseId).toBe("v1")
|
|
1594
|
+
expect(switched.at).toMatch(/^\d{4}-\d{2}-\d{2}T.*Z$/)
|
|
1595
1595
|
} finally {
|
|
1596
1596
|
await daemon.shutdown()
|
|
1597
1597
|
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
@@ -1606,32 +1606,32 @@ test("the events command honors --limit and records failed commands", async () =
|
|
|
1606
1606
|
await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
|
|
1607
1607
|
|
|
1608
1608
|
// An unknown command is rejected and recorded as a "command failed" event.
|
|
1609
|
-
await
|
|
1609
|
+
await expect(sendControlCommand({
|
|
1610
1610
|
command: {command: "bogus"},
|
|
1611
1611
|
path: fixture.config.control.path
|
|
1612
|
-
}))
|
|
1612
|
+
})).rejects.toThrow()
|
|
1613
1613
|
|
|
1614
1614
|
const all = /** @type {import("../src/event-log.js").DaemonEvent[]} */ ((await sendControlCommand({
|
|
1615
1615
|
command: {command: "events"},
|
|
1616
1616
|
path: fixture.config.control.path
|
|
1617
1617
|
})).events)
|
|
1618
1618
|
|
|
1619
|
-
|
|
1619
|
+
expect(all.some((event) => event.message === "command failed")).toBeTruthy()
|
|
1620
1620
|
|
|
1621
1621
|
const limited = /** @type {import("../src/event-log.js").DaemonEvent[]} */ ((await sendControlCommand({
|
|
1622
1622
|
command: {command: "events", limit: 1},
|
|
1623
1623
|
path: fixture.config.control.path
|
|
1624
1624
|
})).events)
|
|
1625
1625
|
|
|
1626
|
-
|
|
1627
|
-
|
|
1626
|
+
expect(limited.length).toBe(1)
|
|
1627
|
+
expect(limited[0]).toEqual(all[all.length - 1])
|
|
1628
1628
|
} finally {
|
|
1629
1629
|
await daemon.shutdown()
|
|
1630
1630
|
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
1631
1631
|
}
|
|
1632
1632
|
})
|
|
1633
1633
|
|
|
1634
|
-
|
|
1634
|
+
linuxTest("a process over its memory limit is restarted with reason memory", async () => {
|
|
1635
1635
|
const limitBytes = 64 * 1024 * 1024
|
|
1636
1636
|
const fixture = await createFixture({memoryLimitBytes: limitBytes})
|
|
1637
1637
|
const daemon = await startDaemon(fixture.config)
|
|
@@ -1644,17 +1644,17 @@ test("a process over its memory limit is restarted with reason memory", {skip: p
|
|
|
1644
1644
|
|
|
1645
1645
|
const hog = activeProcessStatus(daemon, "hog")
|
|
1646
1646
|
|
|
1647
|
-
|
|
1648
|
-
|
|
1649
|
-
|
|
1650
|
-
|
|
1647
|
+
if (!hog) throw new Error("hog process should be present")
|
|
1648
|
+
expect({value: Boolean(hog.memoryRestarts >= 1), context: `expected a memory restart, got ${hog.memoryRestarts}`}).toMatchObject({value: true})
|
|
1649
|
+
expect(hog.lastStartReason).toBe("memory")
|
|
1650
|
+
expect(typeof hog.lastMemoryRestartAt).toBe("string")
|
|
1651
1651
|
|
|
1652
1652
|
// Keep the replacement alive long enough to observe its next monitor sample. The fixture
|
|
1653
1653
|
// remains over the configured limit after every launch, otherwise it can restart again and
|
|
1654
1654
|
// clear rssBytes/children before this polling loop observes them on slower CI runners.
|
|
1655
1655
|
const hogProcess = daemon.activeRelease?.processes.get("hog")
|
|
1656
1656
|
|
|
1657
|
-
|
|
1657
|
+
if (!hogProcess?.memory) throw new Error("Missing required fixture: hogProcess?.memory")
|
|
1658
1658
|
hogProcess.memory.limitBytes = Number.MAX_SAFE_INTEGER
|
|
1659
1659
|
|
|
1660
1660
|
// rssBytes is sampled on the monitor's interval; wait for a measurement of the running process.
|
|
@@ -1667,9 +1667,9 @@ test("a process over its memory limit is restarted with reason memory", {skip: p
|
|
|
1667
1667
|
// The same monitor sample reports the process tree.
|
|
1668
1668
|
const monitored = activeProcessStatus(daemon, "hog")
|
|
1669
1669
|
|
|
1670
|
-
|
|
1671
|
-
|
|
1672
|
-
|
|
1670
|
+
if (!monitored) throw new Error("Missing required fixture: monitored")
|
|
1671
|
+
expect(monitored.children.length >= 1).toBe(true)
|
|
1672
|
+
expect(monitored.children.some((child) => typeof child.rssBytes === "number" && child.rssBytes > 0)).toBeTruthy()
|
|
1673
1673
|
} finally {
|
|
1674
1674
|
await daemon.shutdown()
|
|
1675
1675
|
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
@@ -1684,14 +1684,14 @@ test("rollback re-activates the previous release and switches traffic back", asy
|
|
|
1684
1684
|
await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
|
|
1685
1685
|
await daemon.deploy({releaseId: "v2", releasePath: fixture.root, revision: "v2"})
|
|
1686
1686
|
|
|
1687
|
-
|
|
1687
|
+
expect(await fetchText(daemon, "/release")).toBe("v2")
|
|
1688
1688
|
|
|
1689
1689
|
const result = await daemon.rollback()
|
|
1690
1690
|
|
|
1691
|
-
|
|
1692
|
-
|
|
1693
|
-
|
|
1694
|
-
|
|
1691
|
+
expect(result.activeReleaseId).toBe("v1")
|
|
1692
|
+
expect(result.previousReleaseId).toBe("v2")
|
|
1693
|
+
expect(daemon.status().activeReleaseId).toBe("v1")
|
|
1694
|
+
expect(await fetchText(daemon, "/release")).toBe("v1")
|
|
1695
1695
|
} finally {
|
|
1696
1696
|
await daemon.shutdown()
|
|
1697
1697
|
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
@@ -1709,8 +1709,8 @@ test("rollback --release-id targets a specific retained release", async () => {
|
|
|
1709
1709
|
|
|
1710
1710
|
const result = await daemon.rollback({releaseId: "v1"})
|
|
1711
1711
|
|
|
1712
|
-
|
|
1713
|
-
|
|
1712
|
+
expect(result.activeReleaseId).toBe("v1")
|
|
1713
|
+
expect(await fetchText(daemon, "/release")).toBe("v1")
|
|
1714
1714
|
} finally {
|
|
1715
1715
|
await daemon.shutdown()
|
|
1716
1716
|
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
@@ -1724,9 +1724,9 @@ test("rollback rejects no-previous, unknown, and already-active targets", async
|
|
|
1724
1724
|
try {
|
|
1725
1725
|
await daemon.deploy({releaseId: "v1", releasePath: fixture.root, revision: "v1"})
|
|
1726
1726
|
|
|
1727
|
-
await
|
|
1728
|
-
await
|
|
1729
|
-
await
|
|
1727
|
+
await expect(daemon.rollback()).rejects.toThrow(/No previous release/)
|
|
1728
|
+
await expect(daemon.rollback({releaseId: "v1"})).rejects.toThrow(/already active/)
|
|
1729
|
+
await expect(daemon.rollback({releaseId: "nope"})).rejects.toThrow(/No retained release "nope"/)
|
|
1730
1730
|
} finally {
|
|
1731
1731
|
await daemon.shutdown()
|
|
1732
1732
|
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
@@ -1748,17 +1748,17 @@ test("rollback to a still-draining release stops the old instance instead of orp
|
|
|
1748
1748
|
|
|
1749
1749
|
const draining = statusRelease(daemon, "v1")
|
|
1750
1750
|
|
|
1751
|
-
|
|
1751
|
+
expect(draining.state).toBe("draining")
|
|
1752
1752
|
|
|
1753
1753
|
const oldWebPid = draining.processes.find((processStatus) => processStatus.id === "web")?.pid
|
|
1754
1754
|
|
|
1755
|
-
|
|
1755
|
+
if (!oldWebPid) throw new Error("the draining release should have a running web process")
|
|
1756
1756
|
|
|
1757
1757
|
await daemon.rollback({releaseId: "v1"})
|
|
1758
1758
|
|
|
1759
|
-
|
|
1759
|
+
expect(daemon.status().activeReleaseId).toBe("v1")
|
|
1760
1760
|
// The old draining instance was stopped before its id was reused, so its process is gone.
|
|
1761
|
-
|
|
1761
|
+
await expect(() => process.kill(/** @type {number} */ (oldWebPid), 0)).toThrow(/ESRCH/)
|
|
1762
1762
|
} finally {
|
|
1763
1763
|
if (socket) socket.close()
|
|
1764
1764
|
await daemon.shutdown()
|
|
@@ -1779,8 +1779,8 @@ test("the rollback control command switches traffic over the socket", async () =
|
|
|
1779
1779
|
path: fixture.config.control.path
|
|
1780
1780
|
})
|
|
1781
1781
|
|
|
1782
|
-
|
|
1783
|
-
|
|
1782
|
+
expect(response.activeReleaseId).toBe("v1")
|
|
1783
|
+
expect(await fetchText(daemon, "/release")).toBe("v1")
|
|
1784
1784
|
} finally {
|
|
1785
1785
|
await daemon.shutdown()
|
|
1786
1786
|
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
@@ -1807,8 +1807,8 @@ test("control socket accepts deploy and status commands", async () => {
|
|
|
1807
1807
|
path: fixture.config.control.path
|
|
1808
1808
|
})
|
|
1809
1809
|
|
|
1810
|
-
|
|
1811
|
-
|
|
1810
|
+
expect(status.activeReleaseId).toBe("control-v1")
|
|
1811
|
+
expect(await fetchText(daemon, "/release")).toBe("control-v1")
|
|
1812
1812
|
} finally {
|
|
1813
1813
|
await daemon.shutdown()
|
|
1814
1814
|
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
@@ -1824,28 +1824,23 @@ test("starting a second daemon on a live control socket reports the running daem
|
|
|
1824
1824
|
|
|
1825
1825
|
const second = new RollbridgeDaemon({config: fixture.config, logger: () => {}})
|
|
1826
1826
|
|
|
1827
|
-
|
|
1828
|
-
() => second.prepareControlSocketPath(),
|
|
1829
|
-
(error) => {
|
|
1830
|
-
assert.ok(error instanceof Error)
|
|
1831
|
-
assert.match(error.message, /A Rollbridge daemon for application "rollbridge-test" is already running/)
|
|
1832
|
-
assert.match(error.message, /active release: v1/)
|
|
1833
|
-
assert.match(error.message, /rollbridge shutdown/)
|
|
1827
|
+
const preparation = second.prepareControlSocketPath()
|
|
1834
1828
|
|
|
1835
|
-
|
|
1836
|
-
|
|
1837
|
-
)
|
|
1829
|
+
await expect(preparation).rejects.toBeInstanceOf(Error)
|
|
1830
|
+
await expect(preparation).rejects.toMatchObject({message: expect.stringMatching(/A Rollbridge daemon for application "rollbridge-test" is already running/)})
|
|
1831
|
+
await expect(preparation).rejects.toMatchObject({message: expect.stringMatching(/active release: v1/)})
|
|
1832
|
+
await expect(preparation).rejects.toMatchObject({message: expect.stringMatching(/rollbridge shutdown/)})
|
|
1838
1833
|
|
|
1839
1834
|
// The original daemon keeps its socket and still answers control commands.
|
|
1840
1835
|
const status = await sendControlCommand({command: {command: "status"}, path: fixture.config.control.path})
|
|
1841
|
-
|
|
1836
|
+
expect(status.application).toBe("rollbridge-test")
|
|
1842
1837
|
} finally {
|
|
1843
1838
|
await daemon.shutdown()
|
|
1844
1839
|
await fs.rm(fixture.root, {force: true, recursive: true})
|
|
1845
1840
|
}
|
|
1846
1841
|
})
|
|
1847
1842
|
|
|
1848
|
-
|
|
1843
|
+
linuxTest("the daemon applies control.owner and control.group to the bound socket", async () => {
|
|
1849
1844
|
const root = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-test-"))
|
|
1850
1845
|
const socketPath = path.join(root, "rollbridge.sock")
|
|
1851
1846
|
const {uid, username} = os.userInfo()
|
|
@@ -1865,8 +1860,8 @@ test("the daemon applies control.owner and control.group to the bound socket", {
|
|
|
1865
1860
|
|
|
1866
1861
|
const stats = await fs.stat(socketPath)
|
|
1867
1862
|
|
|
1868
|
-
|
|
1869
|
-
|
|
1863
|
+
expect(stats.uid).toBe(uid)
|
|
1864
|
+
expect(stats.gid).toBe(gid)
|
|
1870
1865
|
} finally {
|
|
1871
1866
|
await daemon.shutdown()
|
|
1872
1867
|
await fs.rm(root, {force: true, recursive: true})
|
|
@@ -1895,10 +1890,7 @@ test("a control socket held by a non-Rollbridge process reports a generic confli
|
|
|
1895
1890
|
const daemon = new RollbridgeDaemon({config, logger: () => {}})
|
|
1896
1891
|
|
|
1897
1892
|
try {
|
|
1898
|
-
await
|
|
1899
|
-
() => daemon.prepareControlSocketPath(),
|
|
1900
|
-
/The control socket .* is already in use by another process/
|
|
1901
|
-
)
|
|
1893
|
+
await expect(daemon.prepareControlSocketPath()).rejects.toThrow(/The control socket .* is already in use by another process/)
|
|
1902
1894
|
} finally {
|
|
1903
1895
|
for (const socket of connections) socket.destroy()
|
|
1904
1896
|
await new Promise((resolve) => stranger.close(() => resolve(undefined)))
|
|
@@ -1922,7 +1914,7 @@ test("applies the configured control socket permission mode", async () => {
|
|
|
1922
1914
|
try {
|
|
1923
1915
|
const stats = await fs.stat(socketPath)
|
|
1924
1916
|
|
|
1925
|
-
|
|
1917
|
+
expect(stats.mode & 0o777).toBe(0o660)
|
|
1926
1918
|
} finally {
|
|
1927
1919
|
await daemon.shutdown()
|
|
1928
1920
|
await fs.rm(root, {force: true, recursive: true})
|
|
@@ -1962,10 +1954,10 @@ test("deploy can ensure the daemon before sending the release command", async ()
|
|
|
1962
1954
|
|
|
1963
1955
|
const proxy = /** @type {{port: number}} */ (status.proxy)
|
|
1964
1956
|
|
|
1965
|
-
|
|
1966
|
-
|
|
1967
|
-
|
|
1968
|
-
|
|
1957
|
+
expect(status.activeReleaseId).toBe("ensured-v1")
|
|
1958
|
+
expect(status.bootstrap).toBe(undefined)
|
|
1959
|
+
expect(await fs.readFile(pidPath, "utf8")).toMatch(/\d+/)
|
|
1960
|
+
expect(await fetchTextFromPort(proxy.port, "/release")).toBe("ensured-v1")
|
|
1969
1961
|
} finally {
|
|
1970
1962
|
try {
|
|
1971
1963
|
await sendControlCommand({
|
|
@@ -2144,7 +2136,7 @@ async function fetchText(daemon, pathName) {
|
|
|
2144
2136
|
async function fetchTextFromPort(port, pathName) {
|
|
2145
2137
|
const response = await fetch(`http://127.0.0.1:${port}${pathName}`)
|
|
2146
2138
|
|
|
2147
|
-
|
|
2139
|
+
expect(response.status).toBe(200)
|
|
2148
2140
|
|
|
2149
2141
|
return (await response.text()).trim()
|
|
2150
2142
|
}
|
|
@@ -2173,7 +2165,7 @@ function statusRelease(daemon, releaseId) {
|
|
|
2173
2165
|
const status = daemon.status()
|
|
2174
2166
|
const release = status.releases.find((candidate) => candidate.releaseId === releaseId)
|
|
2175
2167
|
|
|
2176
|
-
|
|
2168
|
+
if (!release) throw new Error(`Release ${releaseId} should be present`)
|
|
2177
2169
|
|
|
2178
2170
|
return release
|
|
2179
2171
|
}
|
|
@@ -2293,3 +2285,4 @@ function activeProcessStatus(daemon, processId) {
|
|
|
2293
2285
|
|
|
2294
2286
|
return release ? release.processes.find((processStatus) => processStatus.id === processId) : undefined
|
|
2295
2287
|
}
|
|
2288
|
+
})
|