rollbridge 0.1.63 → 0.1.65
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/changelog.d/20261115-bound-draining-release-groups.md +62 -0
- package/docs/cli.md +44 -0
- package/docs/config.md +26 -1
- package/package.json +1 -1
- package/src/config.js +13 -2
- package/src/daemon.js +83 -1
- package/src/drain-policy.js +159 -0
- package/src/managed-process.js +99 -3
- package/src/process-guardian.js +4 -5
- package/src/process-memory.js +62 -2
- package/src/release-group.js +93 -2
- package/test/config-validation.test.js +11 -4
- package/test/drain-policy.test.js +199 -0
- package/test/managed-process.test.js +11 -1
- package/test/process-memory.test.js +47 -1
- package/test/release-group.test.js +127 -2
- package/test/rollbridge.test.js +120 -0
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
### Added
|
|
2
|
+
|
|
3
|
+
- `releaseRetention.maxConcurrentDrainingReleaseGroups` (default **12**) bounds how many
|
|
4
|
+
release groups may drain at once, and `deploy` evaluates it **before the handoff**. Every
|
|
5
|
+
activation retires one more generation, so whenever deploys arrive faster than a generation
|
|
6
|
+
drains the backlog grows monotonically. Measured on TensorBuzz production 2026-11-14/15
|
|
7
|
+
(`shared/log/rollbridge.log`, `traffic switched` → `release drained` pairs for 16 releases):
|
|
8
|
+
drain duration 18 min … 114 min, median ≈37 min (`20261114232900` 23:31:24 → 01:25:53 is
|
|
9
|
+
114m29s), against 7 traffic switches in 61 minutes on 2026-11-14 — a cadence of 3–7 minutes.
|
|
10
|
+
A supervisor tolerating one draining group is unsatisfiable at that cadence: 37 / 3 = 12 is
|
|
11
|
+
the backlog the measured median produces at the fastest measured cadence. The count includes
|
|
12
|
+
the incumbent the handoff is about to retire, because that is the only version of the number
|
|
13
|
+
a pre-handoff decision can act on. A deploy over the bound is refused before any process of
|
|
14
|
+
the candidate starts, naming each draining group and its classification:
|
|
15
|
+
`Deploy refused before activation: 2 release groups already draining (20261115063805 wedged
|
|
16
|
+
(…); 20261114232900 progressing (2 live connections still owned by the generation)).
|
|
17
|
+
Activating 20261115073717 would make 3, above the
|
|
18
|
+
releaseRetention.maxConcurrentDrainingReleaseGroups bound of 2.` `0` disables the check.
|
|
19
|
+
Nothing is queued or deferred by a timer: the refusal is an ordinary deploy failure, and the
|
|
20
|
+
incumbent keeps serving.
|
|
21
|
+
- `status` now distinguishes a drain that is still progressing from one that is wedged. Each
|
|
22
|
+
draining release reports `drain: {cpuDeltaMs, reason, sampleWindowMs, status}` with
|
|
23
|
+
`status` one of `progressing`, `wedged`, or `unproven`, and each process reports
|
|
24
|
+
`retirementTimedOut` plus `heldProcessGroup: {liveMemberCount, pgid}`. A group is `wedged`
|
|
25
|
+
only when its drain can no longer progress — no live connection, every retirement budget
|
|
26
|
+
expired, and either no runner child remains or the group's cumulative CPU (procfs
|
|
27
|
+
`utime`+`stime`, defunct members included) is flat across a window of at least 20 000 ms.
|
|
28
|
+
Every inconclusive reading reports `progressing` or `unproven`, never `wedged`: the backlog
|
|
29
|
+
this fleet actually has is honest slow drain, measured as 3 live
|
|
30
|
+
`background-jobs-runner: 1 job` children with a `DockerServerDrainFillJob` `handed_off` for
|
|
31
|
+
30 minutes, whose last job completed 6 seconds before the guardian logged `release drained`.
|
|
32
|
+
Consumers gate on that classification instead of on a raw process count.
|
|
33
|
+
|
|
34
|
+
### Fixed
|
|
35
|
+
|
|
36
|
+
- A process group that outlives its tracked process is now recorded and reported. Rollbridge
|
|
37
|
+
spawns every process `detached: true`, so the tracked pid is the group leader; when the
|
|
38
|
+
leader exits on the stop signal while a member it launched keeps running, the tracked record
|
|
39
|
+
goes `stopped` with no pid while the group still holds the generation's work. `status`
|
|
40
|
+
reports it as `heldProcessGroup`, the drain classifier observes it by group id, and the
|
|
41
|
+
guardian's provenance fence (`childRecordIsGone`, now shared with the daemon instead of
|
|
42
|
+
inlined in `process-guardian.js`) counts it as not gone. The group id is recorded by the stop
|
|
43
|
+
path that already waited for the group to leave, not by `onExit`: that hook fires for every
|
|
44
|
+
retired process, and rediscovering the group there costs a procfs scan (~200 ms measured
|
|
45
|
+
here) on every exit, including the ordinary ones where the group left cleanly.
|
|
46
|
+
- `rollback` refuses a target whose generation is recorded as drained while its processes still
|
|
47
|
+
hold live work, naming each held process and the release the guardian reports as active.
|
|
48
|
+
Re-deploying that release id re-registered its guardian process keys under a new provenance
|
|
49
|
+
while the old processes still held them, and the deploy failed partway through with
|
|
50
|
+
`Guardian provenance mismatch for release:20261115060540:background-jobs-main` (TensorBuzz
|
|
51
|
+
production, run `408591ff-e72e-4351-b15a-7ec239a62f6b`, 2026-11-15) — leaving the deploy
|
|
52
|
+
tool's `current` pointer pinned at `20261115060540` while the guardian reported
|
|
53
|
+
`20261115073717` active. The refusal happens before anything is touched and names the
|
|
54
|
+
guardian's active release, so the pointer is reconciled from `rollbridge status` instead of
|
|
55
|
+
silently diverging.
|
|
56
|
+
- Drain liveness observations are bounded: a generation that still owns live connections is
|
|
57
|
+
classified without reading procfs at all, and a new observation is taken at most once per
|
|
58
|
+
20 000 ms window per draining group (held groups re-verify at most once per 5 000 ms).
|
|
59
|
+
Sampling on every `status` read was measured here at ~200 ms per group on a host with 9 530
|
|
60
|
+
processes, enough to starve a 1 000 ms connection drain in `test/rollbridge.test.js` "a
|
|
61
|
+
nonBlockingDrain worker stops immediately while its release is still draining". No timer,
|
|
62
|
+
poll, or sweeper was added: the bound is a staleness check evaluated when `status` is read.
|
package/docs/cli.md
CHANGED
|
@@ -163,6 +163,13 @@ for old workers, jobs, or HTTP/WebSocket connections to finish. Prints
|
|
|
163
163
|
If the new release fails to start or health-check, the previous release stays
|
|
164
164
|
active and the command errors.
|
|
165
165
|
|
|
166
|
+
A deploy is also refused **before any of the candidate's processes start** when the
|
|
167
|
+
handoff would push the number of concurrently draining release groups past
|
|
168
|
+
[`releaseRetention.maxConcurrentDrainingReleaseGroups`](config.md#releaseretention) —
|
|
169
|
+
the incumbent the handoff is about to retire counts too. The incumbent keeps serving,
|
|
170
|
+
nothing of the candidate is started, and the error names every draining group with its
|
|
171
|
+
`progressing`/`wedged` classification and the bound it exceeded.
|
|
172
|
+
|
|
166
173
|
With an opt-in handoff-service `lifecycle.activateCommand`, deploy journals the
|
|
167
174
|
exact transition, waits for old retirement acknowledgement, waits for candidate
|
|
168
175
|
activation acknowledgement, and synchronously commits the active proxy target.
|
|
@@ -243,6 +250,18 @@ Errors when there is no previous release, the `--release-id` is not a retained
|
|
|
243
250
|
release, or the target is already active. Only releases Rollbridge still retains
|
|
244
251
|
(see [`releaseRetention`](config.md#releaseretention)) can be rolled back to.
|
|
245
252
|
|
|
253
|
+
It also refuses a target whose generation is recorded as drained while its
|
|
254
|
+
processes still hold live work — a process the retirement wait left alive, or a
|
|
255
|
+
process group that outlived the tracked process (`processes[].heldProcessGroup`,
|
|
256
|
+
see [`status`](#status)). Re-deploying that release id would re-register its
|
|
257
|
+
guardian process keys under a new provenance while the old processes still hold
|
|
258
|
+
them, and the guardian answers `Guardian provenance mismatch for
|
|
259
|
+
release:<id>:<key>` after the deploy has already begun. The refusal is raised
|
|
260
|
+
before anything is touched and names each held process together with the release
|
|
261
|
+
the guardian reports as active, so the deploy tool reconciles its own `current`
|
|
262
|
+
pointer with `rollbridge status` instead of restoring a generation that already
|
|
263
|
+
drained.
|
|
264
|
+
|
|
246
265
|
**Migration constraints.** Rollback only manages processes — it does **not**
|
|
247
266
|
revert database migrations or other external state. The target release's on-disk
|
|
248
267
|
directory must still exist, and its code must be compatible with the current
|
|
@@ -323,6 +342,31 @@ Memory-supervised processes also report `rssBytes`, `memoryRestarts`,
|
|
|
323
342
|
`lastMemoryRestartAt`, and `children` (the process tree: each group member's
|
|
324
343
|
`pid`, `command`, and `rssBytes`).
|
|
325
344
|
|
|
345
|
+
Every process also reports `retirementTimedOut` — `true` once the supervisor has
|
|
346
|
+
stopped waiting for it — and `heldProcessGroup` (`{pgid, liveMemberCount}`) when a
|
|
347
|
+
bounded stop left the process group running after the tracked process exited. That
|
|
348
|
+
group is the leak a `stopped` record otherwise hides: nothing in Rollbridge can
|
|
349
|
+
signal or wait for it again, yet it still holds the generation's work. It is
|
|
350
|
+
re-checked at most once every 5 seconds, and clears itself when its members leave.
|
|
351
|
+
|
|
352
|
+
Each **draining** release reports a `drain` classification — `status`, the `reason`
|
|
353
|
+
behind it, the `sampleWindowMs` it rests on, and `cpuDeltaMs`:
|
|
354
|
+
|
|
355
|
+
- `progressing` — the generation still owns live connections, a process is still
|
|
356
|
+
inside its retirement budget, or its process-group CPU advanced across the
|
|
357
|
+
sample window. A legitimate multi-hour drain reports this.
|
|
358
|
+
- `wedged` — the drain can no longer progress: no live connection, every
|
|
359
|
+
retirement budget expired, and either no runner child remains or the group's CPU
|
|
360
|
+
is flat across a window of at least 20 seconds.
|
|
361
|
+
- `unproven` — the evidence is inconclusive: a first observation, a window shorter
|
|
362
|
+
than 20 seconds, or a host where process CPU cannot be read.
|
|
363
|
+
|
|
364
|
+
`drain` is `undefined` for a release that is not draining. Gate on this
|
|
365
|
+
classification rather than on a raw process count: a `progressing` group holding
|
|
366
|
+
three live processes is a healthy drain, a `wedged` group holding one is a
|
|
367
|
+
generation that will never finish. Observations are taken at most once per window
|
|
368
|
+
per draining group, so repeated `status` reads stay cheap.
|
|
369
|
+
|
|
326
370
|
`--no-logs` omits only the captured `logs` array from release, service, and
|
|
327
371
|
singleton process statuses. Use it for bounded machine-readable lifecycle
|
|
328
372
|
attestations that do not need process output. The equivalent control request is
|
package/docs/config.md
CHANGED
|
@@ -88,10 +88,35 @@ to let a deploy group talk to the daemon.
|
|
|
88
88
|
| --- | --- | --- | --- |
|
|
89
89
|
| `releaseRetention.keep` | non-negative integer | `10` | Number of most-recent **stopped** releases the daemon keeps in memory and reports in `status`. |
|
|
90
90
|
| `releaseRetention.maxAgeMs` | non-negative number | `0` (disabled) | Also prune stopped releases older than this many milliseconds. |
|
|
91
|
+
| `releaseRetention.maxConcurrentDrainingReleaseGroups` | non-negative integer | `12` | How many release groups may drain at once. A deploy that would push the count past this bound is **refused before activation** (see below); `0` disables the check. |
|
|
91
92
|
|
|
92
93
|
Active and draining releases are never pruned. This governs Rollbridge's own
|
|
93
94
|
release records; the deploy tool still owns on-disk release directories.
|
|
94
95
|
|
|
96
|
+
### The draining-generation bound
|
|
97
|
+
|
|
98
|
+
Activating a new generation retires the incumbent, so every deploy adds one draining
|
|
99
|
+
group. When a drain outlasts the deploy cadence, that backlog grows without limit: the
|
|
100
|
+
measured TensorBuzz case had 16 releases draining in 18–114 minutes (median ~37 minutes)
|
|
101
|
+
against deploys every 3–7 minutes, and each further deploy added another group that its own
|
|
102
|
+
post-activation check then reported as a failure.
|
|
103
|
+
|
|
104
|
+
`deploy` therefore counts the groups already draining plus the incumbent it is about to
|
|
105
|
+
retire, and refuses before starting any process of the candidate:
|
|
106
|
+
|
|
107
|
+
```text
|
|
108
|
+
Deploy refused before activation: 13 release groups already draining (20261115063805 wedged
|
|
109
|
+
(1 live process past its retirement budget with no runner child remaining); …). Activating
|
|
110
|
+
20261115073717 would make 14, above the releaseRetention.maxConcurrentDrainingReleaseGroups
|
|
111
|
+
bound of 12.
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
The refusal is a deploy failure like any other: the incumbent keeps serving, nothing of the
|
|
115
|
+
candidate is started, and the named reason says which groups are still holding the fleet and
|
|
116
|
+
whether each is `progressing` or `wedged` (see [`status` in the CLI
|
|
117
|
+
guide](cli.md#status)). A refused deploy is retried once a drain finishes; the bound is a
|
|
118
|
+
policy, not a queue.
|
|
119
|
+
|
|
95
120
|
## `statePath`
|
|
96
121
|
|
|
97
122
|
When set, the daemon persists a sanitized operational state snapshot — the active and
|
|
@@ -582,7 +607,7 @@ Rollbridge sets these in every managed process's environment (the process's own
|
|
|
582
607
|
- `port` must be a positive port number or an ascending `{from, to}` range.
|
|
583
608
|
- `control.mode` must be an octal mode between `0` and `0o777`.
|
|
584
609
|
- `control.owner` and `control.group` must each be a non-negative integer id or a non-empty name (resolved at daemon start).
|
|
585
|
-
- `outputLines` and `releaseRetention.keep` must be positive/non-negative integers; `health.startDelayMs` and `releaseRetention.
|
|
610
|
+
- `outputLines` and `releaseRetention.keep` must be positive/non-negative integers; `health.startDelayMs`, `releaseRetention.maxAgeMs`, and `releaseRetention.maxConcurrentDrainingReleaseGroups` must be non-negative numbers.
|
|
586
611
|
- `restart.maxRestarts` must be a non-negative integer (omit it for unlimited restarts); `restart.backoffFactor` must be a number ≥ 1; `restart.windowMs` and `restart.maxDelayMs` must be non-negative numbers.
|
|
587
612
|
- When `memory` is set, `memory.limitBytes` must be a positive integer, `memory.warnBytes` a non-negative integer, and `memory.checkIntervalMs` a positive number.
|
|
588
613
|
- `replicas` must be a positive integer; `replicas > 1` is allowed only on a `companion` process without a `port`. Process ids must not contain `#` (reserved for replica instance ids).
|
package/package.json
CHANGED
package/src/config.js
CHANGED
|
@@ -23,7 +23,7 @@ import {processStartOrder} from "./process-order.js"
|
|
|
23
23
|
* @typedef {{drainTimeoutMs: number, forceStopTimeoutMs: number, healthPath: string, healthTimeoutMs: number, host: string, nonBlockingDrainTimeoutMs: number, port: number, upstreamHost: string}} ProxyConfig
|
|
24
24
|
* @typedef {{includes: string[], name: string}} LegacyTakeoverProcessConfig
|
|
25
25
|
* @typedef {{forceStopTimeoutMs: number, processes: LegacyTakeoverProcessConfig[], screens: string[]}} LegacyTakeoverConfig
|
|
26
|
-
* @typedef {{keep: number, maxAgeMs: number}} ReleaseRetentionConfig
|
|
26
|
+
* @typedef {{keep: number, maxAgeMs: number, maxConcurrentDrainingReleaseGroups: number}} ReleaseRetentionConfig
|
|
27
27
|
* @typedef {{reconnectGraceMs: number}} OwnerRecoveryConfig
|
|
28
28
|
* @typedef {{command: string | undefined, timeoutMs: number}} ComposeDrainConfig
|
|
29
29
|
* @typedef {{command: string | undefined, intervalMs: number, timeoutMs: number}} ComposeInFlightConfig
|
|
@@ -55,6 +55,15 @@ const DEFAULT_DEPENDENCY_TIMEOUT_MS = 30000
|
|
|
55
55
|
// know to configure. A candidate that never becomes healthy still fails the deploy at the
|
|
56
56
|
// deadline while the incumbent keeps serving; the budget only moves when the gate reports.
|
|
57
57
|
const DEFAULT_HEALTH_TIMEOUT_MS = 120000
|
|
58
|
+
// Bound on how many release groups may drain at once before a new handoff is refused.
|
|
59
|
+
// Measured on TensorBuzz production (2026-11-14/15, shared/log/rollbridge.log): 16 releases
|
|
60
|
+
// drained in 18-114 minutes with a median of ~37 minutes, while deploys arrived every
|
|
61
|
+
// 3-7 minutes (7 traffic switches in 61 minutes on 2026-11-14). A bound of 1 — the value a
|
|
62
|
+
// consumer gate effectively had — is unsatisfiable at that cadence, so every deploy adds one
|
|
63
|
+
// more draining generation. 37 / 3 = 12 is the backlog the measured drain median produces at
|
|
64
|
+
// the fastest measured cadence, so this default admits normal operation and refuses only
|
|
65
|
+
// growth beyond it. 0 disables the bound.
|
|
66
|
+
const DEFAULT_MAX_CONCURRENT_DRAINING_RELEASE_GROUPS = 12
|
|
58
67
|
const commonJsRequire = createRequire(import.meta.url)
|
|
59
68
|
|
|
60
69
|
/**
|
|
@@ -876,10 +885,12 @@ function normalizeBoolean(value, key, issues, fallback) {
|
|
|
876
885
|
function normalizeReleaseRetention(source, issues) {
|
|
877
886
|
const keep = normalizeNumber(source.keep, "releaseRetention.keep", issues, {default: 10})
|
|
878
887
|
const maxAgeMs = normalizeNumber(source.maxAgeMs, "releaseRetention.maxAgeMs", issues, {default: 0})
|
|
888
|
+
const maxConcurrentDrainingReleaseGroups = normalizeNumber(source.maxConcurrentDrainingReleaseGroups, "releaseRetention.maxConcurrentDrainingReleaseGroups", issues, {default: DEFAULT_MAX_CONCURRENT_DRAINING_RELEASE_GROUPS})
|
|
879
889
|
|
|
880
890
|
return {
|
|
881
891
|
keep: nonNegativeOrDefault(keep, "releaseRetention.keep", issues, 10, true),
|
|
882
|
-
maxAgeMs: nonNegativeOrDefault(maxAgeMs, "releaseRetention.maxAgeMs", issues, 0, false)
|
|
892
|
+
maxAgeMs: nonNegativeOrDefault(maxAgeMs, "releaseRetention.maxAgeMs", issues, 0, false),
|
|
893
|
+
maxConcurrentDrainingReleaseGroups: nonNegativeOrDefault(maxConcurrentDrainingReleaseGroups, "releaseRetention.maxConcurrentDrainingReleaseGroups", issues, DEFAULT_MAX_CONCURRENT_DRAINING_RELEASE_GROUPS, true)
|
|
883
894
|
}
|
|
884
895
|
}
|
|
885
896
|
|
package/src/daemon.js
CHANGED
|
@@ -9,8 +9,10 @@ import {isDeepStrictEqual} from "node:util"
|
|
|
9
9
|
import httpProxy from "http-proxy"
|
|
10
10
|
import {loadConfig} from "./config.js"
|
|
11
11
|
import {openControlSession} from "./control-client.js"
|
|
12
|
+
import {admitGenerationRelease} from "./drain-policy.js"
|
|
12
13
|
import EventLog from "./event-log.js"
|
|
13
14
|
import GuardianClient from "./guardian-client.js"
|
|
15
|
+
import {childRecordIsGone} from "./managed-process.js"
|
|
14
16
|
import ReleaseGroup from "./release-group.js"
|
|
15
17
|
import {clearState, isProcessAlive, liveProcesses, readState, writeState} from "./state-store.js"
|
|
16
18
|
import {resolveGroupId, resolveUserId} from "./system-ids.js"
|
|
@@ -1631,6 +1633,17 @@ export default class RollbridgeDaemon {
|
|
|
1631
1633
|
await this.updateGenerationTransition("restoring_committed")
|
|
1632
1634
|
return await this.resumeGenerationTransition()
|
|
1633
1635
|
}
|
|
1636
|
+
const admission = this.evaluateGenerationAdmission(newReleaseId, nextConfig)
|
|
1637
|
+
|
|
1638
|
+
if (!admission.allowed) {
|
|
1639
|
+
this.logger("deploy refused by draining-generation bound", {
|
|
1640
|
+
candidateReleaseId: newReleaseId,
|
|
1641
|
+
draining: admission.draining.map((group) => ({releaseId: group.releaseId, status: group.status})),
|
|
1642
|
+
limit: nextConfig.releaseRetention.maxConcurrentDrainingReleaseGroups,
|
|
1643
|
+
wouldBeDrainingCount: admission.wouldBeDrainingCount
|
|
1644
|
+
})
|
|
1645
|
+
throw new Error(/** @type {string} */ (admission.reason))
|
|
1646
|
+
}
|
|
1634
1647
|
const release = new ReleaseGroup({
|
|
1635
1648
|
config: nextConfig,
|
|
1636
1649
|
logger: this.logger,
|
|
@@ -1687,6 +1700,35 @@ export default class RollbridgeDaemon {
|
|
|
1687
1700
|
return await this.resumeGenerationTransition()
|
|
1688
1701
|
}
|
|
1689
1702
|
|
|
1703
|
+
/**
|
|
1704
|
+
* Applies the draining-generation bound before a handoff.
|
|
1705
|
+
*
|
|
1706
|
+
* Without this, every deploy retires one more generation and the backlog grows
|
|
1707
|
+
* monotonically whenever a drain outlasts the deploy cadence; a consumer that checks the
|
|
1708
|
+
* count after activation reports the growth it cannot undo. Refusing here names the
|
|
1709
|
+
* draining groups and whether each is still progressing or wedged, before any process of
|
|
1710
|
+
* the candidate is started.
|
|
1711
|
+
* @param {string} candidateReleaseId - Release the handoff would activate.
|
|
1712
|
+
* @param {import("./config.js").RollbridgeConfig} config - Config governing the handoff.
|
|
1713
|
+
* @returns {import("./drain-policy.js").GenerationAdmission} Admission decision and the groups behind it.
|
|
1714
|
+
*/
|
|
1715
|
+
evaluateGenerationAdmission(candidateReleaseId, config) {
|
|
1716
|
+
const draining = [...this.releases.values()]
|
|
1717
|
+
.filter((release) => release.state === "draining")
|
|
1718
|
+
.map((release) => {
|
|
1719
|
+
const drain = release.drainStatus()
|
|
1720
|
+
|
|
1721
|
+
return {reason: drain?.reason ?? "no liveness observation recorded", releaseId: release.releaseId, status: drain?.status ?? "unproven"}
|
|
1722
|
+
})
|
|
1723
|
+
|
|
1724
|
+
return admitGenerationRelease({
|
|
1725
|
+
candidateReleaseId,
|
|
1726
|
+
draining,
|
|
1727
|
+
incumbentWillDrain: this.activeRelease !== undefined,
|
|
1728
|
+
maxConcurrentDrainingReleaseGroups: config.releaseRetention.maxConcurrentDrainingReleaseGroups
|
|
1729
|
+
})
|
|
1730
|
+
}
|
|
1731
|
+
|
|
1690
1732
|
/**
|
|
1691
1733
|
* Continues one exact durable generation transition without an internal retry loop.
|
|
1692
1734
|
* @returns {Promise<Record<string, JsonValue>>} Deploy result after commit.
|
|
@@ -2407,6 +2449,46 @@ export default class RollbridgeDaemon {
|
|
|
2407
2449
|
throw new Error(`Release "${target.releaseId}" is already active.`)
|
|
2408
2450
|
}
|
|
2409
2451
|
|
|
2452
|
+
// A generation recorded as `stopped` whose processes are still alive is the shape a
|
|
2453
|
+
// restore cannot honour: re-deploying that release id re-registers its guardian process
|
|
2454
|
+
// keys under a new provenance while the old processes still hold them, and the guardian
|
|
2455
|
+
// answers with a bare provenance mismatch. Use the guardian's own fence here so the
|
|
2456
|
+
// refusal is decided before anything is touched, and name the divergence — including a
|
|
2457
|
+
// process group that outlived the tracked process and is invisible as a pid — together
|
|
2458
|
+
// with the release the guardian actually reports as active, so the deployment tool can
|
|
2459
|
+
// reconcile its own current pointer instead of restoring a generation that already drained.
|
|
2460
|
+
const heldProcesses = target.state === "stopped"
|
|
2461
|
+
? target.status().processes.filter((processStatus) => !childRecordIsGone(processStatus))
|
|
2462
|
+
: []
|
|
2463
|
+
|
|
2464
|
+
if (heldProcesses.length > 0) {
|
|
2465
|
+
const activeReleaseId = this.activeRelease?.releaseId ?? null
|
|
2466
|
+
const held = heldProcesses.map((processStatus) => {
|
|
2467
|
+
if (processStatus.heldProcessGroup) {
|
|
2468
|
+
const {liveMemberCount, pgid} = processStatus.heldProcessGroup
|
|
2469
|
+
const members = liveMemberCount === undefined ? "" : ` with ${liveMemberCount} live member(s)`
|
|
2470
|
+
|
|
2471
|
+
return `${processStatus.id}: process group ${pgid}${members} still alive after the tracked process exited`
|
|
2472
|
+
}
|
|
2473
|
+
|
|
2474
|
+
return `${processStatus.id}: state ${processStatus.state}, pid ${processStatus.pid ?? "unknown"}`
|
|
2475
|
+
}).join("; ")
|
|
2476
|
+
|
|
2477
|
+
this.logger("rollback refused: drained generation still holds live processes", {
|
|
2478
|
+
activeReleaseId,
|
|
2479
|
+
heldProcesses: heldProcesses.map((processStatus) => ({
|
|
2480
|
+
heldProcessGroupPgid: processStatus.heldProcessGroup?.pgid ?? null,
|
|
2481
|
+
pid: processStatus.pid ?? null,
|
|
2482
|
+
processId: processStatus.id,
|
|
2483
|
+
state: processStatus.state
|
|
2484
|
+
})),
|
|
2485
|
+
releaseId: target.releaseId,
|
|
2486
|
+
revision: target.revision
|
|
2487
|
+
})
|
|
2488
|
+
|
|
2489
|
+
throw new Error(`Release "${target.releaseId}" is recorded as drained while ${heldProcesses.length} process${heldProcesses.length === 1 ? "" : "es"} of that generation still hold live work (${held}). Restoring it would re-register those processes under a new provenance and fail with a guardian provenance mismatch. The guardian's active release is ${activeReleaseId ? `"${activeReleaseId}"` : "none"}; reconcile the deployment pointer with \`rollbridge status\` before restoring.`)
|
|
2490
|
+
}
|
|
2491
|
+
|
|
2410
2492
|
// The target may still be draining a prior deploy (live processes). Stop it before the
|
|
2411
2493
|
// deploy below re-uses its id in this.releases, otherwise the still-running instance
|
|
2412
2494
|
// would be dropped from status/pruning/shutdown and could be orphaned.
|
|
@@ -3455,7 +3537,7 @@ function requiredNonNegativeInteger(value, key) {
|
|
|
3455
3537
|
/**
|
|
3456
3538
|
* Selects stopped releases to prune by the retention policy, keeping the most recent.
|
|
3457
3539
|
* @param {PrunableRelease[]} releases - Status of all tracked releases, in deploy order (oldest first).
|
|
3458
|
-
* @param {
|
|
3540
|
+
* @param {{keep: number, maxAgeMs: number}} policy - Retention policy; draining generations are never pruned here.
|
|
3459
3541
|
* @param {number} now - Current epoch milliseconds.
|
|
3460
3542
|
* @returns {string[]} Release ids to remove.
|
|
3461
3543
|
*/
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
// @ts-check
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* @typedef {"progressing" | "wedged" | "unproven"} DrainProgress
|
|
5
|
+
* @typedef {{cpuMs: number | undefined, heldProcessGroupPgid: number | undefined, id: string, liveChildCount: number | undefined, pid: number | undefined, retirementTimedOut: boolean}} DrainProcessObservation
|
|
6
|
+
* @typedef {{atMs: number, processes: DrainProcessObservation[]}} DrainLivenessSample
|
|
7
|
+
* @typedef {{cpuDeltaMs: number | undefined, reason: string, sampleWindowMs: number, status: DrainProgress}} DrainClassification
|
|
8
|
+
* @typedef {{releaseId: string, reason: string, status: DrainProgress}} DrainGroupReport
|
|
9
|
+
* @typedef {{allowed: boolean, draining: DrainGroupReport[], reason: string | undefined, wouldBeDrainingCount: number}} GenerationAdmission
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Classifies one draining release group as still draining or wedged.
|
|
14
|
+
*
|
|
15
|
+
* A group is wedged only when its drain can no longer progress. Every inconclusive reading
|
|
16
|
+
* — a live connection, a process still inside its retirement budget, an unmeasurable CPU
|
|
17
|
+
* counter, or a sample window shorter than `minSampleWindowMs` — reports `progressing` or
|
|
18
|
+
* `unproven`, never `wedged`: a legitimate multi-hour drain is valid and must not be
|
|
19
|
+
* mislabelled as a wedge.
|
|
20
|
+
* @param {object} args - Classification inputs.
|
|
21
|
+
* @param {number} args.connectionCount - Live proxy connections this generation still owns.
|
|
22
|
+
* @param {DrainLivenessSample} args.currentSample - Newest liveness observation.
|
|
23
|
+
* @param {number} args.minSampleWindowMs - Shortest window that can prove flat CPU.
|
|
24
|
+
* @param {DrainLivenessSample | undefined} args.previousSample - Older observation, when one is in range.
|
|
25
|
+
* @returns {DrainClassification} The classification and the evidence it rests on.
|
|
26
|
+
*/
|
|
27
|
+
export function classifyDrainProgress({connectionCount, currentSample, minSampleWindowMs, previousSample}) {
|
|
28
|
+
const live = currentSample.processes.filter((process) => process.pid !== undefined || process.heldProcessGroupPgid !== undefined)
|
|
29
|
+
const stillInsideBudget = live.filter((process) => !process.retirementTimedOut)
|
|
30
|
+
|
|
31
|
+
if (connectionCount > 0) {
|
|
32
|
+
return classify("progressing", `${connectionCount} live connection${connectionCount === 1 ? "" : "s"} still owned by the generation`, currentSample, previousSample)
|
|
33
|
+
}
|
|
34
|
+
if (stillInsideBudget.length > 0) {
|
|
35
|
+
return classify("progressing", `process ${stillInsideBudget[0].id} is still inside its retirement budget`, currentSample, previousSample)
|
|
36
|
+
}
|
|
37
|
+
if (live.length === 0) {
|
|
38
|
+
return classify("progressing", "no live process remains in the generation", currentSample, previousSample)
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
const liveChildCount = live.reduce((total, process) => total + (process.liveChildCount ?? 0), 0)
|
|
42
|
+
|
|
43
|
+
if (liveChildCount === 0) {
|
|
44
|
+
return classify("wedged", `${live.length} live process${live.length === 1 ? "" : "es"} past its retirement budget with no runner child remaining${heldGroupNote(live)}`, currentSample, previousSample)
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
if (!previousSample) {
|
|
48
|
+
return classify("unproven", `first liveness observation; a ${minSampleWindowMs}ms window is required before CPU flatness can be claimed`, currentSample, previousSample)
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
const sampleWindowMs = currentSample.atMs - previousSample.atMs
|
|
52
|
+
|
|
53
|
+
if (sampleWindowMs < minSampleWindowMs) {
|
|
54
|
+
return classify("unproven", `liveness window ${sampleWindowMs}ms is below the ${minSampleWindowMs}ms minimum`, currentSample, previousSample)
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
const currentCpuMs = totalCpuMs(currentSample)
|
|
58
|
+
const previousCpuMs = totalCpuMs(previousSample)
|
|
59
|
+
|
|
60
|
+
if (currentCpuMs === undefined || previousCpuMs === undefined) {
|
|
61
|
+
return classify("unproven", "process CPU is not measurable on this host", currentSample, previousSample)
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
const cpuDeltaMs = currentCpuMs - previousCpuMs
|
|
65
|
+
|
|
66
|
+
if (cpuDeltaMs !== 0) {
|
|
67
|
+
const change = cpuDeltaMs > 0
|
|
68
|
+
? `CPU advanced ${cpuDeltaMs}ms`
|
|
69
|
+
: `the group changed (${live.length} live process${live.length === 1 ? "" : "es"}, ${cpuDeltaMs}ms of consumed CPU left the group)`
|
|
70
|
+
|
|
71
|
+
return classify("progressing", `${change} across ${sampleWindowMs}ms`, currentSample, previousSample)
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
return classify("wedged", `${live.length} live process${live.length === 1 ? "" : "es"} with ${liveChildCount} runner ${liveChildCount === 1 ? "child" : "children"}, no live connection, every retirement budget expired, CPU flat across ${sampleWindowMs}ms${heldGroupNote(live)}`, currentSample, previousSample)
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* Decides whether a new release generation may be activated under the draining-group bound.
|
|
79
|
+
*
|
|
80
|
+
* The incumbent counts as a draining group because activating the candidate retires it: the
|
|
81
|
+
* count is what the fleet would have *after* the handoff, which is the only version of the
|
|
82
|
+
* number a pre-handoff decision can act on.
|
|
83
|
+
* @param {object} args - Admission inputs.
|
|
84
|
+
* @param {string} args.candidateReleaseId - Release the handoff would activate.
|
|
85
|
+
* @param {DrainGroupReport[]} args.draining - Every release group currently draining.
|
|
86
|
+
* @param {boolean} args.incumbentWillDrain - Whether an active generation is retired by this handoff.
|
|
87
|
+
* @param {number} args.maxConcurrentDrainingReleaseGroups - Policy bound; 0 disables the check.
|
|
88
|
+
* @returns {GenerationAdmission} Allow/deny plus the named reason and the groups behind it.
|
|
89
|
+
*/
|
|
90
|
+
export function admitGenerationRelease({candidateReleaseId, draining, incumbentWillDrain, maxConcurrentDrainingReleaseGroups}) {
|
|
91
|
+
const wouldBeDrainingCount = draining.length + (incumbentWillDrain ? 1 : 0)
|
|
92
|
+
|
|
93
|
+
if (maxConcurrentDrainingReleaseGroups === 0 || wouldBeDrainingCount <= maxConcurrentDrainingReleaseGroups) {
|
|
94
|
+
return {allowed: true, draining, reason: undefined, wouldBeDrainingCount}
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
const reported = draining.map((group) => `${group.releaseId} ${group.status} (${group.reason})`).join("; ")
|
|
98
|
+
const reason = `Deploy refused before activation: ${draining.length} release group${draining.length === 1 ? "" : "s"} already draining${reported ? ` (${reported})` : ""}. Activating ${candidateReleaseId} would make ${wouldBeDrainingCount}, above the releaseRetention.maxConcurrentDrainingReleaseGroups bound of ${maxConcurrentDrainingReleaseGroups}.`
|
|
99
|
+
|
|
100
|
+
return {allowed: false, draining, reason, wouldBeDrainingCount}
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* Names the process groups that outlived their tracked process.
|
|
105
|
+
*
|
|
106
|
+
* A tracked process can exit while the group it led keeps running (a shell wrapper killed by
|
|
107
|
+
* the stop signal, a runner child that outlives its parent). Those members still hold the
|
|
108
|
+
* generation's work and no rollbridge record can signal them, so a wedge claim has to say
|
|
109
|
+
* which group is holding it.
|
|
110
|
+
* @param {DrainProcessObservation[]} processes - Live observations of one generation.
|
|
111
|
+
* @returns {string} A note naming the held groups, or "" when every group is still tracked.
|
|
112
|
+
*/
|
|
113
|
+
function heldGroupNote(processes) {
|
|
114
|
+
const held = processes.filter((process) => process.heldProcessGroupPgid !== undefined)
|
|
115
|
+
|
|
116
|
+
if (held.length === 0) return ""
|
|
117
|
+
|
|
118
|
+
const named = held.map((process) => `${process.id} pgid ${process.heldProcessGroupPgid}`).join(", ")
|
|
119
|
+
|
|
120
|
+
return `; ${held.length} tracked process${held.length === 1 ? "" : "es"} already exited and left ${named} alive`
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/**
|
|
124
|
+
* @param {DrainProgress} status - Classification.
|
|
125
|
+
* @param {string} reason - Named evidence behind it.
|
|
126
|
+
* @param {DrainLivenessSample} currentSample - Newest observation.
|
|
127
|
+
* @param {DrainLivenessSample | undefined} previousSample - Older observation, when in range.
|
|
128
|
+
* @returns {DrainClassification} Classification payload.
|
|
129
|
+
*/
|
|
130
|
+
function classify(status, reason, currentSample, previousSample) {
|
|
131
|
+
const sampleWindowMs = previousSample ? currentSample.atMs - previousSample.atMs : 0
|
|
132
|
+
const currentCpuMs = totalCpuMs(currentSample)
|
|
133
|
+
const previousCpuMs = totalCpuMs(previousSample)
|
|
134
|
+
|
|
135
|
+
return {
|
|
136
|
+
cpuDeltaMs: currentCpuMs === undefined || previousCpuMs === undefined ? undefined : currentCpuMs - previousCpuMs,
|
|
137
|
+
reason,
|
|
138
|
+
sampleWindowMs,
|
|
139
|
+
status
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/**
|
|
144
|
+
* @param {DrainLivenessSample | undefined} sample - Observation to total.
|
|
145
|
+
* @returns {number | undefined} Summed CPU, or undefined when any live process is unmeasurable.
|
|
146
|
+
*/
|
|
147
|
+
function totalCpuMs(sample) {
|
|
148
|
+
if (!sample) return undefined
|
|
149
|
+
|
|
150
|
+
let total = 0
|
|
151
|
+
|
|
152
|
+
for (const process of sample.processes) {
|
|
153
|
+
if (process.pid === undefined && process.heldProcessGroupPgid === undefined) continue
|
|
154
|
+
if (process.cpuMs === undefined) return undefined
|
|
155
|
+
total += process.cpuMs
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
return total
|
|
159
|
+
}
|