rollbridge 0.1.63 → 0.1.66
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/changelog.d/20261115-bound-draining-release-groups.md +62 -0
- package/changelog.d/20261115-restore-deploy-refuses-held-generation.md +27 -0
- package/docs/cli.md +59 -0
- package/docs/config.md +26 -1
- package/package.json +1 -1
- package/src/config.js +13 -2
- package/src/daemon.js +146 -1
- package/src/drain-policy.js +159 -0
- package/src/managed-process.js +99 -3
- package/src/process-guardian.js +4 -5
- package/src/process-memory.js +62 -2
- package/src/release-group.js +93 -2
- package/test/config-validation.test.js +11 -4
- package/test/drain-policy.test.js +199 -0
- package/test/managed-process.test.js +11 -1
- package/test/process-memory.test.js +47 -1
- package/test/release-group.test.js +127 -2
- package/test/rollbridge.test.js +193 -0
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
### Added
|
|
2
|
+
|
|
3
|
+
- `releaseRetention.maxConcurrentDrainingReleaseGroups` (default **12**) bounds how many
|
|
4
|
+
release groups may drain at once, and `deploy` evaluates it **before the handoff**. Every
|
|
5
|
+
activation retires one more generation, so whenever deploys arrive faster than a generation
|
|
6
|
+
drains the backlog grows monotonically. Measured on TensorBuzz production 2026-11-14/15
|
|
7
|
+
(`shared/log/rollbridge.log`, `traffic switched` → `release drained` pairs for 16 releases):
|
|
8
|
+
drain duration 18 min … 114 min, median ≈37 min (`20261114232900` 23:31:24 → 01:25:53 is
|
|
9
|
+
114m29s), against 7 traffic switches in 61 minutes on 2026-11-14 — a cadence of 3–7 minutes.
|
|
10
|
+
A supervisor tolerating one draining group is unsatisfiable at that cadence: 37 / 3 = 12 is
|
|
11
|
+
the backlog the measured median produces at the fastest measured cadence. The count includes
|
|
12
|
+
the incumbent the handoff is about to retire, because that is the only version of the number
|
|
13
|
+
a pre-handoff decision can act on. A deploy over the bound is refused before any process of
|
|
14
|
+
the candidate starts, naming each draining group and its classification:
|
|
15
|
+
`Deploy refused before activation: 2 release groups already draining (20261115063805 wedged
|
|
16
|
+
(…); 20261114232900 progressing (2 live connections still owned by the generation)).
|
|
17
|
+
Activating 20261115073717 would make 3, above the
|
|
18
|
+
releaseRetention.maxConcurrentDrainingReleaseGroups bound of 2.` `0` disables the check.
|
|
19
|
+
Nothing is queued or deferred by a timer: the refusal is an ordinary deploy failure, and the
|
|
20
|
+
incumbent keeps serving.
|
|
21
|
+
- `status` now distinguishes a drain that is still progressing from one that is wedged. Each
|
|
22
|
+
draining release reports `drain: {cpuDeltaMs, reason, sampleWindowMs, status}` with
|
|
23
|
+
`status` one of `progressing`, `wedged`, or `unproven`, and each process reports
|
|
24
|
+
`retirementTimedOut` plus `heldProcessGroup: {liveMemberCount, pgid}`. A group is `wedged`
|
|
25
|
+
only when its drain can no longer progress — no live connection, every retirement budget
|
|
26
|
+
expired, and either no runner child remains or the group's cumulative CPU (procfs
|
|
27
|
+
`utime`+`stime`, defunct members included) is flat across a window of at least 20 000 ms.
|
|
28
|
+
Every inconclusive reading reports `progressing` or `unproven`, never `wedged`: the backlog
|
|
29
|
+
this fleet actually has is honest slow drain, measured as 3 live
|
|
30
|
+
`background-jobs-runner: 1 job` children with a `DockerServerDrainFillJob` `handed_off` for
|
|
31
|
+
30 minutes, whose last job completed 6 seconds before the guardian logged `release drained`.
|
|
32
|
+
Consumers gate on that classification instead of on a raw process count.
|
|
33
|
+
|
|
34
|
+
### Fixed
|
|
35
|
+
|
|
36
|
+
- A process group that outlives its tracked process is now recorded and reported. Rollbridge
|
|
37
|
+
spawns every process `detached: true`, so the tracked pid is the group leader; when the
|
|
38
|
+
leader exits on the stop signal while a member it launched keeps running, the tracked record
|
|
39
|
+
goes `stopped` with no pid while the group still holds the generation's work. `status`
|
|
40
|
+
reports it as `heldProcessGroup`, the drain classifier observes it by group id, and the
|
|
41
|
+
guardian's provenance fence (`childRecordIsGone`, now shared with the daemon instead of
|
|
42
|
+
inlined in `process-guardian.js`) counts it as not gone. The group id is recorded by the stop
|
|
43
|
+
path that already waited for the group to leave, not by `onExit`: that hook fires for every
|
|
44
|
+
retired process, and rediscovering the group there costs a procfs scan (~200 ms measured
|
|
45
|
+
here) on every exit, including the ordinary ones where the group left cleanly.
|
|
46
|
+
- `rollback` refuses a target whose generation is recorded as drained while its processes still
|
|
47
|
+
hold live work, naming each held process and the release the guardian reports as active.
|
|
48
|
+
Re-deploying that release id re-registered its guardian process keys under a new provenance
|
|
49
|
+
while the old processes still held them, and the deploy failed partway through with
|
|
50
|
+
`Guardian provenance mismatch for release:20261115060540:background-jobs-main` (TensorBuzz
|
|
51
|
+
production, run `408591ff-e72e-4351-b15a-7ec239a62f6b`, 2026-11-15) — leaving the deploy
|
|
52
|
+
tool's `current` pointer pinned at `20261115060540` while the guardian reported
|
|
53
|
+
`20261115073717` active. The refusal happens before anything is touched and names the
|
|
54
|
+
guardian's active release, so the pointer is reconciled from `rollbridge status` instead of
|
|
55
|
+
silently diverging.
|
|
56
|
+
- Drain liveness observations are bounded: a generation that still owns live connections is
|
|
57
|
+
classified without reading procfs at all, and a new observation is taken at most once per
|
|
58
|
+
20 000 ms window per draining group (held groups re-verify at most once per 5 000 ms).
|
|
59
|
+
Sampling on every `status` read was measured here at ~200 ms per group on a host with 9 530
|
|
60
|
+
processes, enough to starve a 1 000 ms connection drain in `test/rollbridge.test.js` "a
|
|
61
|
+
nonBlockingDrain worker stops immediately while its release is still draining". No timer,
|
|
62
|
+
poll, or sweeper was added: the bound is a staleness check evaluated when `status` is read.
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
### Fixed
|
|
2
|
+
|
|
3
|
+
- `deploy` refuses a `--release-id` it still retains while that generation holds live work,
|
|
4
|
+
before any process of the candidate starts. PR #12 fenced `rollback` only; the restore that
|
|
5
|
+
actually ran in TensorBuzz production was a deploy command naming the drained release id, and
|
|
6
|
+
it failed twice in twelve minutes (`shared/log/rollbridge.log`, server3, 2026-11-15):
|
|
7
|
+
`20:29:51Z traffic switched 20261115195051 → 20261115202720` then `20:29:55Z release startup
|
|
8
|
+
failed` / `deploy failed` with `Guardian provenance mismatch for
|
|
9
|
+
release:20261115195051:background-jobs-main`, and the identical pair at `20:41:21Z` /
|
|
10
|
+
`20:41:26Z` against `20261115203845`. Only one `release drained` landed in that window
|
|
11
|
+
(`20261115164249` at 20:37:03Z), and a 20:54Z process census found all three generations alive
|
|
12
|
+
at once — `20261115195051` (main + worker + 3 runners), `20261115202720` (main + worker + 4
|
|
13
|
+
runners), `20261115203845` (main + worker + 4 runners) — with `20261115195051`'s
|
|
14
|
+
`velocious background-jobs-main` still running as pid 2310504. The provenance key therefore
|
|
15
|
+
belonged to a live generation, so the 2026-10-25 carve-out that lets a dead generation's key be
|
|
16
|
+
re-registered never applied; the collision aborted the deploy after the handoff had already
|
|
17
|
+
moved traffic, leaving the deploy tool's `current` pointer at `releases/20261115195051` while
|
|
18
|
+
the guardian served `20261115203845`. The refusal is decided from the retained record with the
|
|
19
|
+
guardian's own `childRecordIsGone` fence, names the retained state, each held process (a held
|
|
20
|
+
process group included), and the release the guardian reports as active, and starts nothing:
|
|
21
|
+
`Deploy refused before activation: release "20261115195051" is retained in state draining while
|
|
22
|
+
1 process of that generation still hold live work (…). … The guardian's active release is
|
|
23
|
+
"20261115203845"; deploy a new release id or reconcile the deployment pointer with \`rollbridge
|
|
24
|
+
status\`.` Journaled generation-transition resume is exempt — it re-drives the transaction that
|
|
25
|
+
already owns the release id — and a rollback to a still-draining target still stops that
|
|
26
|
+
generation first, which is what makes its id reusable; the same fence then applies after the
|
|
27
|
+
stop.
|
package/docs/cli.md
CHANGED
|
@@ -163,6 +163,25 @@ for old workers, jobs, or HTTP/WebSocket connections to finish. Prints
|
|
|
163
163
|
If the new release fails to start or health-check, the previous release stays
|
|
164
164
|
active and the command errors.
|
|
165
165
|
|
|
166
|
+
A deploy is also refused **before any of the candidate's processes start** when the
|
|
167
|
+
handoff would push the number of concurrently draining release groups past
|
|
168
|
+
[`releaseRetention.maxConcurrentDrainingReleaseGroups`](config.md#releaseretention) —
|
|
169
|
+
the incumbent the handoff is about to retire counts too. The incumbent keeps serving,
|
|
170
|
+
nothing of the candidate is started, and the error names every draining group with its
|
|
171
|
+
`progressing`/`wedged` classification and the bound it exceeded.
|
|
172
|
+
|
|
173
|
+
The same pre-activation refusal applies when `--release-id` names a release Rollbridge
|
|
174
|
+
still **retains** while that generation holds live work: draining with processes alive, or
|
|
175
|
+
recorded drained while a process group outlived its tracked process
|
|
176
|
+
(`processes[].heldProcessGroup`). Starting a second generation under that id re-registers
|
|
177
|
+
the guardian process keys the retained generation still holds, so the deploy aborts
|
|
178
|
+
partway through startup with `Guardian provenance mismatch for release:<id>:<key>` — the
|
|
179
|
+
shape a failed-deploy restore produced twice on 2026-11-15 (`20261115195051` at 20:29:55Z
|
|
180
|
+
and again at 20:41:26Z, 12 minutes apart). The refusal names the retained state, each held
|
|
181
|
+
process, and the release the guardian reports as active, and leaves the retained generation
|
|
182
|
+
untouched. A generation whose processes are genuinely gone stays re-deployable: the
|
|
183
|
+
2026-10-25 carve-out for re-registering a dead generation is unchanged.
|
|
184
|
+
|
|
166
185
|
With an opt-in handoff-service `lifecycle.activateCommand`, deploy journals the
|
|
167
186
|
exact transition, waits for old retirement acknowledgement, waits for candidate
|
|
168
187
|
activation acknowledgement, and synchronously commits the active proxy target.
|
|
@@ -243,6 +262,21 @@ Errors when there is no previous release, the `--release-id` is not a retained
|
|
|
243
262
|
release, or the target is already active. Only releases Rollbridge still retains
|
|
244
263
|
(see [`releaseRetention`](config.md#releaseretention)) can be rolled back to.
|
|
245
264
|
|
|
265
|
+
It also refuses a target whose generation is recorded as drained while its
|
|
266
|
+
processes still hold live work — a process the retirement wait left alive, or a
|
|
267
|
+
process group that outlived the tracked process (`processes[].heldProcessGroup`,
|
|
268
|
+
see [`status`](#status)). Re-deploying that release id would re-register its
|
|
269
|
+
guardian process keys under a new provenance while the old processes still hold
|
|
270
|
+
them, and the guardian answers `Guardian provenance mismatch for
|
|
271
|
+
release:<id>:<key>` after the deploy has already begun. The refusal is raised
|
|
272
|
+
before anything is touched and names each held process together with the release
|
|
273
|
+
the guardian reports as active, so the deploy tool reconciles its own `current`
|
|
274
|
+
pointer with `rollbridge status` instead of restoring a generation that already
|
|
275
|
+
drained. A target that is still **draining** is not refused here: stopping it is
|
|
276
|
+
what makes its id reusable, and the deploy it then runs applies the same fence
|
|
277
|
+
after that stop — so a stop that leaves the generation's process group alive is
|
|
278
|
+
reported as the refusal above instead of colliding mid-startup.
|
|
279
|
+
|
|
246
280
|
**Migration constraints.** Rollback only manages processes — it does **not**
|
|
247
281
|
revert database migrations or other external state. The target release's on-disk
|
|
248
282
|
directory must still exist, and its code must be compatible with the current
|
|
@@ -323,6 +357,31 @@ Memory-supervised processes also report `rssBytes`, `memoryRestarts`,
|
|
|
323
357
|
`lastMemoryRestartAt`, and `children` (the process tree: each group member's
|
|
324
358
|
`pid`, `command`, and `rssBytes`).
|
|
325
359
|
|
|
360
|
+
Every process also reports `retirementTimedOut` — `true` once the supervisor has
|
|
361
|
+
stopped waiting for it — and `heldProcessGroup` (`{pgid, liveMemberCount}`) when a
|
|
362
|
+
bounded stop left the process group running after the tracked process exited. That
|
|
363
|
+
group is the leak a `stopped` record otherwise hides: nothing in Rollbridge can
|
|
364
|
+
signal or wait for it again, yet it still holds the generation's work. It is
|
|
365
|
+
re-checked at most once every 5 seconds, and clears itself when its members leave.
|
|
366
|
+
|
|
367
|
+
Each **draining** release reports a `drain` classification — `status`, the `reason`
|
|
368
|
+
behind it, the `sampleWindowMs` it rests on, and `cpuDeltaMs`:
|
|
369
|
+
|
|
370
|
+
- `progressing` — the generation still owns live connections, a process is still
|
|
371
|
+
inside its retirement budget, or its process-group CPU advanced across the
|
|
372
|
+
sample window. A legitimate multi-hour drain reports this.
|
|
373
|
+
- `wedged` — the drain can no longer progress: no live connection, every
|
|
374
|
+
retirement budget expired, and either no runner child remains or the group's CPU
|
|
375
|
+
is flat across a window of at least 20 seconds.
|
|
376
|
+
- `unproven` — the evidence is inconclusive: a first observation, a window shorter
|
|
377
|
+
than 20 seconds, or a host where process CPU cannot be read.
|
|
378
|
+
|
|
379
|
+
`drain` is `undefined` for a release that is not draining. Gate on this
|
|
380
|
+
classification rather than on a raw process count: a `progressing` group holding
|
|
381
|
+
three live processes is a healthy drain, a `wedged` group holding one is a
|
|
382
|
+
generation that will never finish. Observations are taken at most once per window
|
|
383
|
+
per draining group, so repeated `status` reads stay cheap.
|
|
384
|
+
|
|
326
385
|
`--no-logs` omits only the captured `logs` array from release, service, and
|
|
327
386
|
singleton process statuses. Use it for bounded machine-readable lifecycle
|
|
328
387
|
attestations that do not need process output. The equivalent control request is
|
package/docs/config.md
CHANGED
|
@@ -88,10 +88,35 @@ to let a deploy group talk to the daemon.
|
|
|
88
88
|
| --- | --- | --- | --- |
|
|
89
89
|
| `releaseRetention.keep` | non-negative integer | `10` | Number of most-recent **stopped** releases the daemon keeps in memory and reports in `status`. |
|
|
90
90
|
| `releaseRetention.maxAgeMs` | non-negative number | `0` (disabled) | Also prune stopped releases older than this many milliseconds. |
|
|
91
|
+
| `releaseRetention.maxConcurrentDrainingReleaseGroups` | non-negative integer | `12` | How many release groups may drain at once. A deploy that would push the count past this bound is **refused before activation** (see below); `0` disables the check. |
|
|
91
92
|
|
|
92
93
|
Active and draining releases are never pruned. This governs Rollbridge's own
|
|
93
94
|
release records; the deploy tool still owns on-disk release directories.
|
|
94
95
|
|
|
96
|
+
### The draining-generation bound
|
|
97
|
+
|
|
98
|
+
Activating a new generation retires the incumbent, so every deploy adds one draining
|
|
99
|
+
group. When a drain outlasts the deploy cadence, that backlog grows without limit: the
|
|
100
|
+
measured TensorBuzz case had 16 releases draining in 18–114 minutes (median ~37 minutes)
|
|
101
|
+
against deploys every 3–7 minutes, and each further deploy added another group that its own
|
|
102
|
+
post-activation check then reported as a failure.
|
|
103
|
+
|
|
104
|
+
`deploy` therefore counts the groups already draining plus the incumbent it is about to
|
|
105
|
+
retire, and refuses before starting any process of the candidate:
|
|
106
|
+
|
|
107
|
+
```text
|
|
108
|
+
Deploy refused before activation: 13 release groups already draining (20261115063805 wedged
|
|
109
|
+
(1 live process past its retirement budget with no runner child remaining); …). Activating
|
|
110
|
+
20261115073717 would make 14, above the releaseRetention.maxConcurrentDrainingReleaseGroups
|
|
111
|
+
bound of 12.
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
The refusal is a deploy failure like any other: the incumbent keeps serving, nothing of the
|
|
115
|
+
candidate is started, and the named reason says which groups are still holding the fleet and
|
|
116
|
+
whether each is `progressing` or `wedged` (see [`status` in the CLI
|
|
117
|
+
guide](cli.md#status)). A refused deploy is retried once a drain finishes; the bound is a
|
|
118
|
+
policy, not a queue.
|
|
119
|
+
|
|
95
120
|
## `statePath`
|
|
96
121
|
|
|
97
122
|
When set, the daemon persists a sanitized operational state snapshot — the active and
|
|
@@ -582,7 +607,7 @@ Rollbridge sets these in every managed process's environment (the process's own
|
|
|
582
607
|
- `port` must be a positive port number or an ascending `{from, to}` range.
|
|
583
608
|
- `control.mode` must be an octal mode between `0` and `0o777`.
|
|
584
609
|
- `control.owner` and `control.group` must each be a non-negative integer id or a non-empty name (resolved at daemon start).
|
|
585
|
-
- `outputLines` and `releaseRetention.keep` must be positive/non-negative integers; `health.startDelayMs` and `releaseRetention.
|
|
610
|
+
- `outputLines` and `releaseRetention.keep` must be positive/non-negative integers; `health.startDelayMs`, `releaseRetention.maxAgeMs`, and `releaseRetention.maxConcurrentDrainingReleaseGroups` must be non-negative numbers.
|
|
586
611
|
- `restart.maxRestarts` must be a non-negative integer (omit it for unlimited restarts); `restart.backoffFactor` must be a number ≥ 1; `restart.windowMs` and `restart.maxDelayMs` must be non-negative numbers.
|
|
587
612
|
- When `memory` is set, `memory.limitBytes` must be a positive integer, `memory.warnBytes` a non-negative integer, and `memory.checkIntervalMs` a positive number.
|
|
588
613
|
- `replicas` must be a positive integer; `replicas > 1` is allowed only on a `companion` process without a `port`. Process ids must not contain `#` (reserved for replica instance ids).
|
package/package.json
CHANGED
package/src/config.js
CHANGED
|
@@ -23,7 +23,7 @@ import {processStartOrder} from "./process-order.js"
|
|
|
23
23
|
* @typedef {{drainTimeoutMs: number, forceStopTimeoutMs: number, healthPath: string, healthTimeoutMs: number, host: string, nonBlockingDrainTimeoutMs: number, port: number, upstreamHost: string}} ProxyConfig
|
|
24
24
|
* @typedef {{includes: string[], name: string}} LegacyTakeoverProcessConfig
|
|
25
25
|
* @typedef {{forceStopTimeoutMs: number, processes: LegacyTakeoverProcessConfig[], screens: string[]}} LegacyTakeoverConfig
|
|
26
|
-
* @typedef {{keep: number, maxAgeMs: number}} ReleaseRetentionConfig
|
|
26
|
+
* @typedef {{keep: number, maxAgeMs: number, maxConcurrentDrainingReleaseGroups: number}} ReleaseRetentionConfig
|
|
27
27
|
* @typedef {{reconnectGraceMs: number}} OwnerRecoveryConfig
|
|
28
28
|
* @typedef {{command: string | undefined, timeoutMs: number}} ComposeDrainConfig
|
|
29
29
|
* @typedef {{command: string | undefined, intervalMs: number, timeoutMs: number}} ComposeInFlightConfig
|
|
@@ -55,6 +55,15 @@ const DEFAULT_DEPENDENCY_TIMEOUT_MS = 30000
|
|
|
55
55
|
// know to configure. A candidate that never becomes healthy still fails the deploy at the
|
|
56
56
|
// deadline while the incumbent keeps serving; the budget only moves when the gate reports.
|
|
57
57
|
const DEFAULT_HEALTH_TIMEOUT_MS = 120000
|
|
58
|
+
// Bound on how many release groups may drain at once before a new handoff is refused.
|
|
59
|
+
// Measured on TensorBuzz production (2026-11-14/15, shared/log/rollbridge.log): 16 releases
|
|
60
|
+
// drained in 18-114 minutes with a median of ~37 minutes, while deploys arrived every
|
|
61
|
+
// 3-7 minutes (7 traffic switches in 61 minutes on 2026-11-14). A bound of 1 — the value a
|
|
62
|
+
// consumer gate effectively had — is unsatisfiable at that cadence, so every deploy adds one
|
|
63
|
+
// more draining generation. 37 / 3 = 12 is the backlog the measured drain median produces at
|
|
64
|
+
// the fastest measured cadence, so this default admits normal operation and refuses only
|
|
65
|
+
// growth beyond it. 0 disables the bound.
|
|
66
|
+
const DEFAULT_MAX_CONCURRENT_DRAINING_RELEASE_GROUPS = 12
|
|
58
67
|
const commonJsRequire = createRequire(import.meta.url)
|
|
59
68
|
|
|
60
69
|
/**
|
|
@@ -876,10 +885,12 @@ function normalizeBoolean(value, key, issues, fallback) {
|
|
|
876
885
|
function normalizeReleaseRetention(source, issues) {
|
|
877
886
|
const keep = normalizeNumber(source.keep, "releaseRetention.keep", issues, {default: 10})
|
|
878
887
|
const maxAgeMs = normalizeNumber(source.maxAgeMs, "releaseRetention.maxAgeMs", issues, {default: 0})
|
|
888
|
+
const maxConcurrentDrainingReleaseGroups = normalizeNumber(source.maxConcurrentDrainingReleaseGroups, "releaseRetention.maxConcurrentDrainingReleaseGroups", issues, {default: DEFAULT_MAX_CONCURRENT_DRAINING_RELEASE_GROUPS})
|
|
879
889
|
|
|
880
890
|
return {
|
|
881
891
|
keep: nonNegativeOrDefault(keep, "releaseRetention.keep", issues, 10, true),
|
|
882
|
-
maxAgeMs: nonNegativeOrDefault(maxAgeMs, "releaseRetention.maxAgeMs", issues, 0, false)
|
|
892
|
+
maxAgeMs: nonNegativeOrDefault(maxAgeMs, "releaseRetention.maxAgeMs", issues, 0, false),
|
|
893
|
+
maxConcurrentDrainingReleaseGroups: nonNegativeOrDefault(maxConcurrentDrainingReleaseGroups, "releaseRetention.maxConcurrentDrainingReleaseGroups", issues, DEFAULT_MAX_CONCURRENT_DRAINING_RELEASE_GROUPS, true)
|
|
883
894
|
}
|
|
884
895
|
}
|
|
885
896
|
|
package/src/daemon.js
CHANGED
|
@@ -9,8 +9,10 @@ import {isDeepStrictEqual} from "node:util"
|
|
|
9
9
|
import httpProxy from "http-proxy"
|
|
10
10
|
import {loadConfig} from "./config.js"
|
|
11
11
|
import {openControlSession} from "./control-client.js"
|
|
12
|
+
import {admitGenerationRelease} from "./drain-policy.js"
|
|
12
13
|
import EventLog from "./event-log.js"
|
|
13
14
|
import GuardianClient from "./guardian-client.js"
|
|
15
|
+
import {childRecordIsGone} from "./managed-process.js"
|
|
14
16
|
import ReleaseGroup from "./release-group.js"
|
|
15
17
|
import {clearState, isProcessAlive, liveProcesses, readState, writeState} from "./state-store.js"
|
|
16
18
|
import {resolveGroupId, resolveUserId} from "./system-ids.js"
|
|
@@ -31,6 +33,7 @@ const STATE_PERSIST_INTERVAL_MS = 5000
|
|
|
31
33
|
* @typedef {Omit<import("./release-group.js").ReleaseStatus, "processes"> & {processes: ManagedProcessStatusWithoutLogs[]}} ReleaseStatusWithoutLogs
|
|
32
34
|
* @typedef {Omit<ProcessStatus, "process"> & {process: ManagedProcessStatusWithoutLogs}} ProcessStatusWithoutLogs
|
|
33
35
|
* @typedef {Omit<DaemonStatus, "releases" | "services" | "singletons"> & {releases: ReleaseStatusWithoutLogs[], services: ProcessStatusWithoutLogs[], singletons: ProcessStatusWithoutLogs[]}} DaemonStatusWithoutLogs
|
|
36
|
+
* @typedef {{count: number, hold: string[], processes: {heldProcessGroupPgid: number | null, pid: number | null, processId: string, state: import("./managed-process.js").ManagedProcessState}[], state: import("./release-group.js").ReleaseState}} HeldGenerationHold
|
|
34
37
|
* @typedef {{configDigest: string, format: number, guardian: {pid?: number, socketPath: string, token: string}, reconnectGraceMs: number}} OwnerRecoveryMetadata
|
|
35
38
|
* @typedef {DaemonStatus & {recovery: OwnerRecoveryMetadata, serviceReleaseIds?: Record<string, string>, singletonReleaseIds?: Record<string, string>}} OwnerRecoverySnapshot
|
|
36
39
|
* @typedef {{authority: JsonValue, config: import("./config.js").RollbridgeConfig, listenerConnectionSources?: Record<string, Record<string, {http: number, websocket: number}>>, listenerSourceId?: string, recovery?: {command: JsonValue, reconnectGraceMs: number, startupTimeoutMs: number}, releaseConfigs?: Record<string, import("./config.js").RollbridgeConfig>, retiredListenerHandoff?: number, serviceReleaseIds?: Record<string, string>, singletonReleaseIds?: Record<string, string>, snapshot: OwnerRecoverySnapshot}} PrivateOwnerState
|
|
@@ -1631,6 +1634,40 @@ export default class RollbridgeDaemon {
|
|
|
1631
1634
|
await this.updateGenerationTransition("restoring_committed")
|
|
1632
1635
|
return await this.resumeGenerationTransition()
|
|
1633
1636
|
}
|
|
1637
|
+
// A release id that is still retained by a generation holding live work cannot be started
|
|
1638
|
+
// a second time: the candidate's processes re-register the guardian keys that the retained
|
|
1639
|
+
// generation's own processes still hold, so the deploy aborts partway through `start()`
|
|
1640
|
+
// with a bare `Guardian provenance mismatch` — after the caller has committed to this
|
|
1641
|
+
// revision and, for a restore, after the deployment pointer has already moved. Refusing
|
|
1642
|
+
// from the retained record before any process starts keeps the incumbent serving, names the
|
|
1643
|
+
// release the guardian reports as active, and leaves the retained generation intact for
|
|
1644
|
+
// `rollbridge status` to reconcile against. Journaled transition resume above is exempt:
|
|
1645
|
+
// it re-drives the transaction that already owns this release id.
|
|
1646
|
+
const heldGeneration = this.retainedGenerationHoldingLiveWork(newReleaseId)
|
|
1647
|
+
|
|
1648
|
+
if (heldGeneration) {
|
|
1649
|
+
const activeReleaseId = this.activeRelease?.releaseId ?? null
|
|
1650
|
+
|
|
1651
|
+
this.logger("deploy refused: retained generation still holds live processes", {
|
|
1652
|
+
activeReleaseId,
|
|
1653
|
+
heldProcesses: heldGeneration.processes,
|
|
1654
|
+
releaseId: newReleaseId,
|
|
1655
|
+
retainedState: heldGeneration.state
|
|
1656
|
+
})
|
|
1657
|
+
|
|
1658
|
+
throw new Error(`Deploy refused before activation: release "${newReleaseId}" is retained in state ${heldGeneration.state} while ${heldGeneration.count} process${heldGeneration.count === 1 ? "" : "es"} of that generation still hold live work (${heldGeneration.hold.join("; ")}). Activating that release id again would re-register those processes under a new provenance and abort with a guardian provenance mismatch. The guardian's active release is ${activeReleaseId ? `"${activeReleaseId}"` : "none"}; deploy a new release id or reconcile the deployment pointer with \`rollbridge status\`.`)
|
|
1659
|
+
}
|
|
1660
|
+
const admission = this.evaluateGenerationAdmission(newReleaseId, nextConfig)
|
|
1661
|
+
|
|
1662
|
+
if (!admission.allowed) {
|
|
1663
|
+
this.logger("deploy refused by draining-generation bound", {
|
|
1664
|
+
candidateReleaseId: newReleaseId,
|
|
1665
|
+
draining: admission.draining.map((group) => ({releaseId: group.releaseId, status: group.status})),
|
|
1666
|
+
limit: nextConfig.releaseRetention.maxConcurrentDrainingReleaseGroups,
|
|
1667
|
+
wouldBeDrainingCount: admission.wouldBeDrainingCount
|
|
1668
|
+
})
|
|
1669
|
+
throw new Error(/** @type {string} */ (admission.reason))
|
|
1670
|
+
}
|
|
1634
1671
|
const release = new ReleaseGroup({
|
|
1635
1672
|
config: nextConfig,
|
|
1636
1673
|
logger: this.logger,
|
|
@@ -1687,6 +1724,53 @@ export default class RollbridgeDaemon {
|
|
|
1687
1724
|
return await this.resumeGenerationTransition()
|
|
1688
1725
|
}
|
|
1689
1726
|
|
|
1727
|
+
/**
|
|
1728
|
+
* Applies the draining-generation bound before a handoff.
|
|
1729
|
+
*
|
|
1730
|
+
* Without this, every deploy retires one more generation and the backlog grows
|
|
1731
|
+
* monotonically whenever a drain outlasts the deploy cadence; a consumer that checks the
|
|
1732
|
+
* count after activation reports the growth it cannot undo. Refusing here names the
|
|
1733
|
+
* draining groups and whether each is still progressing or wedged, before any process of
|
|
1734
|
+
* the candidate is started.
|
|
1735
|
+
* @param {string} candidateReleaseId - Release the handoff would activate.
|
|
1736
|
+
* @param {import("./config.js").RollbridgeConfig} config - Config governing the handoff.
|
|
1737
|
+
* @returns {import("./drain-policy.js").GenerationAdmission} Admission decision and the groups behind it.
|
|
1738
|
+
*/
|
|
1739
|
+
evaluateGenerationAdmission(candidateReleaseId, config) {
|
|
1740
|
+
const draining = [...this.releases.values()]
|
|
1741
|
+
.filter((release) => release.state === "draining")
|
|
1742
|
+
.map((release) => {
|
|
1743
|
+
const drain = release.drainStatus()
|
|
1744
|
+
|
|
1745
|
+
return {reason: drain?.reason ?? "no liveness observation recorded", releaseId: release.releaseId, status: drain?.status ?? "unproven"}
|
|
1746
|
+
})
|
|
1747
|
+
|
|
1748
|
+
return admitGenerationRelease({
|
|
1749
|
+
candidateReleaseId,
|
|
1750
|
+
draining,
|
|
1751
|
+
incumbentWillDrain: this.activeRelease !== undefined,
|
|
1752
|
+
maxConcurrentDrainingReleaseGroups: config.releaseRetention.maxConcurrentDrainingReleaseGroups
|
|
1753
|
+
})
|
|
1754
|
+
}
|
|
1755
|
+
|
|
1756
|
+
/**
|
|
1757
|
+
* Reports whether a retained release group still holds live work that a re-activation would collide with.
|
|
1758
|
+
*
|
|
1759
|
+
* `stopped` is not the same claim as `drained`: the guardian's provenance fence only lets a
|
|
1760
|
+
* process key be re-registered once the record that holds it is gone. A generation recorded
|
|
1761
|
+
* as drained while its process group is still alive — the shape an indefinite worker drain
|
|
1762
|
+
* produces — still holds its keys, so its release id is not available for a restore.
|
|
1763
|
+
* @param {string} releaseId - Release id a handoff would activate.
|
|
1764
|
+
* @returns {HeldGenerationHold | undefined} The live hold, or undefined when the id is free.
|
|
1765
|
+
*/
|
|
1766
|
+
retainedGenerationHoldingLiveWork(releaseId) {
|
|
1767
|
+
const retained = this.releases.get(releaseId)
|
|
1768
|
+
|
|
1769
|
+
if (!retained || retained === this.activeRelease) return undefined
|
|
1770
|
+
|
|
1771
|
+
return heldGenerationHold(retained)
|
|
1772
|
+
}
|
|
1773
|
+
|
|
1690
1774
|
/**
|
|
1691
1775
|
* Continues one exact durable generation transition without an internal retry loop.
|
|
1692
1776
|
* @returns {Promise<Record<string, JsonValue>>} Deploy result after commit.
|
|
@@ -2407,6 +2491,31 @@ export default class RollbridgeDaemon {
|
|
|
2407
2491
|
throw new Error(`Release "${target.releaseId}" is already active.`)
|
|
2408
2492
|
}
|
|
2409
2493
|
|
|
2494
|
+
// A generation recorded as `stopped` whose processes are still alive is the shape a
|
|
2495
|
+
// restore cannot honour: re-deploying that release id re-registers its guardian process
|
|
2496
|
+
// keys under a new provenance while the old processes still hold them, and the guardian
|
|
2497
|
+
// answers with a bare provenance mismatch. Use the guardian's own fence here so the
|
|
2498
|
+
// refusal is decided before anything is touched, and name the divergence — including a
|
|
2499
|
+
// process group that outlived the tracked process and is invisible as a pid — together
|
|
2500
|
+
// with the release the guardian actually reports as active, so the deployment tool can
|
|
2501
|
+
// reconcile its own current pointer instead of restoring a generation that already drained.
|
|
2502
|
+
// A target that is still draining is not refused here: stopping it is what makes its id
|
|
2503
|
+
// reusable, and the deploy below applies the same fence after that stop.
|
|
2504
|
+
const heldGeneration = target.state === "stopped" ? heldGenerationHold(target) : undefined
|
|
2505
|
+
|
|
2506
|
+
if (heldGeneration) {
|
|
2507
|
+
const activeReleaseId = this.activeRelease?.releaseId ?? null
|
|
2508
|
+
|
|
2509
|
+
this.logger("rollback refused: drained generation still holds live processes", {
|
|
2510
|
+
activeReleaseId,
|
|
2511
|
+
heldProcesses: heldGeneration.processes,
|
|
2512
|
+
releaseId: target.releaseId,
|
|
2513
|
+
revision: target.revision
|
|
2514
|
+
})
|
|
2515
|
+
|
|
2516
|
+
throw new Error(`Release "${target.releaseId}" is recorded as drained while ${heldGeneration.count} process${heldGeneration.count === 1 ? "" : "es"} of that generation still hold live work (${heldGeneration.hold.join("; ")}). Restoring it would re-register those processes under a new provenance and fail with a guardian provenance mismatch. The guardian's active release is ${activeReleaseId ? `"${activeReleaseId}"` : "none"}; reconcile the deployment pointer with \`rollbridge status\` before restoring.`)
|
|
2517
|
+
}
|
|
2518
|
+
|
|
2410
2519
|
// The target may still be draining a prior deploy (live processes). Stop it before the
|
|
2411
2520
|
// deploy below re-uses its id in this.releases, otherwise the still-running instance
|
|
2412
2521
|
// would be dropped from status/pruning/shutdown and could be orphaned.
|
|
@@ -3073,6 +3182,42 @@ function statusIncludeLogs(value) {
|
|
|
3073
3182
|
return value
|
|
3074
3183
|
}
|
|
3075
3184
|
|
|
3185
|
+
/**
|
|
3186
|
+
* Names the live work a retained release group still holds, using the guardian's own fence.
|
|
3187
|
+
*
|
|
3188
|
+
* A process counts as holding work when the guardian would refuse to hand its key to a new
|
|
3189
|
+
* provenance: a running or stopping process, or a record already `stopped` whose process group
|
|
3190
|
+
* outlived it and is invisible as a pid.
|
|
3191
|
+
* @param {ReleaseGroup} release - Retained generation to inspect.
|
|
3192
|
+
* @returns {HeldGenerationHold | undefined} The hold and its log payload, or undefined when every process is gone.
|
|
3193
|
+
*/
|
|
3194
|
+
function heldGenerationHold(release) {
|
|
3195
|
+
const heldProcesses = release.status().processes.filter((processStatus) => !childRecordIsGone(processStatus))
|
|
3196
|
+
|
|
3197
|
+
if (heldProcesses.length === 0) return undefined
|
|
3198
|
+
|
|
3199
|
+
return {
|
|
3200
|
+
count: heldProcesses.length,
|
|
3201
|
+
hold: heldProcesses.map((processStatus) => {
|
|
3202
|
+
if (processStatus.heldProcessGroup) {
|
|
3203
|
+
const {liveMemberCount, pgid} = processStatus.heldProcessGroup
|
|
3204
|
+
const members = liveMemberCount === undefined ? "" : ` with ${liveMemberCount} live member(s)`
|
|
3205
|
+
|
|
3206
|
+
return `${processStatus.id}: process group ${pgid}${members} still alive after the tracked process exited`
|
|
3207
|
+
}
|
|
3208
|
+
|
|
3209
|
+
return `${processStatus.id}: state ${processStatus.state}, pid ${processStatus.pid ?? "unknown"}`
|
|
3210
|
+
}),
|
|
3211
|
+
processes: heldProcesses.map((processStatus) => ({
|
|
3212
|
+
heldProcessGroupPgid: processStatus.heldProcessGroup?.pgid ?? null,
|
|
3213
|
+
pid: processStatus.pid ?? null,
|
|
3214
|
+
processId: processStatus.id,
|
|
3215
|
+
state: processStatus.state
|
|
3216
|
+
})),
|
|
3217
|
+
state: release.state
|
|
3218
|
+
}
|
|
3219
|
+
}
|
|
3220
|
+
|
|
3076
3221
|
/**
|
|
3077
3222
|
* @param {PrivateOwnerState} transfer - Authenticated private incumbent state.
|
|
3078
3223
|
* @param {GenerationTransition} transition - Unresolved generation transition.
|
|
@@ -3455,7 +3600,7 @@ function requiredNonNegativeInteger(value, key) {
|
|
|
3455
3600
|
/**
|
|
3456
3601
|
* Selects stopped releases to prune by the retention policy, keeping the most recent.
|
|
3457
3602
|
* @param {PrunableRelease[]} releases - Status of all tracked releases, in deploy order (oldest first).
|
|
3458
|
-
* @param {
|
|
3603
|
+
* @param {{keep: number, maxAgeMs: number}} policy - Retention policy; draining generations are never pruned here.
|
|
3459
3604
|
* @param {number} now - Current epoch milliseconds.
|
|
3460
3605
|
* @returns {string[]} Release ids to remove.
|
|
3461
3606
|
*/
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
// @ts-check
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* @typedef {"progressing" | "wedged" | "unproven"} DrainProgress
|
|
5
|
+
* @typedef {{cpuMs: number | undefined, heldProcessGroupPgid: number | undefined, id: string, liveChildCount: number | undefined, pid: number | undefined, retirementTimedOut: boolean}} DrainProcessObservation
|
|
6
|
+
* @typedef {{atMs: number, processes: DrainProcessObservation[]}} DrainLivenessSample
|
|
7
|
+
* @typedef {{cpuDeltaMs: number | undefined, reason: string, sampleWindowMs: number, status: DrainProgress}} DrainClassification
|
|
8
|
+
* @typedef {{releaseId: string, reason: string, status: DrainProgress}} DrainGroupReport
|
|
9
|
+
* @typedef {{allowed: boolean, draining: DrainGroupReport[], reason: string | undefined, wouldBeDrainingCount: number}} GenerationAdmission
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Classifies one draining release group as still draining or wedged.
|
|
14
|
+
*
|
|
15
|
+
* A group is wedged only when its drain can no longer progress. Every inconclusive reading
|
|
16
|
+
* — a live connection, a process still inside its retirement budget, an unmeasurable CPU
|
|
17
|
+
* counter, or a sample window shorter than `minSampleWindowMs` — reports `progressing` or
|
|
18
|
+
* `unproven`, never `wedged`: a legitimate multi-hour drain is valid and must not be
|
|
19
|
+
* mislabelled as a wedge.
|
|
20
|
+
* @param {object} args - Classification inputs.
|
|
21
|
+
* @param {number} args.connectionCount - Live proxy connections this generation still owns.
|
|
22
|
+
* @param {DrainLivenessSample} args.currentSample - Newest liveness observation.
|
|
23
|
+
* @param {number} args.minSampleWindowMs - Shortest window that can prove flat CPU.
|
|
24
|
+
* @param {DrainLivenessSample | undefined} args.previousSample - Older observation, when one is in range.
|
|
25
|
+
* @returns {DrainClassification} The classification and the evidence it rests on.
|
|
26
|
+
*/
|
|
27
|
+
export function classifyDrainProgress({connectionCount, currentSample, minSampleWindowMs, previousSample}) {
|
|
28
|
+
const live = currentSample.processes.filter((process) => process.pid !== undefined || process.heldProcessGroupPgid !== undefined)
|
|
29
|
+
const stillInsideBudget = live.filter((process) => !process.retirementTimedOut)
|
|
30
|
+
|
|
31
|
+
if (connectionCount > 0) {
|
|
32
|
+
return classify("progressing", `${connectionCount} live connection${connectionCount === 1 ? "" : "s"} still owned by the generation`, currentSample, previousSample)
|
|
33
|
+
}
|
|
34
|
+
if (stillInsideBudget.length > 0) {
|
|
35
|
+
return classify("progressing", `process ${stillInsideBudget[0].id} is still inside its retirement budget`, currentSample, previousSample)
|
|
36
|
+
}
|
|
37
|
+
if (live.length === 0) {
|
|
38
|
+
return classify("progressing", "no live process remains in the generation", currentSample, previousSample)
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
const liveChildCount = live.reduce((total, process) => total + (process.liveChildCount ?? 0), 0)
|
|
42
|
+
|
|
43
|
+
if (liveChildCount === 0) {
|
|
44
|
+
return classify("wedged", `${live.length} live process${live.length === 1 ? "" : "es"} past its retirement budget with no runner child remaining${heldGroupNote(live)}`, currentSample, previousSample)
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
if (!previousSample) {
|
|
48
|
+
return classify("unproven", `first liveness observation; a ${minSampleWindowMs}ms window is required before CPU flatness can be claimed`, currentSample, previousSample)
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
const sampleWindowMs = currentSample.atMs - previousSample.atMs
|
|
52
|
+
|
|
53
|
+
if (sampleWindowMs < minSampleWindowMs) {
|
|
54
|
+
return classify("unproven", `liveness window ${sampleWindowMs}ms is below the ${minSampleWindowMs}ms minimum`, currentSample, previousSample)
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
const currentCpuMs = totalCpuMs(currentSample)
|
|
58
|
+
const previousCpuMs = totalCpuMs(previousSample)
|
|
59
|
+
|
|
60
|
+
if (currentCpuMs === undefined || previousCpuMs === undefined) {
|
|
61
|
+
return classify("unproven", "process CPU is not measurable on this host", currentSample, previousSample)
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
const cpuDeltaMs = currentCpuMs - previousCpuMs
|
|
65
|
+
|
|
66
|
+
if (cpuDeltaMs !== 0) {
|
|
67
|
+
const change = cpuDeltaMs > 0
|
|
68
|
+
? `CPU advanced ${cpuDeltaMs}ms`
|
|
69
|
+
: `the group changed (${live.length} live process${live.length === 1 ? "" : "es"}, ${cpuDeltaMs}ms of consumed CPU left the group)`
|
|
70
|
+
|
|
71
|
+
return classify("progressing", `${change} across ${sampleWindowMs}ms`, currentSample, previousSample)
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
return classify("wedged", `${live.length} live process${live.length === 1 ? "" : "es"} with ${liveChildCount} runner ${liveChildCount === 1 ? "child" : "children"}, no live connection, every retirement budget expired, CPU flat across ${sampleWindowMs}ms${heldGroupNote(live)}`, currentSample, previousSample)
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* Decides whether a new release generation may be activated under the draining-group bound.
|
|
79
|
+
*
|
|
80
|
+
* The incumbent counts as a draining group because activating the candidate retires it: the
|
|
81
|
+
* count is what the fleet would have *after* the handoff, which is the only version of the
|
|
82
|
+
* number a pre-handoff decision can act on.
|
|
83
|
+
* @param {object} args - Admission inputs.
|
|
84
|
+
* @param {string} args.candidateReleaseId - Release the handoff would activate.
|
|
85
|
+
* @param {DrainGroupReport[]} args.draining - Every release group currently draining.
|
|
86
|
+
* @param {boolean} args.incumbentWillDrain - Whether an active generation is retired by this handoff.
|
|
87
|
+
* @param {number} args.maxConcurrentDrainingReleaseGroups - Policy bound; 0 disables the check.
|
|
88
|
+
* @returns {GenerationAdmission} Allow/deny plus the named reason and the groups behind it.
|
|
89
|
+
*/
|
|
90
|
+
export function admitGenerationRelease({candidateReleaseId, draining, incumbentWillDrain, maxConcurrentDrainingReleaseGroups}) {
|
|
91
|
+
const wouldBeDrainingCount = draining.length + (incumbentWillDrain ? 1 : 0)
|
|
92
|
+
|
|
93
|
+
if (maxConcurrentDrainingReleaseGroups === 0 || wouldBeDrainingCount <= maxConcurrentDrainingReleaseGroups) {
|
|
94
|
+
return {allowed: true, draining, reason: undefined, wouldBeDrainingCount}
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
const reported = draining.map((group) => `${group.releaseId} ${group.status} (${group.reason})`).join("; ")
|
|
98
|
+
const reason = `Deploy refused before activation: ${draining.length} release group${draining.length === 1 ? "" : "s"} already draining${reported ? ` (${reported})` : ""}. Activating ${candidateReleaseId} would make ${wouldBeDrainingCount}, above the releaseRetention.maxConcurrentDrainingReleaseGroups bound of ${maxConcurrentDrainingReleaseGroups}.`
|
|
99
|
+
|
|
100
|
+
return {allowed: false, draining, reason, wouldBeDrainingCount}
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* Names the process groups that outlived their tracked process.
|
|
105
|
+
*
|
|
106
|
+
* A tracked process can exit while the group it led keeps running (a shell wrapper killed by
|
|
107
|
+
* the stop signal, a runner child that outlives its parent). Those members still hold the
|
|
108
|
+
* generation's work and no rollbridge record can signal them, so a wedge claim has to say
|
|
109
|
+
* which group is holding it.
|
|
110
|
+
* @param {DrainProcessObservation[]} processes - Live observations of one generation.
|
|
111
|
+
* @returns {string} A note naming the held groups, or "" when every group is still tracked.
|
|
112
|
+
*/
|
|
113
|
+
function heldGroupNote(processes) {
|
|
114
|
+
const held = processes.filter((process) => process.heldProcessGroupPgid !== undefined)
|
|
115
|
+
|
|
116
|
+
if (held.length === 0) return ""
|
|
117
|
+
|
|
118
|
+
const named = held.map((process) => `${process.id} pgid ${process.heldProcessGroupPgid}`).join(", ")
|
|
119
|
+
|
|
120
|
+
return `; ${held.length} tracked process${held.length === 1 ? "" : "es"} already exited and left ${named} alive`
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/**
|
|
124
|
+
* @param {DrainProgress} status - Classification.
|
|
125
|
+
* @param {string} reason - Named evidence behind it.
|
|
126
|
+
* @param {DrainLivenessSample} currentSample - Newest observation.
|
|
127
|
+
* @param {DrainLivenessSample | undefined} previousSample - Older observation, when in range.
|
|
128
|
+
* @returns {DrainClassification} Classification payload.
|
|
129
|
+
*/
|
|
130
|
+
function classify(status, reason, currentSample, previousSample) {
|
|
131
|
+
const sampleWindowMs = previousSample ? currentSample.atMs - previousSample.atMs : 0
|
|
132
|
+
const currentCpuMs = totalCpuMs(currentSample)
|
|
133
|
+
const previousCpuMs = totalCpuMs(previousSample)
|
|
134
|
+
|
|
135
|
+
return {
|
|
136
|
+
cpuDeltaMs: currentCpuMs === undefined || previousCpuMs === undefined ? undefined : currentCpuMs - previousCpuMs,
|
|
137
|
+
reason,
|
|
138
|
+
sampleWindowMs,
|
|
139
|
+
status
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/**
|
|
144
|
+
* @param {DrainLivenessSample | undefined} sample - Observation to total.
|
|
145
|
+
* @returns {number | undefined} Summed CPU, or undefined when any live process is unmeasurable.
|
|
146
|
+
*/
|
|
147
|
+
function totalCpuMs(sample) {
|
|
148
|
+
if (!sample) return undefined
|
|
149
|
+
|
|
150
|
+
let total = 0
|
|
151
|
+
|
|
152
|
+
for (const process of sample.processes) {
|
|
153
|
+
if (process.pid === undefined && process.heldProcessGroupPgid === undefined) continue
|
|
154
|
+
if (process.cpuMs === undefined) return undefined
|
|
155
|
+
total += process.cpuMs
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
return total
|
|
159
|
+
}
|