velocious 1.0.676 → 1.0.677

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -3,7 +3,7 @@
3
3
  "velocious": "build/bin/velocious.js"
4
4
  },
5
5
  "name": "velocious",
6
- "version": "1.0.676",
6
+ "version": "1.0.677",
7
7
  "main": "build/index.js",
8
8
  "types": "build/index.d.ts",
9
9
  "files": [
@@ -1,13 +1,17 @@
1
1
  // @ts-check
2
2
 
3
3
  import { randomUUID } from "node:crypto"
4
+ import timeout from "awaitery/build/timeout.js"
4
5
  import runJobPayload, { BackgroundJobPerformedFailure } from "./job-runner.js"
6
+ import { boundedPooledRunnerInflightJobIds, isPooledChildShutdownReason, isPooledChildShutdownSignal } from "./pooled-runner-shutdown.js"
5
7
  import { closeRunnerConnections, closeRunnerFrameworkConnections, currentConfigurationOrNull } from "./runner-graceful-shutdown.js"
6
8
  import setRunnerProcessTitle from "./runner-process-title.js"
7
9
  import PooledRunnerBrokerIdentity from "./pooled-runner-broker-identity.js"
8
10
  import { runWithSharedTransactionBrokerConfig } from "../testing/shared-transaction-proxy-driver.js"
9
11
 
10
12
  const BASE_PROCESS_TITLE = "velocious background-jobs-runner"
13
+ /** A shutdown observation may delay resource teardown only for this bounded IPC send. */
14
+ const SHUTDOWN_OBSERVATION_SEND_TIMEOUT_MS = 100
11
15
  /** Stable identity of this pooled child process for the life of the process. */
12
16
  const childInstanceId = randomUUID()
13
17
 
@@ -20,13 +24,18 @@ let shutdownPromise
20
24
  * Closes the runner's connections — releasing any advisory lock a killed-mid-pass
21
25
  * job still holds — before exiting, instead of leaving a half-open session that
22
26
  * keeps the lock until the DB server's `wait_timeout`.
23
- * @param {number} exitCode - Process exit code.
27
+ * @param {object} args - Shutdown observation.
28
+ * @param {number} args.exitCode - Process exit code.
29
+ * @param {import("./types.js").PooledChildShutdownReason} args.reason - Exact shutdown reason observed by the child.
30
+ * @param {number | null} [args.shutdownRequestedAtMs] - Parent request timestamp when supplied over IPC.
31
+ * @param {import("node:child_process").ChildProcess["signalCode"]} [args.signal] - Requested or observed signal.
24
32
  * @returns {Promise<void>}
25
33
  */
26
- function shutdownRunner(exitCode) {
34
+ function shutdownRunner({exitCode, reason, shutdownRequestedAtMs = null, signal = null}) {
27
35
  if (shutdownPromise) return shutdownPromise
28
36
 
29
37
  shutdownPromise = (async () => {
38
+ await sendShutdownObservation({reason, shutdownRequestedAtMs, signal})
30
39
  await closeRunnerConnections(currentConfigurationOrNull())
31
40
  process.exit(exitCode)
32
41
  })()
@@ -60,6 +69,52 @@ function updateProcessTitle() {
60
69
  process.title = count > 0 ? `${BASE_PROCESS_TITLE}: ${count} ${count === 1 ? "job" : "jobs"}` : BASE_PROCESS_TITLE
61
70
  }
62
71
 
72
+ /**
73
+ * Sends one bounded lifecycle observation before application/framework teardown.
74
+ * A closed IPC channel is expected for `ipc_disconnect`; other send failures are
75
+ * surfaced on stderr but never hold connection cleanup past the fixed bound.
76
+ * @param {object} args - Shutdown observation.
77
+ * @param {import("./types.js").PooledChildShutdownReason} args.reason - Shutdown reason.
78
+ * @param {number | null} args.shutdownRequestedAtMs - Parent request timestamp.
79
+ * @param {import("node:child_process").ChildProcess["signalCode"]} args.signal - Requested or observed signal.
80
+ * @returns {Promise<void>} - Resolves after IPC accepts the observation or the bound expires.
81
+ */
82
+ async function sendShutdownObservation({reason, shutdownRequestedAtMs, signal}) {
83
+ if (!process.send || !process.connected) return
84
+
85
+ const boundedInflight = boundedPooledRunnerInflightJobIds(runningJobIds)
86
+ /** @type {import("./types.js").PooledChildShutdownObservation & {type: "shutdown-observation"}} */
87
+ const message = {
88
+ childInstanceId,
89
+ ...boundedInflight,
90
+ reason,
91
+ shutdownObservedAtMs: Date.now(),
92
+ shutdownRequestedAtMs,
93
+ signal,
94
+ type: "shutdown-observation"
95
+ }
96
+
97
+ try {
98
+ await timeout({timeout: SHUTDOWN_OBSERVATION_SEND_TIMEOUT_MS}, async () => {
99
+ await new Promise((resolve, reject) => {
100
+ try {
101
+ process.send?.(message, (error) => {
102
+ if (error) {
103
+ reject(error)
104
+ } else {
105
+ resolve(undefined)
106
+ }
107
+ })
108
+ } catch (error) {
109
+ reject(error)
110
+ }
111
+ })
112
+ })
113
+ } catch (error) {
114
+ console.error("Pooled background job runner could not send its shutdown observation:", error)
115
+ }
116
+ }
117
+
63
118
  /**
64
119
  * Reports one acceptance observation (job received / perform started) to the
65
120
  * worker over IPC. The message carries the job's exact handoff lease so the
@@ -105,6 +160,22 @@ function isJobMessage(message) {
105
160
  return record.type === "job" && !!record.payload && typeof record.payload === "object" && typeof record.payload.id === "string"
106
161
  }
107
162
 
163
+ /**
164
+ * Checks whether an IPC value requests a typed pooled-child shutdown.
165
+ * @param {ReturnType<typeof JSON.parse>} message - IPC message.
166
+ * @returns {message is {type: "shutdown-request", reason: import("./types.js").PooledChildShutdownReason, shutdownRequestedAtMs: number, signal: import("node:child_process").ChildProcess["signalCode"]}} - Whether this is a valid parent shutdown request.
167
+ */
168
+ function isShutdownRequestMessage(message) {
169
+ if (!message || typeof message !== "object") return false
170
+ const record = /** @type {{type?: ReturnType<typeof JSON.parse>, reason?: ReturnType<typeof JSON.parse>, shutdownRequestedAtMs?: ReturnType<typeof JSON.parse>, signal?: ReturnType<typeof JSON.parse>}} */ (message)
171
+
172
+ return record.type === "shutdown-request"
173
+ && isPooledChildShutdownReason(record.reason)
174
+ && typeof record.shutdownRequestedAtMs === "number"
175
+ && Number.isFinite(record.shutdownRequestedAtMs)
176
+ && isPooledChildShutdownSignal(record.signal)
177
+ }
178
+
108
179
  /**
109
180
  * Sends the terminal outcome after the main/DB report has been acknowledged or rejected.
110
181
  * @param {object} args - Outcome.
@@ -176,6 +247,16 @@ async function runJob(payload, sharedTransactionBroker) {
176
247
  * @returns {void}
177
248
  */
178
249
  function handleMessage(message) {
250
+ if (isShutdownRequestMessage(message)) {
251
+ void shutdownRunner({
252
+ exitCode: message.reason === "parent_retire_drained" ? 0 : 1,
253
+ reason: message.reason,
254
+ shutdownRequestedAtMs: message.shutdownRequestedAtMs,
255
+ signal: message.signal
256
+ })
257
+ return
258
+ }
259
+
179
260
  if (!isJobMessage(message) || runningJobIds.has(message.payload.id)) return
180
261
 
181
262
  runningJobIds.add(message.payload.id)
@@ -185,6 +266,7 @@ function handleMessage(message) {
185
266
  }
186
267
 
187
268
  process.on("message", (message) => handleMessage(message))
188
- process.once("disconnect", () => void shutdownRunner(0))
189
- for (const signal of ["SIGTERM", "SIGINT"]) process.once(signal, () => void shutdownRunner(1))
190
- if (process.send) process.send({type: "ready"})
269
+ process.once("disconnect", () => void shutdownRunner({exitCode: 0, reason: "ipc_disconnect"}))
270
+ process.once("SIGTERM", () => void shutdownRunner({exitCode: 1, reason: "signal_sigterm", signal: "SIGTERM"}))
271
+ process.once("SIGINT", () => void shutdownRunner({exitCode: 1, reason: "signal_sigint", signal: "SIGINT"}))
272
+ if (process.send) process.send({childInstanceId, childPid: process.pid, type: "ready"})
@@ -0,0 +1,55 @@
1
+ // @ts-check
2
+
3
+ import { constants } from "node:os"
4
+
5
+ /** Maximum job ids copied into a pooled-child shutdown observation or failure snapshot. */
6
+ export const POOLED_RUNNER_INFLIGHT_JOB_ID_LIMIT = 100
7
+
8
+ /** @type {readonly import("./types.js").PooledChildShutdownReason[]} */
9
+ export const POOLED_CHILD_SHUTDOWN_REASONS = Object.freeze([
10
+ "parent_retire_drained",
11
+ "job_timeout",
12
+ "worker_stop",
13
+ "signal_sigterm",
14
+ "signal_sigint",
15
+ "signal_sigkill",
16
+ "signal_other",
17
+ "ipc_disconnect",
18
+ "process_error",
19
+ "unexpected_exit"
20
+ ])
21
+
22
+ /**
23
+ * Checks one dynamic IPC reason against the complete shutdown contract.
24
+ * @param {ReturnType<typeof JSON.parse>} value - Candidate shutdown reason.
25
+ * @returns {value is import("./types.js").PooledChildShutdownReason} - Whether the value is a known reason.
26
+ */
27
+ export function isPooledChildShutdownReason(value) {
28
+ if (typeof value !== "string") return false
29
+
30
+ return POOLED_CHILD_SHUTDOWN_REASONS.some((reason) => reason === value)
31
+ }
32
+
33
+ /**
34
+ * Checks one dynamic IPC value against Node's named process signals.
35
+ * @param {ReturnType<typeof JSON.parse>} value - Candidate signal.
36
+ * @returns {value is import("node:child_process").ChildProcess["signalCode"]} - Whether the value is null or a named signal.
37
+ */
38
+ export function isPooledChildShutdownSignal(value) {
39
+ return value === null || (typeof value === "string" && Object.hasOwn(constants.signals, value))
40
+ }
41
+
42
+ /**
43
+ * Sorts, deduplicates, and bounds job ids before they cross IPC/report boundaries.
44
+ * @param {Iterable<string>} jobIds - In-flight durable job ids.
45
+ * @returns {{inflightJobIds: string[], inflightJobIdsTruncatedCount: number}} - Bounded snapshot.
46
+ */
47
+ export function boundedPooledRunnerInflightJobIds(jobIds) {
48
+ const sortedJobIds = [...new Set(jobIds)].sort((left, right) => left.localeCompare(right))
49
+ const inflightJobIds = sortedJobIds.slice(0, POOLED_RUNNER_INFLIGHT_JOB_ID_LIMIT)
50
+
51
+ return {
52
+ inflightJobIds,
53
+ inflightJobIdsTruncatedCount: sortedJobIds.length - inflightJobIds.length
54
+ }
55
+ }
@@ -11,7 +11,11 @@
11
11
  /** @typedef {BackgroundJobActiveStatus | BackgroundJobTerminalStatus} BackgroundJobStatus */
12
12
  /** @typedef {"exit" | "process-error" | "ipc-send"} PooledRunnerFailureOrigin */
13
13
  /** @typedef {"starting" | "running" | "retiring"} PooledRunnerLifecycleState */
14
- /** @typedef {"unexpected" | "job-timeout" | "worker-shutdown-timeout"} PooledRunnerTerminationReason */
14
+ /** @typedef {"parent_retire_drained" | "job_timeout" | "worker_stop" | "signal_sigterm" | "signal_sigint" | "signal_sigkill" | "signal_other" | "ipc_disconnect" | "process_error" | "unexpected_exit"} PooledChildShutdownReason */
15
+ /**
16
+ * @deprecated Use PooledChildShutdownReason for exact shutdown provenance.
17
+ * @typedef {"unexpected" | "job-timeout" | "worker-shutdown-timeout"} PooledRunnerTerminationReason
18
+ */
15
19
  /** @typedef {"running" | "retiring" | "stopping"} BackgroundJobsWorkerLifecycleState */
16
20
  /**
17
21
  * Exact durable handoff ownership carried by an executing job when it produces
@@ -34,8 +38,11 @@
34
38
  * One process-failure snapshot shared by every job lost with a pooled child.
35
39
  * @typedef {object} PooledRunnerFailure
36
40
  * @property {PooledRunnerActiveJob[]} activeJobs - Jobs that were in flight when the child failed, ordered by job id.
41
+ * @property {string | null} childInstanceId - Stable pooled-child identity when its startup or shutdown observation arrived.
37
42
  * @property {number | null} exitCode - Child exit code, or null for signal/process errors.
38
43
  * @property {string | null} generationId - Release generation identity, or null in legacy mode.
44
+ * @property {string[]} inflightJobIds - Bounded job ids in flight when failure handling started.
45
+ * @property {number} inflightJobIdsTruncatedCount - In-flight ids omitted from the bounded snapshot.
39
46
  * @property {boolean | null} oomKilled - False when the observed exit rules OOM out; null when an unexpected SIGKILL cannot be distinguished from an OOM kill without supervisor/kernel evidence.
40
47
  * @property {PooledRunnerFailureOrigin} origin - Worker observation that initiated failure handling.
41
48
  * @property {number} runnerAgeMs - Child age when failure handling started.
@@ -45,12 +52,27 @@
45
52
  * @property {PooledRunnerLifecycleState} runnerLifecycle - Child lifecycle immediately before recovery.
46
53
  * @property {number | null} runnerPid - Child process id when available.
47
54
  * @property {import("node:child_process").ChildProcess["signalCode"]} signal - Child termination signal when available.
48
- * @property {PooledRunnerTerminationReason} terminationReason - Why the worker expected or did not expect termination.
55
+ * @property {number | null} shutdownObservedAtMs - Time the child or parent observed shutdown beginning.
56
+ * @property {number | null} shutdownRequestedAtMs - Exact parent request timestamp, or null for external/unrequested shutdown.
57
+ * @property {PooledChildShutdownReason} shutdownReason - Exact recorded parent request or observed child shutdown cause.
58
+ * @property {import("node:child_process").ChildProcess["signalCode"]} shutdownSignal - Parent-requested or child-observed shutdown signal when available.
59
+ * @property {PooledRunnerTerminationReason} terminationReason - Deprecated compatibility category; use shutdownReason for exact provenance.
49
60
  * @property {string | null} timeoutJobId - Job whose timeout initiated child termination, or null.
50
61
  * @property {string} workerId - Stable generation-qualified worker id.
51
62
  * @property {BackgroundJobsWorkerLifecycleState} workerLifecycle - Parent worker lifecycle immediately before recovery.
52
63
  * @property {number} workerPid - Parent worker process id.
53
64
  */
65
+ /**
66
+ * Best-effort observation sent before a pooled child closes its resources.
67
+ * @typedef {object} PooledChildShutdownObservation
68
+ * @property {string} childInstanceId - Stable pooled-child identity.
69
+ * @property {string[]} inflightJobIds - Bounded in-flight durable job ids.
70
+ * @property {number} inflightJobIdsTruncatedCount - In-flight ids omitted from the bounded snapshot.
71
+ * @property {PooledChildShutdownReason} reason - Shutdown reason observed by the child.
72
+ * @property {number} shutdownObservedAtMs - Child observation timestamp.
73
+ * @property {number | null} shutdownRequestedAtMs - Parent request timestamp when supplied over IPC.
74
+ * @property {import("node:child_process").ChildProcess["signalCode"]} signal - Requested or observed signal when available.
75
+ */
54
76
  /**
55
77
  * @typedef {object} LocalBackgroundJobsClock
56
78
  * @property {() => number} now - Current epoch milliseconds.
@@ -14,6 +14,7 @@ import performBackgroundJob from "./perform-job.js"
14
14
  import { runWithBackgroundJobPayload } from "./execution-context.js"
15
15
  import { createGenerationWorkerId } from "./generation-identity.js"
16
16
  import BackgroundJobsGenerationHandshakeTimeoutError, { DEFAULT_GENERATION_HANDSHAKE_TIMEOUT_MS, validateGenerationHandshakeTimeoutMs } from "./generation-handshake-timeout-error.js"
17
+ import { POOLED_RUNNER_INFLIGHT_JOB_ID_LIMIT, boundedPooledRunnerInflightJobIds, isPooledChildShutdownReason, isPooledChildShutdownSignal } from "./pooled-runner-shutdown.js"
17
18
 
18
19
  /**
19
20
  * Per-forked-child timeout bookkeeping.
@@ -33,18 +34,26 @@ import BackgroundJobsGenerationHandshakeTimeoutError, { DEFAULT_GENERATION_HANDS
33
34
  /**
34
35
  * @typedef {object} PooledChildState
35
36
  * @property {number} createdAtMs - Child creation timestamp.
37
+ * @property {string} [childInstanceId] - Stable identity reported by the child.
36
38
  * @property {number} jobsRun - Acknowledged jobs completed by this child.
37
39
  * @property {Map<string, PooledJobEntry>} inflight - Jobs currently owned by this child.
38
40
  * @property {number} lastDispatchSeq - Round-robin dispatch sequence.
39
41
  * @property {boolean} retiring - Whether this child is draining before retirement.
40
42
  * @property {boolean} [started] - Whether the child completed its startup handshake.
41
43
  * @property {boolean} [settling] - Whether failure handling already owns this child.
44
+ * @property {number} [ipcDisconnectedAtMs] - Parent observation of IPC disconnect.
45
+ * @property {import("./types.js").PooledChildShutdownObservation} [shutdownObservation] - Child observation sent before teardown.
46
+ * @property {import("./types.js").PooledChildShutdownReason} [shutdownReason] - Exact parent-requested shutdown reason.
47
+ * @property {number} [shutdownRequestedAtMs] - Exact parent shutdown-request timestamp.
48
+ * @property {import("node:child_process").ChildProcess["signalCode"]} [shutdownSignal] - Signal selected by the parent request.
49
+ * @property {ReturnType<typeof setTimeout>} [shutdownSignalTimer] - Drained-retirement fallback signal timer.
42
50
  * @property {ReturnType<typeof setTimeout> | null} [timeoutSigkillTimer] - Pending timeout SIGKILL timer.
43
- * @property {import("./types.js").PooledRunnerTerminationReason} [terminationReason] - Expected termination reason.
44
51
  * @property {string} [timeoutJobId] - Job whose timeout initiated termination.
45
52
  */
46
53
  /** Grace period after SIGTERM before a lingering process runner is SIGKILLed. */
47
54
  const FORKED_CHILD_SIGKILL_GRACE_MS = 5000
55
+ /** Time a drained retirement gives the child IPC request to begin teardown before SIGTERM fallback. */
56
+ const POOLED_RUNNER_SHUTDOWN_REQUEST_GRACE_MS = 250
48
57
  /**
49
58
  * Largest delay Node's `setTimeout` accepts without overflowing to a 1ms delay
50
59
  * (a 32-bit signed int of ms, ~24.8 days). A `jobTimeoutMs` above this — or a
@@ -100,6 +109,29 @@ function isChildAcceptanceMessage(message) {
100
109
  && (record.childPid === undefined || Number.isInteger(record.childPid))
101
110
  }
102
111
 
112
+ /**
113
+ * Checks whether an IPC value is the child's bounded pre-teardown observation.
114
+ * @param {ReturnType<typeof JSON.parse>} message - IPC message.
115
+ * @returns {message is import("./types.js").PooledChildShutdownObservation & {type: "shutdown-observation"}} - Whether this is a valid shutdown observation.
116
+ */
117
+ function isChildShutdownObservationMessage(message) {
118
+ if (!message || typeof message !== "object") return false
119
+ const record = /** @type {{childInstanceId?: ReturnType<typeof JSON.parse>, inflightJobIds?: ReturnType<typeof JSON.parse>, inflightJobIdsTruncatedCount?: ReturnType<typeof JSON.parse>, reason?: ReturnType<typeof JSON.parse>, shutdownObservedAtMs?: ReturnType<typeof JSON.parse>, shutdownRequestedAtMs?: ReturnType<typeof JSON.parse>, signal?: ReturnType<typeof JSON.parse>, type?: ReturnType<typeof JSON.parse>}} */ (message)
120
+
121
+ return record.type === "shutdown-observation"
122
+ && typeof record.childInstanceId === "string"
123
+ && Array.isArray(record.inflightJobIds)
124
+ && record.inflightJobIds.length <= POOLED_RUNNER_INFLIGHT_JOB_ID_LIMIT
125
+ && record.inflightJobIds.every((jobId) => typeof jobId === "string")
126
+ && Number.isInteger(record.inflightJobIdsTruncatedCount)
127
+ && record.inflightJobIdsTruncatedCount >= 0
128
+ && isPooledChildShutdownReason(record.reason)
129
+ && typeof record.shutdownObservedAtMs === "number"
130
+ && Number.isFinite(record.shutdownObservedAtMs)
131
+ && (record.shutdownRequestedAtMs === null || (typeof record.shutdownRequestedAtMs === "number" && Number.isFinite(record.shutdownRequestedAtMs)))
132
+ && isPooledChildShutdownSignal(record.signal)
133
+ }
134
+
103
135
  /**
104
136
  * Normalizes a candidate pooled-runner resource limit.
105
137
  * @param {number | undefined} value - Candidate positive number.
@@ -561,14 +593,14 @@ export default class BackgroundJobsWorker {
561
593
 
562
594
  for (const child of this.inflightProcessChildren) {
563
595
  const pooledState = this.pooledChildStates.get(child)
564
- if (pooledState && pooledState.inflight.size > 0 && !pooledState.terminationReason) {
565
- pooledState.terminationReason = "worker-shutdown-timeout"
566
- }
567
-
568
- try {
569
- child.kill("SIGTERM")
570
- } catch {
571
- // Child already exited; nothing to do.
596
+ if (pooledState) {
597
+ this._requestPooledChildShutdown({child, reason: "worker_stop", signal: "SIGTERM"})
598
+ } else {
599
+ try {
600
+ child.kill("SIGTERM")
601
+ } catch {
602
+ // Child already exited; nothing to do.
603
+ }
572
604
  }
573
605
  }
574
606
 
@@ -1185,16 +1217,10 @@ export default class BackgroundJobsWorker {
1185
1217
  const state = this.pooledChildStates.get(child)
1186
1218
 
1187
1219
  // Already settling/gone, or the job finished in the race with this timer.
1188
- if (!state || state.settling || state.terminationReason || !state.inflight.has(jobId)) return
1220
+ if (!state || state.settling || state.shutdownReason || !state.inflight.has(jobId)) return
1189
1221
 
1190
- state.terminationReason = "job-timeout"
1191
1222
  state.timeoutJobId = jobId
1192
-
1193
- try {
1194
- child.kill("SIGTERM")
1195
- } catch {
1196
- // Child already exited; nothing to do.
1197
- }
1223
+ this._requestPooledChildShutdown({child, reason: "job_timeout", signal: "SIGTERM"})
1198
1224
 
1199
1225
  state.timeoutSigkillTimer = setTimeout(() => {
1200
1226
  try {
@@ -1234,6 +1260,10 @@ export default class BackgroundJobsWorker {
1234
1260
  origin: "process-error",
1235
1261
  signal: child.signalCode
1236
1262
  }))
1263
+ child.once("disconnect", () => {
1264
+ const state = this.pooledChildStates.get(child)
1265
+ if (state) state.ipcDisconnectedAtMs ??= Date.now()
1266
+ })
1237
1267
  return child
1238
1268
  }
1239
1269
 
@@ -1247,10 +1277,24 @@ export default class BackgroundJobsWorker {
1247
1277
  */
1248
1278
  _handlePooledChildMessage({child, message}) {
1249
1279
  if (!message || typeof message !== "object") return
1250
- const record = /** @type {{type?: ReturnType<typeof JSON.parse>, jobId?: ReturnType<typeof JSON.parse>, acknowledged?: ReturnType<typeof JSON.parse>, rssBytes?: ReturnType<typeof JSON.parse>, error?: ReturnType<typeof JSON.parse>}} */ (message)
1280
+ const record = /** @type {{type?: ReturnType<typeof JSON.parse>, childInstanceId?: ReturnType<typeof JSON.parse>, jobId?: ReturnType<typeof JSON.parse>, acknowledged?: ReturnType<typeof JSON.parse>, rssBytes?: ReturnType<typeof JSON.parse>, error?: ReturnType<typeof JSON.parse>}} */ (message)
1251
1281
  const state = this.pooledChildStates.get(child)
1252
1282
  if (record.type === "ready") {
1253
- if (state) state.started = true
1283
+ if (state) {
1284
+ state.started = true
1285
+ if (typeof record.childInstanceId === "string") state.childInstanceId = record.childInstanceId
1286
+ }
1287
+ return
1288
+ }
1289
+ if (isChildShutdownObservationMessage(message)) {
1290
+ if (state) {
1291
+ state.childInstanceId = message.childInstanceId
1292
+ state.shutdownObservation = message
1293
+ if (state.shutdownSignalTimer) {
1294
+ clearTimeout(state.shutdownSignalTimer)
1295
+ state.shutdownSignalTimer = undefined
1296
+ }
1297
+ }
1254
1298
  return
1255
1299
  }
1256
1300
  if (isChildAcceptanceMessage(message)) {
@@ -1354,10 +1398,78 @@ export default class BackgroundJobsWorker {
1354
1398
  * @returns {void}
1355
1399
  */
1356
1400
  _retirePooledChild(child) {
1401
+ const state = this.pooledChildStates.get(child)
1402
+ if (!state) throw new Error("Cannot retire pooled child without tracked state")
1403
+ if (state.inflight.size > 0) {
1404
+ throw new Error(`Cannot retire pooled child while ${state.inflight.size} ${state.inflight.size === 1 ? "job remains" : "jobs remain"} in flight`)
1405
+ }
1406
+
1357
1407
  this.pooledChildren.delete(child)
1358
- this.pooledChildStates.delete(child)
1359
- this.inflightProcessChildren.delete(child)
1360
- child.kill("SIGTERM")
1408
+ state.retiring = true
1409
+ this._requestPooledChildShutdown({child, reason: "parent_retire_drained", signal: "SIGTERM", waitForObservation: true})
1410
+ }
1411
+
1412
+ /**
1413
+ * Records an exact parent request before IPC or signal delivery. Drained
1414
+ * retirement gets a brief IPC-first grace so its zero-job observation is
1415
+ * deterministic; timeout/worker-stop paths signal immediately.
1416
+ * @param {object} args - Shutdown request.
1417
+ * @param {import("node:child_process").ChildProcess} args.child - Pooled child.
1418
+ * @param {import("./types.js").PooledChildShutdownReason} args.reason - Exact parent reason.
1419
+ * @param {keyof typeof import("node:os").constants.signals} args.signal - Signal to deliver.
1420
+ * @param {boolean} [args.waitForObservation] - Whether IPC observation may precede signal fallback.
1421
+ * @returns {void}
1422
+ */
1423
+ _requestPooledChildShutdown({child, reason, signal, waitForObservation = false}) {
1424
+ const state = this.pooledChildStates.get(child)
1425
+ if (!state || state.settling || state.shutdownReason) return
1426
+
1427
+ const shutdownRequestedAtMs = Date.now()
1428
+ state.shutdownReason = reason
1429
+ state.shutdownRequestedAtMs = shutdownRequestedAtMs
1430
+ state.shutdownSignal = signal
1431
+
1432
+ let requestSent = false
1433
+ if (child.connected) {
1434
+ try {
1435
+ child.send({type: "shutdown-request", reason, shutdownRequestedAtMs, signal}, (error) => {
1436
+ if (!error || !waitForObservation || state.settling) return
1437
+
1438
+ if (state.shutdownSignalTimer) {
1439
+ clearTimeout(state.shutdownSignalTimer)
1440
+ state.shutdownSignalTimer = undefined
1441
+ }
1442
+ this._signalPooledChild({child, signal})
1443
+ })
1444
+ requestSent = true
1445
+ } catch {
1446
+ // The signal below remains the bounded shutdown mechanism.
1447
+ }
1448
+ }
1449
+
1450
+ if (waitForObservation && requestSent) {
1451
+ state.shutdownSignalTimer = setTimeout(() => {
1452
+ state.shutdownSignalTimer = undefined
1453
+ this._signalPooledChild({child, signal})
1454
+ }, POOLED_RUNNER_SHUTDOWN_REQUEST_GRACE_MS)
1455
+ state.shutdownSignalTimer.unref()
1456
+ return
1457
+ }
1458
+
1459
+ this._signalPooledChild({child, signal})
1460
+ }
1461
+
1462
+ /**
1463
+ * Delivers one parent-owned process signal without changing recorded provenance.
1464
+ * @param {{child: import("node:child_process").ChildProcess, signal: keyof typeof import("node:os").constants.signals}} args - Signal request.
1465
+ * @returns {void}
1466
+ */
1467
+ _signalPooledChild({child, signal}) {
1468
+ try {
1469
+ child.kill(signal)
1470
+ } catch {
1471
+ // Child already exited; its exit/error handler owns state settlement.
1472
+ }
1361
1473
  }
1362
1474
 
1363
1475
  /**
@@ -1384,6 +1496,7 @@ export default class BackgroundJobsWorker {
1384
1496
  // Cancel this child's pending timers before its in-flight set is reported —
1385
1497
  // the SIGKILL grace from a timeout kill, and every armed per-job backstop.
1386
1498
  if (state.timeoutSigkillTimer) clearTimeout(state.timeoutSigkillTimer)
1499
+ if (state.shutdownSignalTimer) clearTimeout(state.shutdownSignalTimer)
1387
1500
  for (const inflightEntry of state.inflight.values()) {
1388
1501
  if (inflightEntry.timeoutTimer) clearTimeout(inflightEntry.timeoutTimer)
1389
1502
  }
@@ -1415,19 +1528,24 @@ export default class BackgroundJobsWorker {
1415
1528
  // observe a replacement slot before the failed jobs' reports are in flight.
1416
1529
  // The report promises remain tracked below; a slow retry must not hold the
1417
1530
  // newly freed runner capacity hostage.
1418
- if (state && state.started !== false) {
1419
- this._sendReadyIfRunning()
1420
- } else if (state) {
1421
- for (const entry of entries) {
1422
- if (entry.pooledJob) this._pooledStartupFailureJobs.add(entry.pooledJob)
1423
- const queueTracker = this.pooledJobQueueTrackers.get(entry.payload.id)
1424
- if (queueTracker) this._pooledStartupFailureJobs.add(queueTracker)
1531
+ // A drained retirement already advertised its replacement capacity when
1532
+ // the final job completed. Re-advertising here can restore a credit main
1533
+ // consumed before its handoff reached the replacement child.
1534
+ if (state?.shutdownReason !== "parent_retire_drained") {
1535
+ if (state && state.started !== false) {
1536
+ this._sendReadyIfRunning()
1537
+ } else if (state) {
1538
+ for (const entry of entries) {
1539
+ if (entry.pooledJob) this._pooledStartupFailureJobs.add(entry.pooledJob)
1540
+ const queueTracker = this.pooledJobQueueTrackers.get(entry.payload.id)
1541
+ if (queueTracker) this._pooledStartupFailureJobs.add(queueTracker)
1542
+ }
1543
+ // A previous ready message may still have unconsumed pooled credits at the
1544
+ // main. Revoke them authoritatively without suppressing valid inline or
1545
+ // process-runner readiness; otherwise queued jobs can trigger a startup
1546
+ // crash loop using the stale credits.
1547
+ this._sendReadyIfRunning({revokePooledAdmission: true})
1425
1548
  }
1426
- // A previous ready message may still have unconsumed pooled credits at the
1427
- // main. Revoke them authoritatively without suppressing valid inline or
1428
- // process-runner readiness; otherwise queued jobs can trigger a startup
1429
- // crash loop using the stale credits.
1430
- this._sendReadyIfRunning({revokePooledAdmission: true})
1431
1549
  }
1432
1550
 
1433
1551
  await Promise.allSettled(failureReports)
@@ -1444,9 +1562,31 @@ export default class BackgroundJobsWorker {
1444
1562
  * @returns {import("./types.js").PooledRunnerFailure} - Shared failure provenance.
1445
1563
  */
1446
1564
  _pooledRunnerFailure({child, exitCode, origin, signal, state}) {
1447
- const terminationReason = state.terminationReason ?? "unexpected"
1565
+ const observation = state.shutdownObservation
1566
+ let shutdownReason = state.shutdownReason ?? observation?.reason
1567
+ if (!shutdownReason) {
1568
+ if (origin === "process-error" || origin === "ipc-send") {
1569
+ shutdownReason = "process_error"
1570
+ } else if (signal === "SIGTERM") {
1571
+ shutdownReason = "signal_sigterm"
1572
+ } else if (signal === "SIGINT") {
1573
+ shutdownReason = "signal_sigint"
1574
+ } else if (signal === "SIGKILL") {
1575
+ shutdownReason = "signal_sigkill"
1576
+ } else if (signal) {
1577
+ shutdownReason = "signal_other"
1578
+ } else if (state.ipcDisconnectedAtMs && exitCode === 0) {
1579
+ shutdownReason = "ipc_disconnect"
1580
+ } else {
1581
+ shutdownReason = "unexpected_exit"
1582
+ }
1583
+ }
1584
+ const terminationReason = shutdownReason === "job_timeout"
1585
+ ? "job-timeout"
1586
+ : shutdownReason === "worker_stop" ? "worker-shutdown-timeout" : "unexpected"
1448
1587
  const workerLifecycle = this.shouldStop ? "stopping" : this.isRetiring ? "retiring" : "running"
1449
1588
  const runnerLifecycle = state.started === false ? "starting" : state.retiring ? "retiring" : "running"
1589
+ const boundedInflight = boundedPooledRunnerInflightJobIds(state.inflight.keys())
1450
1590
  const activeJobs = [...state.inflight.values()]
1451
1591
  .map((entry) => ({
1452
1592
  handoffId: entry.payload.handoffId ?? null,
@@ -1459,9 +1599,11 @@ export default class BackgroundJobsWorker {
1459
1599
 
1460
1600
  return Object.freeze({
1461
1601
  activeJobs,
1602
+ childInstanceId: state.childInstanceId ?? observation?.childInstanceId ?? null,
1462
1603
  exitCode,
1463
1604
  generationId: this.generationId ?? null,
1464
- oomKilled: signal === "SIGKILL" && terminationReason === "unexpected" ? null : false,
1605
+ ...boundedInflight,
1606
+ oomKilled: signal === "SIGKILL" && shutdownReason === "signal_sigkill" ? null : false,
1465
1607
  origin,
1466
1608
  runnerAgeMs: Math.max(0, Date.now() - state.createdAtMs),
1467
1609
  runnerCreatedAtMs: state.createdAtMs,
@@ -1470,6 +1612,10 @@ export default class BackgroundJobsWorker {
1470
1612
  runnerLifecycle,
1471
1613
  runnerPid: child.pid ?? null,
1472
1614
  signal,
1615
+ shutdownObservedAtMs: observation?.shutdownObservedAtMs ?? state.ipcDisconnectedAtMs ?? null,
1616
+ shutdownRequestedAtMs: state.shutdownRequestedAtMs ?? observation?.shutdownRequestedAtMs ?? null,
1617
+ shutdownReason,
1618
+ shutdownSignal: state.shutdownSignal ?? observation?.signal ?? signal,
1473
1619
  terminationReason,
1474
1620
  timeoutJobId: state.timeoutJobId ?? null,
1475
1621
  workerId: this.workerId,