velocious 1.0.676 → 1.0.677
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/build/background-jobs/pooled-runner-child.js +87 -5
- package/build/background-jobs/pooled-runner-shutdown.js +55 -0
- package/build/background-jobs/types.js +24 -2
- package/build/background-jobs/worker.js +182 -36
- package/build/src/background-jobs/pooled-runner-child.js +87 -7
- package/build/src/background-jobs/pooled-runner-shutdown.d.ts +26 -0
- package/build/src/background-jobs/pooled-runner-shutdown.d.ts.map +1 -0
- package/build/src/background-jobs/pooled-runner-shutdown.js +49 -0
- package/build/src/background-jobs/types.d.ts +84 -3
- package/build/src/background-jobs/types.d.ts.map +1 -1
- package/build/src/background-jobs/types.js +25 -3
- package/build/src/background-jobs/worker.d.ts +54 -4
- package/build/src/background-jobs/worker.d.ts.map +1 -1
- package/build/src/background-jobs/worker.js +191 -39
- package/package.json +1 -1
- package/src/background-jobs/pooled-runner-child.js +87 -5
- package/src/background-jobs/pooled-runner-shutdown.js +55 -0
- package/src/background-jobs/types.js +24 -2
- package/src/background-jobs/worker.js +182 -36
package/package.json
CHANGED
|
@@ -1,13 +1,17 @@
|
|
|
1
1
|
// @ts-check
|
|
2
2
|
|
|
3
3
|
import { randomUUID } from "node:crypto"
|
|
4
|
+
import timeout from "awaitery/build/timeout.js"
|
|
4
5
|
import runJobPayload, { BackgroundJobPerformedFailure } from "./job-runner.js"
|
|
6
|
+
import { boundedPooledRunnerInflightJobIds, isPooledChildShutdownReason, isPooledChildShutdownSignal } from "./pooled-runner-shutdown.js"
|
|
5
7
|
import { closeRunnerConnections, closeRunnerFrameworkConnections, currentConfigurationOrNull } from "./runner-graceful-shutdown.js"
|
|
6
8
|
import setRunnerProcessTitle from "./runner-process-title.js"
|
|
7
9
|
import PooledRunnerBrokerIdentity from "./pooled-runner-broker-identity.js"
|
|
8
10
|
import { runWithSharedTransactionBrokerConfig } from "../testing/shared-transaction-proxy-driver.js"
|
|
9
11
|
|
|
10
12
|
const BASE_PROCESS_TITLE = "velocious background-jobs-runner"
|
|
13
|
+
/** A shutdown observation may delay resource teardown only for this bounded IPC send. */
|
|
14
|
+
const SHUTDOWN_OBSERVATION_SEND_TIMEOUT_MS = 100
|
|
11
15
|
/** Stable identity of this pooled child process for the life of the process. */
|
|
12
16
|
const childInstanceId = randomUUID()
|
|
13
17
|
|
|
@@ -20,13 +24,18 @@ let shutdownPromise
|
|
|
20
24
|
* Closes the runner's connections — releasing any advisory lock a killed-mid-pass
|
|
21
25
|
* job still holds — before exiting, instead of leaving a half-open session that
|
|
22
26
|
* keeps the lock until the DB server's `wait_timeout`.
|
|
23
|
-
* @param {
|
|
27
|
+
* @param {object} args - Shutdown observation.
|
|
28
|
+
* @param {number} args.exitCode - Process exit code.
|
|
29
|
+
* @param {import("./types.js").PooledChildShutdownReason} args.reason - Exact shutdown reason observed by the child.
|
|
30
|
+
* @param {number | null} [args.shutdownRequestedAtMs] - Parent request timestamp when supplied over IPC.
|
|
31
|
+
* @param {import("node:child_process").ChildProcess["signalCode"]} [args.signal] - Requested or observed signal.
|
|
24
32
|
* @returns {Promise<void>}
|
|
25
33
|
*/
|
|
26
|
-
function shutdownRunner(exitCode) {
|
|
34
|
+
function shutdownRunner({exitCode, reason, shutdownRequestedAtMs = null, signal = null}) {
|
|
27
35
|
if (shutdownPromise) return shutdownPromise
|
|
28
36
|
|
|
29
37
|
shutdownPromise = (async () => {
|
|
38
|
+
await sendShutdownObservation({reason, shutdownRequestedAtMs, signal})
|
|
30
39
|
await closeRunnerConnections(currentConfigurationOrNull())
|
|
31
40
|
process.exit(exitCode)
|
|
32
41
|
})()
|
|
@@ -60,6 +69,52 @@ function updateProcessTitle() {
|
|
|
60
69
|
process.title = count > 0 ? `${BASE_PROCESS_TITLE}: ${count} ${count === 1 ? "job" : "jobs"}` : BASE_PROCESS_TITLE
|
|
61
70
|
}
|
|
62
71
|
|
|
72
|
+
/**
|
|
73
|
+
* Sends one bounded lifecycle observation before application/framework teardown.
|
|
74
|
+
* A closed IPC channel is expected for `ipc_disconnect`; other send failures are
|
|
75
|
+
* surfaced on stderr but never hold connection cleanup past the fixed bound.
|
|
76
|
+
* @param {object} args - Shutdown observation.
|
|
77
|
+
* @param {import("./types.js").PooledChildShutdownReason} args.reason - Shutdown reason.
|
|
78
|
+
* @param {number | null} args.shutdownRequestedAtMs - Parent request timestamp.
|
|
79
|
+
* @param {import("node:child_process").ChildProcess["signalCode"]} args.signal - Requested or observed signal.
|
|
80
|
+
* @returns {Promise<void>} - Resolves after IPC accepts the observation or the bound expires.
|
|
81
|
+
*/
|
|
82
|
+
async function sendShutdownObservation({reason, shutdownRequestedAtMs, signal}) {
|
|
83
|
+
if (!process.send || !process.connected) return
|
|
84
|
+
|
|
85
|
+
const boundedInflight = boundedPooledRunnerInflightJobIds(runningJobIds)
|
|
86
|
+
/** @type {import("./types.js").PooledChildShutdownObservation & {type: "shutdown-observation"}} */
|
|
87
|
+
const message = {
|
|
88
|
+
childInstanceId,
|
|
89
|
+
...boundedInflight,
|
|
90
|
+
reason,
|
|
91
|
+
shutdownObservedAtMs: Date.now(),
|
|
92
|
+
shutdownRequestedAtMs,
|
|
93
|
+
signal,
|
|
94
|
+
type: "shutdown-observation"
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
try {
|
|
98
|
+
await timeout({timeout: SHUTDOWN_OBSERVATION_SEND_TIMEOUT_MS}, async () => {
|
|
99
|
+
await new Promise((resolve, reject) => {
|
|
100
|
+
try {
|
|
101
|
+
process.send?.(message, (error) => {
|
|
102
|
+
if (error) {
|
|
103
|
+
reject(error)
|
|
104
|
+
} else {
|
|
105
|
+
resolve(undefined)
|
|
106
|
+
}
|
|
107
|
+
})
|
|
108
|
+
} catch (error) {
|
|
109
|
+
reject(error)
|
|
110
|
+
}
|
|
111
|
+
})
|
|
112
|
+
})
|
|
113
|
+
} catch (error) {
|
|
114
|
+
console.error("Pooled background job runner could not send its shutdown observation:", error)
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
|
|
63
118
|
/**
|
|
64
119
|
* Reports one acceptance observation (job received / perform started) to the
|
|
65
120
|
* worker over IPC. The message carries the job's exact handoff lease so the
|
|
@@ -105,6 +160,22 @@ function isJobMessage(message) {
|
|
|
105
160
|
return record.type === "job" && !!record.payload && typeof record.payload === "object" && typeof record.payload.id === "string"
|
|
106
161
|
}
|
|
107
162
|
|
|
163
|
+
/**
|
|
164
|
+
* Checks whether an IPC value requests a typed pooled-child shutdown.
|
|
165
|
+
* @param {ReturnType<typeof JSON.parse>} message - IPC message.
|
|
166
|
+
* @returns {message is {type: "shutdown-request", reason: import("./types.js").PooledChildShutdownReason, shutdownRequestedAtMs: number, signal: import("node:child_process").ChildProcess["signalCode"]}} - Whether this is a valid parent shutdown request.
|
|
167
|
+
*/
|
|
168
|
+
function isShutdownRequestMessage(message) {
|
|
169
|
+
if (!message || typeof message !== "object") return false
|
|
170
|
+
const record = /** @type {{type?: ReturnType<typeof JSON.parse>, reason?: ReturnType<typeof JSON.parse>, shutdownRequestedAtMs?: ReturnType<typeof JSON.parse>, signal?: ReturnType<typeof JSON.parse>}} */ (message)
|
|
171
|
+
|
|
172
|
+
return record.type === "shutdown-request"
|
|
173
|
+
&& isPooledChildShutdownReason(record.reason)
|
|
174
|
+
&& typeof record.shutdownRequestedAtMs === "number"
|
|
175
|
+
&& Number.isFinite(record.shutdownRequestedAtMs)
|
|
176
|
+
&& isPooledChildShutdownSignal(record.signal)
|
|
177
|
+
}
|
|
178
|
+
|
|
108
179
|
/**
|
|
109
180
|
* Sends the terminal outcome after the main/DB report has been acknowledged or rejected.
|
|
110
181
|
* @param {object} args - Outcome.
|
|
@@ -176,6 +247,16 @@ async function runJob(payload, sharedTransactionBroker) {
|
|
|
176
247
|
* @returns {void}
|
|
177
248
|
*/
|
|
178
249
|
function handleMessage(message) {
|
|
250
|
+
if (isShutdownRequestMessage(message)) {
|
|
251
|
+
void shutdownRunner({
|
|
252
|
+
exitCode: message.reason === "parent_retire_drained" ? 0 : 1,
|
|
253
|
+
reason: message.reason,
|
|
254
|
+
shutdownRequestedAtMs: message.shutdownRequestedAtMs,
|
|
255
|
+
signal: message.signal
|
|
256
|
+
})
|
|
257
|
+
return
|
|
258
|
+
}
|
|
259
|
+
|
|
179
260
|
if (!isJobMessage(message) || runningJobIds.has(message.payload.id)) return
|
|
180
261
|
|
|
181
262
|
runningJobIds.add(message.payload.id)
|
|
@@ -185,6 +266,7 @@ function handleMessage(message) {
|
|
|
185
266
|
}
|
|
186
267
|
|
|
187
268
|
process.on("message", (message) => handleMessage(message))
|
|
188
|
-
process.once("disconnect", () => void shutdownRunner(0))
|
|
189
|
-
|
|
190
|
-
|
|
269
|
+
process.once("disconnect", () => void shutdownRunner({exitCode: 0, reason: "ipc_disconnect"}))
|
|
270
|
+
process.once("SIGTERM", () => void shutdownRunner({exitCode: 1, reason: "signal_sigterm", signal: "SIGTERM"}))
|
|
271
|
+
process.once("SIGINT", () => void shutdownRunner({exitCode: 1, reason: "signal_sigint", signal: "SIGINT"}))
|
|
272
|
+
if (process.send) process.send({childInstanceId, childPid: process.pid, type: "ready"})
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
// @ts-check
|
|
2
|
+
|
|
3
|
+
import { constants } from "node:os"
|
|
4
|
+
|
|
5
|
+
/** Maximum job ids copied into a pooled-child shutdown observation or failure snapshot. */
|
|
6
|
+
export const POOLED_RUNNER_INFLIGHT_JOB_ID_LIMIT = 100
|
|
7
|
+
|
|
8
|
+
/** @type {readonly import("./types.js").PooledChildShutdownReason[]} */
|
|
9
|
+
export const POOLED_CHILD_SHUTDOWN_REASONS = Object.freeze([
|
|
10
|
+
"parent_retire_drained",
|
|
11
|
+
"job_timeout",
|
|
12
|
+
"worker_stop",
|
|
13
|
+
"signal_sigterm",
|
|
14
|
+
"signal_sigint",
|
|
15
|
+
"signal_sigkill",
|
|
16
|
+
"signal_other",
|
|
17
|
+
"ipc_disconnect",
|
|
18
|
+
"process_error",
|
|
19
|
+
"unexpected_exit"
|
|
20
|
+
])
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* Checks one dynamic IPC reason against the complete shutdown contract.
|
|
24
|
+
* @param {ReturnType<typeof JSON.parse>} value - Candidate shutdown reason.
|
|
25
|
+
* @returns {value is import("./types.js").PooledChildShutdownReason} - Whether the value is a known reason.
|
|
26
|
+
*/
|
|
27
|
+
export function isPooledChildShutdownReason(value) {
|
|
28
|
+
if (typeof value !== "string") return false
|
|
29
|
+
|
|
30
|
+
return POOLED_CHILD_SHUTDOWN_REASONS.some((reason) => reason === value)
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* Checks one dynamic IPC value against Node's named process signals.
|
|
35
|
+
* @param {ReturnType<typeof JSON.parse>} value - Candidate signal.
|
|
36
|
+
* @returns {value is import("node:child_process").ChildProcess["signalCode"]} - Whether the value is null or a named signal.
|
|
37
|
+
*/
|
|
38
|
+
export function isPooledChildShutdownSignal(value) {
|
|
39
|
+
return value === null || (typeof value === "string" && Object.hasOwn(constants.signals, value))
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* Sorts, deduplicates, and bounds job ids before they cross IPC/report boundaries.
|
|
44
|
+
* @param {Iterable<string>} jobIds - In-flight durable job ids.
|
|
45
|
+
* @returns {{inflightJobIds: string[], inflightJobIdsTruncatedCount: number}} - Bounded snapshot.
|
|
46
|
+
*/
|
|
47
|
+
export function boundedPooledRunnerInflightJobIds(jobIds) {
|
|
48
|
+
const sortedJobIds = [...new Set(jobIds)].sort((left, right) => left.localeCompare(right))
|
|
49
|
+
const inflightJobIds = sortedJobIds.slice(0, POOLED_RUNNER_INFLIGHT_JOB_ID_LIMIT)
|
|
50
|
+
|
|
51
|
+
return {
|
|
52
|
+
inflightJobIds,
|
|
53
|
+
inflightJobIdsTruncatedCount: sortedJobIds.length - inflightJobIds.length
|
|
54
|
+
}
|
|
55
|
+
}
|
|
@@ -11,7 +11,11 @@
|
|
|
11
11
|
/** @typedef {BackgroundJobActiveStatus | BackgroundJobTerminalStatus} BackgroundJobStatus */
|
|
12
12
|
/** @typedef {"exit" | "process-error" | "ipc-send"} PooledRunnerFailureOrigin */
|
|
13
13
|
/** @typedef {"starting" | "running" | "retiring"} PooledRunnerLifecycleState */
|
|
14
|
-
/** @typedef {"
|
|
14
|
+
/** @typedef {"parent_retire_drained" | "job_timeout" | "worker_stop" | "signal_sigterm" | "signal_sigint" | "signal_sigkill" | "signal_other" | "ipc_disconnect" | "process_error" | "unexpected_exit"} PooledChildShutdownReason */
|
|
15
|
+
/**
|
|
16
|
+
* @deprecated Use PooledChildShutdownReason for exact shutdown provenance.
|
|
17
|
+
* @typedef {"unexpected" | "job-timeout" | "worker-shutdown-timeout"} PooledRunnerTerminationReason
|
|
18
|
+
*/
|
|
15
19
|
/** @typedef {"running" | "retiring" | "stopping"} BackgroundJobsWorkerLifecycleState */
|
|
16
20
|
/**
|
|
17
21
|
* Exact durable handoff ownership carried by an executing job when it produces
|
|
@@ -34,8 +38,11 @@
|
|
|
34
38
|
* One process-failure snapshot shared by every job lost with a pooled child.
|
|
35
39
|
* @typedef {object} PooledRunnerFailure
|
|
36
40
|
* @property {PooledRunnerActiveJob[]} activeJobs - Jobs that were in flight when the child failed, ordered by job id.
|
|
41
|
+
* @property {string | null} childInstanceId - Stable pooled-child identity when its startup or shutdown observation arrived.
|
|
37
42
|
* @property {number | null} exitCode - Child exit code, or null for signal/process errors.
|
|
38
43
|
* @property {string | null} generationId - Release generation identity, or null in legacy mode.
|
|
44
|
+
* @property {string[]} inflightJobIds - Bounded job ids in flight when failure handling started.
|
|
45
|
+
* @property {number} inflightJobIdsTruncatedCount - In-flight ids omitted from the bounded snapshot.
|
|
39
46
|
* @property {boolean | null} oomKilled - False when the observed exit rules OOM out; null when an unexpected SIGKILL cannot be distinguished from an OOM kill without supervisor/kernel evidence.
|
|
40
47
|
* @property {PooledRunnerFailureOrigin} origin - Worker observation that initiated failure handling.
|
|
41
48
|
* @property {number} runnerAgeMs - Child age when failure handling started.
|
|
@@ -45,12 +52,27 @@
|
|
|
45
52
|
* @property {PooledRunnerLifecycleState} runnerLifecycle - Child lifecycle immediately before recovery.
|
|
46
53
|
* @property {number | null} runnerPid - Child process id when available.
|
|
47
54
|
* @property {import("node:child_process").ChildProcess["signalCode"]} signal - Child termination signal when available.
|
|
48
|
-
* @property {
|
|
55
|
+
* @property {number | null} shutdownObservedAtMs - Time the child or parent observed shutdown beginning.
|
|
56
|
+
* @property {number | null} shutdownRequestedAtMs - Exact parent request timestamp, or null for external/unrequested shutdown.
|
|
57
|
+
* @property {PooledChildShutdownReason} shutdownReason - Exact recorded parent request or observed child shutdown cause.
|
|
58
|
+
* @property {import("node:child_process").ChildProcess["signalCode"]} shutdownSignal - Parent-requested or child-observed shutdown signal when available.
|
|
59
|
+
* @property {PooledRunnerTerminationReason} terminationReason - Deprecated compatibility category; use shutdownReason for exact provenance.
|
|
49
60
|
* @property {string | null} timeoutJobId - Job whose timeout initiated child termination, or null.
|
|
50
61
|
* @property {string} workerId - Stable generation-qualified worker id.
|
|
51
62
|
* @property {BackgroundJobsWorkerLifecycleState} workerLifecycle - Parent worker lifecycle immediately before recovery.
|
|
52
63
|
* @property {number} workerPid - Parent worker process id.
|
|
53
64
|
*/
|
|
65
|
+
/**
|
|
66
|
+
* Best-effort observation sent before a pooled child closes its resources.
|
|
67
|
+
* @typedef {object} PooledChildShutdownObservation
|
|
68
|
+
* @property {string} childInstanceId - Stable pooled-child identity.
|
|
69
|
+
* @property {string[]} inflightJobIds - Bounded in-flight durable job ids.
|
|
70
|
+
* @property {number} inflightJobIdsTruncatedCount - In-flight ids omitted from the bounded snapshot.
|
|
71
|
+
* @property {PooledChildShutdownReason} reason - Shutdown reason observed by the child.
|
|
72
|
+
* @property {number} shutdownObservedAtMs - Child observation timestamp.
|
|
73
|
+
* @property {number | null} shutdownRequestedAtMs - Parent request timestamp when supplied over IPC.
|
|
74
|
+
* @property {import("node:child_process").ChildProcess["signalCode"]} signal - Requested or observed signal when available.
|
|
75
|
+
*/
|
|
54
76
|
/**
|
|
55
77
|
* @typedef {object} LocalBackgroundJobsClock
|
|
56
78
|
* @property {() => number} now - Current epoch milliseconds.
|
|
@@ -14,6 +14,7 @@ import performBackgroundJob from "./perform-job.js"
|
|
|
14
14
|
import { runWithBackgroundJobPayload } from "./execution-context.js"
|
|
15
15
|
import { createGenerationWorkerId } from "./generation-identity.js"
|
|
16
16
|
import BackgroundJobsGenerationHandshakeTimeoutError, { DEFAULT_GENERATION_HANDSHAKE_TIMEOUT_MS, validateGenerationHandshakeTimeoutMs } from "./generation-handshake-timeout-error.js"
|
|
17
|
+
import { POOLED_RUNNER_INFLIGHT_JOB_ID_LIMIT, boundedPooledRunnerInflightJobIds, isPooledChildShutdownReason, isPooledChildShutdownSignal } from "./pooled-runner-shutdown.js"
|
|
17
18
|
|
|
18
19
|
/**
|
|
19
20
|
* Per-forked-child timeout bookkeeping.
|
|
@@ -33,18 +34,26 @@ import BackgroundJobsGenerationHandshakeTimeoutError, { DEFAULT_GENERATION_HANDS
|
|
|
33
34
|
/**
|
|
34
35
|
* @typedef {object} PooledChildState
|
|
35
36
|
* @property {number} createdAtMs - Child creation timestamp.
|
|
37
|
+
* @property {string} [childInstanceId] - Stable identity reported by the child.
|
|
36
38
|
* @property {number} jobsRun - Acknowledged jobs completed by this child.
|
|
37
39
|
* @property {Map<string, PooledJobEntry>} inflight - Jobs currently owned by this child.
|
|
38
40
|
* @property {number} lastDispatchSeq - Round-robin dispatch sequence.
|
|
39
41
|
* @property {boolean} retiring - Whether this child is draining before retirement.
|
|
40
42
|
* @property {boolean} [started] - Whether the child completed its startup handshake.
|
|
41
43
|
* @property {boolean} [settling] - Whether failure handling already owns this child.
|
|
44
|
+
* @property {number} [ipcDisconnectedAtMs] - Parent observation of IPC disconnect.
|
|
45
|
+
* @property {import("./types.js").PooledChildShutdownObservation} [shutdownObservation] - Child observation sent before teardown.
|
|
46
|
+
* @property {import("./types.js").PooledChildShutdownReason} [shutdownReason] - Exact parent-requested shutdown reason.
|
|
47
|
+
* @property {number} [shutdownRequestedAtMs] - Exact parent shutdown-request timestamp.
|
|
48
|
+
* @property {import("node:child_process").ChildProcess["signalCode"]} [shutdownSignal] - Signal selected by the parent request.
|
|
49
|
+
* @property {ReturnType<typeof setTimeout>} [shutdownSignalTimer] - Drained-retirement fallback signal timer.
|
|
42
50
|
* @property {ReturnType<typeof setTimeout> | null} [timeoutSigkillTimer] - Pending timeout SIGKILL timer.
|
|
43
|
-
* @property {import("./types.js").PooledRunnerTerminationReason} [terminationReason] - Expected termination reason.
|
|
44
51
|
* @property {string} [timeoutJobId] - Job whose timeout initiated termination.
|
|
45
52
|
*/
|
|
46
53
|
/** Grace period after SIGTERM before a lingering process runner is SIGKILLed. */
|
|
47
54
|
const FORKED_CHILD_SIGKILL_GRACE_MS = 5000
|
|
55
|
+
/** Time a drained retirement gives the child IPC request to begin teardown before SIGTERM fallback. */
|
|
56
|
+
const POOLED_RUNNER_SHUTDOWN_REQUEST_GRACE_MS = 250
|
|
48
57
|
/**
|
|
49
58
|
* Largest delay Node's `setTimeout` accepts without overflowing to a 1ms delay
|
|
50
59
|
* (a 32-bit signed int of ms, ~24.8 days). A `jobTimeoutMs` above this — or a
|
|
@@ -100,6 +109,29 @@ function isChildAcceptanceMessage(message) {
|
|
|
100
109
|
&& (record.childPid === undefined || Number.isInteger(record.childPid))
|
|
101
110
|
}
|
|
102
111
|
|
|
112
|
+
/**
|
|
113
|
+
* Checks whether an IPC value is the child's bounded pre-teardown observation.
|
|
114
|
+
* @param {ReturnType<typeof JSON.parse>} message - IPC message.
|
|
115
|
+
* @returns {message is import("./types.js").PooledChildShutdownObservation & {type: "shutdown-observation"}} - Whether this is a valid shutdown observation.
|
|
116
|
+
*/
|
|
117
|
+
function isChildShutdownObservationMessage(message) {
|
|
118
|
+
if (!message || typeof message !== "object") return false
|
|
119
|
+
const record = /** @type {{childInstanceId?: ReturnType<typeof JSON.parse>, inflightJobIds?: ReturnType<typeof JSON.parse>, inflightJobIdsTruncatedCount?: ReturnType<typeof JSON.parse>, reason?: ReturnType<typeof JSON.parse>, shutdownObservedAtMs?: ReturnType<typeof JSON.parse>, shutdownRequestedAtMs?: ReturnType<typeof JSON.parse>, signal?: ReturnType<typeof JSON.parse>, type?: ReturnType<typeof JSON.parse>}} */ (message)
|
|
120
|
+
|
|
121
|
+
return record.type === "shutdown-observation"
|
|
122
|
+
&& typeof record.childInstanceId === "string"
|
|
123
|
+
&& Array.isArray(record.inflightJobIds)
|
|
124
|
+
&& record.inflightJobIds.length <= POOLED_RUNNER_INFLIGHT_JOB_ID_LIMIT
|
|
125
|
+
&& record.inflightJobIds.every((jobId) => typeof jobId === "string")
|
|
126
|
+
&& Number.isInteger(record.inflightJobIdsTruncatedCount)
|
|
127
|
+
&& record.inflightJobIdsTruncatedCount >= 0
|
|
128
|
+
&& isPooledChildShutdownReason(record.reason)
|
|
129
|
+
&& typeof record.shutdownObservedAtMs === "number"
|
|
130
|
+
&& Number.isFinite(record.shutdownObservedAtMs)
|
|
131
|
+
&& (record.shutdownRequestedAtMs === null || (typeof record.shutdownRequestedAtMs === "number" && Number.isFinite(record.shutdownRequestedAtMs)))
|
|
132
|
+
&& isPooledChildShutdownSignal(record.signal)
|
|
133
|
+
}
|
|
134
|
+
|
|
103
135
|
/**
|
|
104
136
|
* Normalizes a candidate pooled-runner resource limit.
|
|
105
137
|
* @param {number | undefined} value - Candidate positive number.
|
|
@@ -561,14 +593,14 @@ export default class BackgroundJobsWorker {
|
|
|
561
593
|
|
|
562
594
|
for (const child of this.inflightProcessChildren) {
|
|
563
595
|
const pooledState = this.pooledChildStates.get(child)
|
|
564
|
-
if (pooledState
|
|
565
|
-
|
|
566
|
-
}
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
596
|
+
if (pooledState) {
|
|
597
|
+
this._requestPooledChildShutdown({child, reason: "worker_stop", signal: "SIGTERM"})
|
|
598
|
+
} else {
|
|
599
|
+
try {
|
|
600
|
+
child.kill("SIGTERM")
|
|
601
|
+
} catch {
|
|
602
|
+
// Child already exited; nothing to do.
|
|
603
|
+
}
|
|
572
604
|
}
|
|
573
605
|
}
|
|
574
606
|
|
|
@@ -1185,16 +1217,10 @@ export default class BackgroundJobsWorker {
|
|
|
1185
1217
|
const state = this.pooledChildStates.get(child)
|
|
1186
1218
|
|
|
1187
1219
|
// Already settling/gone, or the job finished in the race with this timer.
|
|
1188
|
-
if (!state || state.settling || state.
|
|
1220
|
+
if (!state || state.settling || state.shutdownReason || !state.inflight.has(jobId)) return
|
|
1189
1221
|
|
|
1190
|
-
state.terminationReason = "job-timeout"
|
|
1191
1222
|
state.timeoutJobId = jobId
|
|
1192
|
-
|
|
1193
|
-
try {
|
|
1194
|
-
child.kill("SIGTERM")
|
|
1195
|
-
} catch {
|
|
1196
|
-
// Child already exited; nothing to do.
|
|
1197
|
-
}
|
|
1223
|
+
this._requestPooledChildShutdown({child, reason: "job_timeout", signal: "SIGTERM"})
|
|
1198
1224
|
|
|
1199
1225
|
state.timeoutSigkillTimer = setTimeout(() => {
|
|
1200
1226
|
try {
|
|
@@ -1234,6 +1260,10 @@ export default class BackgroundJobsWorker {
|
|
|
1234
1260
|
origin: "process-error",
|
|
1235
1261
|
signal: child.signalCode
|
|
1236
1262
|
}))
|
|
1263
|
+
child.once("disconnect", () => {
|
|
1264
|
+
const state = this.pooledChildStates.get(child)
|
|
1265
|
+
if (state) state.ipcDisconnectedAtMs ??= Date.now()
|
|
1266
|
+
})
|
|
1237
1267
|
return child
|
|
1238
1268
|
}
|
|
1239
1269
|
|
|
@@ -1247,10 +1277,24 @@ export default class BackgroundJobsWorker {
|
|
|
1247
1277
|
*/
|
|
1248
1278
|
_handlePooledChildMessage({child, message}) {
|
|
1249
1279
|
if (!message || typeof message !== "object") return
|
|
1250
|
-
const record = /** @type {{type?: ReturnType<typeof JSON.parse>, jobId?: ReturnType<typeof JSON.parse>, acknowledged?: ReturnType<typeof JSON.parse>, rssBytes?: ReturnType<typeof JSON.parse>, error?: ReturnType<typeof JSON.parse>}} */ (message)
|
|
1280
|
+
const record = /** @type {{type?: ReturnType<typeof JSON.parse>, childInstanceId?: ReturnType<typeof JSON.parse>, jobId?: ReturnType<typeof JSON.parse>, acknowledged?: ReturnType<typeof JSON.parse>, rssBytes?: ReturnType<typeof JSON.parse>, error?: ReturnType<typeof JSON.parse>}} */ (message)
|
|
1251
1281
|
const state = this.pooledChildStates.get(child)
|
|
1252
1282
|
if (record.type === "ready") {
|
|
1253
|
-
if (state)
|
|
1283
|
+
if (state) {
|
|
1284
|
+
state.started = true
|
|
1285
|
+
if (typeof record.childInstanceId === "string") state.childInstanceId = record.childInstanceId
|
|
1286
|
+
}
|
|
1287
|
+
return
|
|
1288
|
+
}
|
|
1289
|
+
if (isChildShutdownObservationMessage(message)) {
|
|
1290
|
+
if (state) {
|
|
1291
|
+
state.childInstanceId = message.childInstanceId
|
|
1292
|
+
state.shutdownObservation = message
|
|
1293
|
+
if (state.shutdownSignalTimer) {
|
|
1294
|
+
clearTimeout(state.shutdownSignalTimer)
|
|
1295
|
+
state.shutdownSignalTimer = undefined
|
|
1296
|
+
}
|
|
1297
|
+
}
|
|
1254
1298
|
return
|
|
1255
1299
|
}
|
|
1256
1300
|
if (isChildAcceptanceMessage(message)) {
|
|
@@ -1354,10 +1398,78 @@ export default class BackgroundJobsWorker {
|
|
|
1354
1398
|
* @returns {void}
|
|
1355
1399
|
*/
|
|
1356
1400
|
_retirePooledChild(child) {
|
|
1401
|
+
const state = this.pooledChildStates.get(child)
|
|
1402
|
+
if (!state) throw new Error("Cannot retire pooled child without tracked state")
|
|
1403
|
+
if (state.inflight.size > 0) {
|
|
1404
|
+
throw new Error(`Cannot retire pooled child while ${state.inflight.size} ${state.inflight.size === 1 ? "job remains" : "jobs remain"} in flight`)
|
|
1405
|
+
}
|
|
1406
|
+
|
|
1357
1407
|
this.pooledChildren.delete(child)
|
|
1358
|
-
|
|
1359
|
-
this.
|
|
1360
|
-
|
|
1408
|
+
state.retiring = true
|
|
1409
|
+
this._requestPooledChildShutdown({child, reason: "parent_retire_drained", signal: "SIGTERM", waitForObservation: true})
|
|
1410
|
+
}
|
|
1411
|
+
|
|
1412
|
+
/**
|
|
1413
|
+
* Records an exact parent request before IPC or signal delivery. Drained
|
|
1414
|
+
* retirement gets a brief IPC-first grace so its zero-job observation is
|
|
1415
|
+
* deterministic; timeout/worker-stop paths signal immediately.
|
|
1416
|
+
* @param {object} args - Shutdown request.
|
|
1417
|
+
* @param {import("node:child_process").ChildProcess} args.child - Pooled child.
|
|
1418
|
+
* @param {import("./types.js").PooledChildShutdownReason} args.reason - Exact parent reason.
|
|
1419
|
+
* @param {keyof typeof import("node:os").constants.signals} args.signal - Signal to deliver.
|
|
1420
|
+
* @param {boolean} [args.waitForObservation] - Whether IPC observation may precede signal fallback.
|
|
1421
|
+
* @returns {void}
|
|
1422
|
+
*/
|
|
1423
|
+
_requestPooledChildShutdown({child, reason, signal, waitForObservation = false}) {
|
|
1424
|
+
const state = this.pooledChildStates.get(child)
|
|
1425
|
+
if (!state || state.settling || state.shutdownReason) return
|
|
1426
|
+
|
|
1427
|
+
const shutdownRequestedAtMs = Date.now()
|
|
1428
|
+
state.shutdownReason = reason
|
|
1429
|
+
state.shutdownRequestedAtMs = shutdownRequestedAtMs
|
|
1430
|
+
state.shutdownSignal = signal
|
|
1431
|
+
|
|
1432
|
+
let requestSent = false
|
|
1433
|
+
if (child.connected) {
|
|
1434
|
+
try {
|
|
1435
|
+
child.send({type: "shutdown-request", reason, shutdownRequestedAtMs, signal}, (error) => {
|
|
1436
|
+
if (!error || !waitForObservation || state.settling) return
|
|
1437
|
+
|
|
1438
|
+
if (state.shutdownSignalTimer) {
|
|
1439
|
+
clearTimeout(state.shutdownSignalTimer)
|
|
1440
|
+
state.shutdownSignalTimer = undefined
|
|
1441
|
+
}
|
|
1442
|
+
this._signalPooledChild({child, signal})
|
|
1443
|
+
})
|
|
1444
|
+
requestSent = true
|
|
1445
|
+
} catch {
|
|
1446
|
+
// The signal below remains the bounded shutdown mechanism.
|
|
1447
|
+
}
|
|
1448
|
+
}
|
|
1449
|
+
|
|
1450
|
+
if (waitForObservation && requestSent) {
|
|
1451
|
+
state.shutdownSignalTimer = setTimeout(() => {
|
|
1452
|
+
state.shutdownSignalTimer = undefined
|
|
1453
|
+
this._signalPooledChild({child, signal})
|
|
1454
|
+
}, POOLED_RUNNER_SHUTDOWN_REQUEST_GRACE_MS)
|
|
1455
|
+
state.shutdownSignalTimer.unref()
|
|
1456
|
+
return
|
|
1457
|
+
}
|
|
1458
|
+
|
|
1459
|
+
this._signalPooledChild({child, signal})
|
|
1460
|
+
}
|
|
1461
|
+
|
|
1462
|
+
/**
|
|
1463
|
+
* Delivers one parent-owned process signal without changing recorded provenance.
|
|
1464
|
+
* @param {{child: import("node:child_process").ChildProcess, signal: keyof typeof import("node:os").constants.signals}} args - Signal request.
|
|
1465
|
+
* @returns {void}
|
|
1466
|
+
*/
|
|
1467
|
+
_signalPooledChild({child, signal}) {
|
|
1468
|
+
try {
|
|
1469
|
+
child.kill(signal)
|
|
1470
|
+
} catch {
|
|
1471
|
+
// Child already exited; its exit/error handler owns state settlement.
|
|
1472
|
+
}
|
|
1361
1473
|
}
|
|
1362
1474
|
|
|
1363
1475
|
/**
|
|
@@ -1384,6 +1496,7 @@ export default class BackgroundJobsWorker {
|
|
|
1384
1496
|
// Cancel this child's pending timers before its in-flight set is reported —
|
|
1385
1497
|
// the SIGKILL grace from a timeout kill, and every armed per-job backstop.
|
|
1386
1498
|
if (state.timeoutSigkillTimer) clearTimeout(state.timeoutSigkillTimer)
|
|
1499
|
+
if (state.shutdownSignalTimer) clearTimeout(state.shutdownSignalTimer)
|
|
1387
1500
|
for (const inflightEntry of state.inflight.values()) {
|
|
1388
1501
|
if (inflightEntry.timeoutTimer) clearTimeout(inflightEntry.timeoutTimer)
|
|
1389
1502
|
}
|
|
@@ -1415,19 +1528,24 @@ export default class BackgroundJobsWorker {
|
|
|
1415
1528
|
// observe a replacement slot before the failed jobs' reports are in flight.
|
|
1416
1529
|
// The report promises remain tracked below; a slow retry must not hold the
|
|
1417
1530
|
// newly freed runner capacity hostage.
|
|
1418
|
-
|
|
1419
|
-
|
|
1420
|
-
|
|
1421
|
-
|
|
1422
|
-
|
|
1423
|
-
|
|
1424
|
-
|
|
1531
|
+
// A drained retirement already advertised its replacement capacity when
|
|
1532
|
+
// the final job completed. Re-advertising here can restore a credit main
|
|
1533
|
+
// consumed before its handoff reached the replacement child.
|
|
1534
|
+
if (state?.shutdownReason !== "parent_retire_drained") {
|
|
1535
|
+
if (state && state.started !== false) {
|
|
1536
|
+
this._sendReadyIfRunning()
|
|
1537
|
+
} else if (state) {
|
|
1538
|
+
for (const entry of entries) {
|
|
1539
|
+
if (entry.pooledJob) this._pooledStartupFailureJobs.add(entry.pooledJob)
|
|
1540
|
+
const queueTracker = this.pooledJobQueueTrackers.get(entry.payload.id)
|
|
1541
|
+
if (queueTracker) this._pooledStartupFailureJobs.add(queueTracker)
|
|
1542
|
+
}
|
|
1543
|
+
// A previous ready message may still have unconsumed pooled credits at the
|
|
1544
|
+
// main. Revoke them authoritatively without suppressing valid inline or
|
|
1545
|
+
// process-runner readiness; otherwise queued jobs can trigger a startup
|
|
1546
|
+
// crash loop using the stale credits.
|
|
1547
|
+
this._sendReadyIfRunning({revokePooledAdmission: true})
|
|
1425
1548
|
}
|
|
1426
|
-
// A previous ready message may still have unconsumed pooled credits at the
|
|
1427
|
-
// main. Revoke them authoritatively without suppressing valid inline or
|
|
1428
|
-
// process-runner readiness; otherwise queued jobs can trigger a startup
|
|
1429
|
-
// crash loop using the stale credits.
|
|
1430
|
-
this._sendReadyIfRunning({revokePooledAdmission: true})
|
|
1431
1549
|
}
|
|
1432
1550
|
|
|
1433
1551
|
await Promise.allSettled(failureReports)
|
|
@@ -1444,9 +1562,31 @@ export default class BackgroundJobsWorker {
|
|
|
1444
1562
|
* @returns {import("./types.js").PooledRunnerFailure} - Shared failure provenance.
|
|
1445
1563
|
*/
|
|
1446
1564
|
_pooledRunnerFailure({child, exitCode, origin, signal, state}) {
|
|
1447
|
-
const
|
|
1565
|
+
const observation = state.shutdownObservation
|
|
1566
|
+
let shutdownReason = state.shutdownReason ?? observation?.reason
|
|
1567
|
+
if (!shutdownReason) {
|
|
1568
|
+
if (origin === "process-error" || origin === "ipc-send") {
|
|
1569
|
+
shutdownReason = "process_error"
|
|
1570
|
+
} else if (signal === "SIGTERM") {
|
|
1571
|
+
shutdownReason = "signal_sigterm"
|
|
1572
|
+
} else if (signal === "SIGINT") {
|
|
1573
|
+
shutdownReason = "signal_sigint"
|
|
1574
|
+
} else if (signal === "SIGKILL") {
|
|
1575
|
+
shutdownReason = "signal_sigkill"
|
|
1576
|
+
} else if (signal) {
|
|
1577
|
+
shutdownReason = "signal_other"
|
|
1578
|
+
} else if (state.ipcDisconnectedAtMs && exitCode === 0) {
|
|
1579
|
+
shutdownReason = "ipc_disconnect"
|
|
1580
|
+
} else {
|
|
1581
|
+
shutdownReason = "unexpected_exit"
|
|
1582
|
+
}
|
|
1583
|
+
}
|
|
1584
|
+
const terminationReason = shutdownReason === "job_timeout"
|
|
1585
|
+
? "job-timeout"
|
|
1586
|
+
: shutdownReason === "worker_stop" ? "worker-shutdown-timeout" : "unexpected"
|
|
1448
1587
|
const workerLifecycle = this.shouldStop ? "stopping" : this.isRetiring ? "retiring" : "running"
|
|
1449
1588
|
const runnerLifecycle = state.started === false ? "starting" : state.retiring ? "retiring" : "running"
|
|
1589
|
+
const boundedInflight = boundedPooledRunnerInflightJobIds(state.inflight.keys())
|
|
1450
1590
|
const activeJobs = [...state.inflight.values()]
|
|
1451
1591
|
.map((entry) => ({
|
|
1452
1592
|
handoffId: entry.payload.handoffId ?? null,
|
|
@@ -1459,9 +1599,11 @@ export default class BackgroundJobsWorker {
|
|
|
1459
1599
|
|
|
1460
1600
|
return Object.freeze({
|
|
1461
1601
|
activeJobs,
|
|
1602
|
+
childInstanceId: state.childInstanceId ?? observation?.childInstanceId ?? null,
|
|
1462
1603
|
exitCode,
|
|
1463
1604
|
generationId: this.generationId ?? null,
|
|
1464
|
-
|
|
1605
|
+
...boundedInflight,
|
|
1606
|
+
oomKilled: signal === "SIGKILL" && shutdownReason === "signal_sigkill" ? null : false,
|
|
1465
1607
|
origin,
|
|
1466
1608
|
runnerAgeMs: Math.max(0, Date.now() - state.createdAtMs),
|
|
1467
1609
|
runnerCreatedAtMs: state.createdAtMs,
|
|
@@ -1470,6 +1612,10 @@ export default class BackgroundJobsWorker {
|
|
|
1470
1612
|
runnerLifecycle,
|
|
1471
1613
|
runnerPid: child.pid ?? null,
|
|
1472
1614
|
signal,
|
|
1615
|
+
shutdownObservedAtMs: observation?.shutdownObservedAtMs ?? state.ipcDisconnectedAtMs ?? null,
|
|
1616
|
+
shutdownRequestedAtMs: state.shutdownRequestedAtMs ?? observation?.shutdownRequestedAtMs ?? null,
|
|
1617
|
+
shutdownReason,
|
|
1618
|
+
shutdownSignal: state.shutdownSignal ?? observation?.signal ?? signal,
|
|
1473
1619
|
terminationReason,
|
|
1474
1620
|
timeoutJobId: state.timeoutJobId ?? null,
|
|
1475
1621
|
workerId: this.workerId,
|