velocious 1.0.722 → 1.0.724
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/build/background-jobs/forked-runner-child.js +44 -16
- package/build/background-jobs/worker.js +195 -39
- package/build/database/pool/single-multi-use.js +3 -2
- package/build/src/background-jobs/forked-runner-child.d.ts +14 -1
- package/build/src/background-jobs/forked-runner-child.d.ts.map +1 -1
- package/build/src/background-jobs/forked-runner-child.js +41 -16
- package/build/src/background-jobs/worker.d.ts +130 -28
- package/build/src/background-jobs/worker.d.ts.map +1 -1
- package/build/src/background-jobs/worker.js +185 -39
- package/build/src/database/pool/single-multi-use.d.ts.map +1 -1
- package/build/src/database/pool/single-multi-use.js +4 -3
- package/build/src/sync/sync-applied-sequence-store.d.ts +27 -8
- package/build/src/sync/sync-applied-sequence-store.d.ts.map +1 -1
- package/build/src/sync/sync-applied-sequence-store.js +43 -18
- package/build/src/sync/sync-client.js +3 -3
- package/build/sync/sync-applied-sequence-store.js +46 -17
- package/build/sync/sync-client.js +2 -2
- package/package.json +1 -1
- package/src/background-jobs/forked-runner-child.js +44 -16
- package/src/background-jobs/worker.js +195 -39
- package/src/database/pool/single-multi-use.js +3 -2
- package/src/sync/sync-applied-sequence-store.js +46 -17
- package/src/sync/sync-client.js +2 -2
|
@@ -69,9 +69,10 @@ export default class SyncAppliedSequenceStore {
|
|
|
69
69
|
|
|
70
70
|
/**
|
|
71
71
|
* Ensures the backing table exists.
|
|
72
|
+
* @param {{operation?: import("../database/operation.js").default | null}} [args] - Optional caller-owned operation.
|
|
72
73
|
* @returns {Promise<void>} Resolves when ready.
|
|
73
74
|
*/
|
|
74
|
-
async ensureReady() {
|
|
75
|
+
async ensureReady({operation} = {}) {
|
|
75
76
|
if (this._isReady) return
|
|
76
77
|
|
|
77
78
|
if (this._usesMemoryStorage()) {
|
|
@@ -79,43 +80,43 @@ export default class SyncAppliedSequenceStore {
|
|
|
79
80
|
return
|
|
80
81
|
}
|
|
81
82
|
|
|
82
|
-
await this._withDb(async (db) => await this._ensureReadyWithDb(db))
|
|
83
|
+
await this._withDb(async (db) => await this._ensureReadyWithDb(db), operation)
|
|
83
84
|
}
|
|
84
85
|
|
|
85
86
|
/**
|
|
86
87
|
* Returns the highest server sequence already applied to a record.
|
|
87
|
-
* @param {{resourceType: string, resourceId: string}} identity - Resource identity.
|
|
88
|
+
* @param {{resourceType: string, resourceId: string, operation?: import("../database/operation.js").default | null}} identity - Resource identity and optional caller-owned operation.
|
|
88
89
|
* @returns {Promise<number | null>} Applied sequence, or null when none is recorded.
|
|
89
90
|
*/
|
|
90
|
-
async lastAppliedSequence(
|
|
91
|
-
await this.ensureReady()
|
|
91
|
+
async lastAppliedSequence({resourceId, resourceType, operation}) {
|
|
92
|
+
await this.ensureReady({operation})
|
|
92
93
|
|
|
93
94
|
if (this._usesMemoryStorage()) {
|
|
94
|
-
return this._memorySequences.get(memoryKey(
|
|
95
|
+
return this._memorySequences.get(memoryKey({resourceId, resourceType})) ?? null
|
|
95
96
|
}
|
|
96
97
|
|
|
97
98
|
return await this._withDb(async (db) => {
|
|
98
|
-
const row = await this._rowByResource(db,
|
|
99
|
+
const row = await this._rowByResource(db, {resourceId, resourceType})
|
|
99
100
|
|
|
100
101
|
return row ? row.serverSequence : null
|
|
101
|
-
})
|
|
102
|
+
}, operation)
|
|
102
103
|
}
|
|
103
104
|
|
|
104
105
|
/**
|
|
105
106
|
* Records a successfully applied server sequence for a record, keeping only
|
|
106
107
|
* the highest value. Writes for the same record are serialized in-process so
|
|
107
108
|
* concurrent pull/realtime applies cannot move the watermark backwards.
|
|
108
|
-
* @param {{resourceType: string, resourceId: string, serverSequence: number}} args - Applied change identity and
|
|
109
|
+
* @param {{resourceType: string, resourceId: string, serverSequence: number, operation?: import("../database/operation.js").default | null}} args - Applied change identity, sequence and optional caller-owned operation.
|
|
109
110
|
* @returns {Promise<void>} Resolves when recorded.
|
|
110
111
|
*/
|
|
111
|
-
async recordAppliedSequence({resourceId, resourceType, serverSequence}) {
|
|
112
|
-
await this.ensureReady()
|
|
112
|
+
async recordAppliedSequence({resourceId, resourceType, serverSequence, operation}) {
|
|
113
|
+
await this.ensureReady({operation})
|
|
113
114
|
|
|
114
115
|
const identity = {resourceId, resourceType}
|
|
115
116
|
const key = memoryKey(identity)
|
|
116
117
|
const previousWrite = this._writeQueues.get(key) || Promise.resolve()
|
|
117
118
|
const write = previousWrite
|
|
118
|
-
.then(async () => await this._recordSequential({identity, key, serverSequence}))
|
|
119
|
+
.then(async () => await this._recordSequential({identity, key, operation, serverSequence}))
|
|
119
120
|
.finally(() => {
|
|
120
121
|
if (this._writeQueues.get(key) === write) this._writeQueues.delete(key)
|
|
121
122
|
})
|
|
@@ -126,11 +127,11 @@ export default class SyncAppliedSequenceStore {
|
|
|
126
127
|
|
|
127
128
|
/**
|
|
128
129
|
* Applies one serialized watermark write.
|
|
129
|
-
* @param {{identity: {resourceType: string, resourceId: string}, key: string, serverSequence: number}} args - Write args.
|
|
130
|
+
* @param {{identity: {resourceType: string, resourceId: string}, key: string, serverSequence: number, operation?: import("../database/operation.js").default | null}} args - Write args.
|
|
130
131
|
* @returns {Promise<void>} Resolves when written.
|
|
131
132
|
*/
|
|
132
|
-
async _recordSequential({identity, key, serverSequence}) {
|
|
133
|
-
const currentSequence = await this.lastAppliedSequence(identity)
|
|
133
|
+
async _recordSequential({identity, key, serverSequence, operation}) {
|
|
134
|
+
const currentSequence = await this.lastAppliedSequence({resourceType: identity.resourceType, resourceId: identity.resourceId, operation})
|
|
134
135
|
|
|
135
136
|
if (currentSequence !== null && currentSequence >= serverSequence) return
|
|
136
137
|
|
|
@@ -153,7 +154,7 @@ export default class SyncAppliedSequenceStore {
|
|
|
153
154
|
tableName: TABLE_NAME,
|
|
154
155
|
updateColumns: ["server_sequence", "updated_at"]
|
|
155
156
|
})
|
|
156
|
-
})
|
|
157
|
+
}, operation)
|
|
157
158
|
}
|
|
158
159
|
|
|
159
160
|
/**
|
|
@@ -227,11 +228,23 @@ export default class SyncAppliedSequenceStore {
|
|
|
227
228
|
|
|
228
229
|
/**
|
|
229
230
|
* Runs a callback with a database connection.
|
|
231
|
+
*
|
|
232
|
+
* A caller that already owns an operation on this store's physical database
|
|
233
|
+
* must hand it in: taking a second checkout from the same physical database
|
|
234
|
+
* would queue behind the operation the caller is running inside and wait
|
|
235
|
+
* behind itself.
|
|
230
236
|
* @template Result
|
|
231
237
|
* @param {(db: import("../database/drivers/base.js").default) => Promise<Result>} callback - Database callback.
|
|
238
|
+
* @param {import("../database/operation.js").default | null} [operation] - Caller-owned operation to run on.
|
|
232
239
|
* @returns {Promise<Result>} Callback result.
|
|
233
240
|
*/
|
|
234
|
-
async _withDb(callback) {
|
|
241
|
+
async _withDb(callback, operation) {
|
|
242
|
+
if (operation) {
|
|
243
|
+
this.assertOperationOwnership(operation)
|
|
244
|
+
|
|
245
|
+
return await callback(operation.connection())
|
|
246
|
+
}
|
|
247
|
+
|
|
235
248
|
if (this.tenantHandle) {
|
|
236
249
|
return await this.tenantHandle.databaseOperation({databaseIdentifier: this.databaseIdentifier, name: "Tenant sync applied-sequence store"}, async (operation) => {
|
|
237
250
|
return await callback(operation.connection())
|
|
@@ -247,6 +260,22 @@ export default class SyncAppliedSequenceStore {
|
|
|
247
260
|
})
|
|
248
261
|
}
|
|
249
262
|
|
|
263
|
+
/**
|
|
264
|
+
* Fails closed when a caller hands in an operation that does not belong to this store's database.
|
|
265
|
+
* @param {import("../database/operation.js").default} operation - Caller-owned operation.
|
|
266
|
+
* @returns {void} Nothing when the operation matches the store.
|
|
267
|
+
* @throws {Error} When the operation belongs to another database or another physical tenant database.
|
|
268
|
+
*/
|
|
269
|
+
assertOperationOwnership(operation) {
|
|
270
|
+
if (operation.databaseIdentifier() !== this.databaseIdentifier) {
|
|
271
|
+
throw new Error(`Sync applied-sequence store operation belongs to another database: ${operation.databaseIdentifier()}`)
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
if (this.tenantHandle && operation.databaseIdentity() !== this.storeIdentity) {
|
|
275
|
+
throw new Error(`Sync applied-sequence store operation belongs to another physical database: ${operation.databaseIdentity()}`)
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
|
|
250
279
|
/**
|
|
251
280
|
* Ensures the applied-sequences table exists.
|
|
252
281
|
* @param {import("../database/drivers/base.js").default} db - Database connection.
|
|
@@ -861,7 +861,7 @@ export default class SyncClient {
|
|
|
861
861
|
const appliedSequences = this.appliedSequenceStore()
|
|
862
862
|
|
|
863
863
|
if (serverSequence !== null) {
|
|
864
|
-
const lastAppliedSequence = await appliedSequences.lastAppliedSequence({resourceId, resourceType})
|
|
864
|
+
const lastAppliedSequence = await appliedSequences.lastAppliedSequence({resourceId, resourceType, operation})
|
|
865
865
|
|
|
866
866
|
if (lastAppliedSequence !== null && serverSequence <= lastAppliedSequence) {
|
|
867
867
|
return {changed: false, resourceType}
|
|
@@ -888,7 +888,7 @@ export default class SyncClient {
|
|
|
888
888
|
const result = await applier(sync)
|
|
889
889
|
|
|
890
890
|
if (serverSequence !== null) {
|
|
891
|
-
await appliedSequences.recordAppliedSequence({resourceId, resourceType, serverSequence})
|
|
891
|
+
await appliedSequences.recordAppliedSequence({resourceId, resourceType, serverSequence, operation})
|
|
892
892
|
}
|
|
893
893
|
|
|
894
894
|
return result
|
package/package.json
CHANGED
|
@@ -1,9 +1,16 @@
|
|
|
1
1
|
// @ts-check
|
|
2
2
|
|
|
3
|
-
import runJobPayload from "./job-runner.js"
|
|
3
|
+
import runJobPayload, { BackgroundJobPerformedFailure } from "./job-runner.js"
|
|
4
4
|
import { closeRunnerConnections, currentConfigurationOrNull } from "./runner-graceful-shutdown.js"
|
|
5
5
|
import setRunnerProcessTitle from "./runner-process-title.js"
|
|
6
6
|
|
|
7
|
+
/**
|
|
8
|
+
* @typedef {object} ForkedJobOutcome
|
|
9
|
+
* @property {number} exitCode - Exit code the runner exits with once the outcome is reported.
|
|
10
|
+
* @property {boolean} acknowledged - Whether the job's terminal report was recorded.
|
|
11
|
+
* @property {string} [error] - Why the terminal report could not be recorded.
|
|
12
|
+
*/
|
|
13
|
+
|
|
7
14
|
// Name the process so `ps`/`top`/`htop` can identify forked job runners at a
|
|
8
15
|
// glance instead of a wall of generic "node" entries. Updated to the specific
|
|
9
16
|
// job name once one arrives (see runJobMessage), so operators can see exactly
|
|
@@ -47,30 +54,39 @@ function isJobMessage(message) {
|
|
|
47
54
|
|
|
48
55
|
/**
|
|
49
56
|
* Runs finish.
|
|
50
|
-
*
|
|
57
|
+
*
|
|
58
|
+
* The report tells the parent whether this runner's job has its terminal result.
|
|
59
|
+
* The parent frees its in-flight entry on an acknowledged outcome instead of
|
|
60
|
+
* waiting for this process to exit: a runner that holds no job has no reason to
|
|
61
|
+
* exit, and a parent that waits for one never finishes its shutdown drain.
|
|
62
|
+
* @param {ForkedJobOutcome & {jobId?: string}} args - Outcome report.
|
|
51
63
|
* @returns {Promise<void>}
|
|
52
64
|
*/
|
|
53
|
-
async function finish(
|
|
65
|
+
async function finish(args) {
|
|
66
|
+
const {acknowledged, error, exitCode, jobId} = args
|
|
67
|
+
|
|
54
68
|
if (process.send) {
|
|
55
|
-
|
|
69
|
+
/** @type {{acknowledged: boolean, error?: string, jobId?: string, type: "job-reported"}} */
|
|
70
|
+
const report = {type: "job-reported", acknowledged}
|
|
71
|
+
if (typeof jobId === "string") report.jobId = jobId
|
|
72
|
+
if (typeof error === "string") report.error = error
|
|
73
|
+
|
|
74
|
+
await new Promise((resolve) => process.send?.(report, () => resolve(undefined)))
|
|
56
75
|
}
|
|
76
|
+
|
|
57
77
|
await shutdownRunner(exitCode)
|
|
58
78
|
}
|
|
59
79
|
|
|
60
80
|
/**
|
|
61
81
|
* Runs run job message.
|
|
62
|
-
* @param {
|
|
82
|
+
* @param {import("./types.js").BackgroundJobPayload} payload - Job payload.
|
|
63
83
|
* @returns {Promise<void>} - Resolves after the payload has run.
|
|
64
84
|
*/
|
|
65
|
-
async function runJobMessage(
|
|
66
|
-
if (!isJobMessage(message)) {
|
|
67
|
-
throw new Error("Forked background job runner received invalid payload")
|
|
68
|
-
}
|
|
69
|
-
|
|
85
|
+
async function runJobMessage(payload) {
|
|
70
86
|
// The per-job process title (and its restore) is set inside runJobPayload,
|
|
71
87
|
// which reads the job class's `static processTitle`. This process boots with
|
|
72
88
|
// the base "velocious background-jobs-runner" title set at module load above.
|
|
73
|
-
await runJobPayload(
|
|
89
|
+
await runJobPayload(payload, {
|
|
74
90
|
closeConnections: false,
|
|
75
91
|
processType: "background-jobs-forked-runner"
|
|
76
92
|
})
|
|
@@ -82,17 +98,29 @@ async function runJobMessage(message) {
|
|
|
82
98
|
* @returns {Promise<void>} - Resolves after completion is reported.
|
|
83
99
|
*/
|
|
84
100
|
async function handleJobMessage(message) {
|
|
85
|
-
|
|
101
|
+
if (!isJobMessage(message)) {
|
|
102
|
+
await finish({acknowledged: false, error: "Forked background job runner received invalid payload", exitCode: 1})
|
|
103
|
+
|
|
104
|
+
return
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/** @type {ForkedJobOutcome} */
|
|
108
|
+
let outcome
|
|
86
109
|
|
|
87
110
|
try {
|
|
88
|
-
await runJobMessage(message)
|
|
89
|
-
|
|
111
|
+
await runJobMessage(message.payload)
|
|
112
|
+
outcome = {acknowledged: true, exitCode: 0}
|
|
90
113
|
} catch (error) {
|
|
91
114
|
console.error("Forked background job runner failed:", error)
|
|
92
|
-
|
|
115
|
+
// A perform error whose terminal report was acknowledged is still an outcome:
|
|
116
|
+
// the job has its result. Only a failure that left the job with no
|
|
117
|
+
// acknowledged terminal report has to be reclaimed by the parent.
|
|
118
|
+
outcome = error instanceof BackgroundJobPerformedFailure
|
|
119
|
+
? {acknowledged: true, exitCode: 1}
|
|
120
|
+
: {acknowledged: false, error: error instanceof Error ? error.message : String(error), exitCode: 1}
|
|
93
121
|
}
|
|
94
122
|
|
|
95
|
-
await finish(
|
|
123
|
+
await finish({...outcome, jobId: message.payload.id})
|
|
96
124
|
}
|
|
97
125
|
|
|
98
126
|
for (const signal of ["SIGTERM", "SIGINT"]) {
|
|
@@ -17,12 +17,24 @@ import BackgroundJobsGenerationHandshakeTimeoutError, { DEFAULT_GENERATION_HANDS
|
|
|
17
17
|
import { POOLED_RUNNER_INFLIGHT_JOB_ID_LIMIT, boundedPooledRunnerInflightJobIds, isPooledChildShutdownReason, isPooledChildShutdownSignal } from "./pooled-runner-shutdown.js"
|
|
18
18
|
|
|
19
19
|
/**
|
|
20
|
-
* Per-forked-child
|
|
21
|
-
*
|
|
20
|
+
* Per-forked-child bookkeeping, shared by the wall-clock backstop, the runner's
|
|
21
|
+
* own job-outcome report, and the exit/error handlers.
|
|
22
|
+
* @typedef {object} ForkedJobState
|
|
22
23
|
* @property {boolean} timedOut - Whether the timeout fired and the child was terminated.
|
|
23
24
|
* @property {number | null} timeoutMs - The armed timeout in ms, or null when disabled.
|
|
24
25
|
* @property {ReturnType<typeof setTimeout> | null} timer - The pending timeout timer, cleared on exit.
|
|
25
26
|
* @property {ReturnType<typeof setTimeout> | null} sigkillTimer - The pending SIGKILL grace timer, cleared on exit.
|
|
27
|
+
* @property {boolean} outcomeReported - Whether the runner reported its job outcome over IPC.
|
|
28
|
+
* @property {boolean} outcomeAcknowledged - Whether that report confirmed the outcome was recorded.
|
|
29
|
+
* @property {boolean} settled - Whether the worker's in-flight entry for this job has been freed.
|
|
30
|
+
* @property {(value: void) => void} resolve - Resolver that frees the in-flight entry; a placeholder until the promise that owns the entry is created.
|
|
31
|
+
*/
|
|
32
|
+
/**
|
|
33
|
+
* One live runner child, kept so a shutdown that runs out of drain time can name
|
|
34
|
+
* the exact process it is still waiting on instead of a bare count.
|
|
35
|
+
* @typedef {object} ProcessJobHolder
|
|
36
|
+
* @property {string} jobId - Job the child was handed.
|
|
37
|
+
* @property {number} startedAtMs - When the worker handed it that job.
|
|
26
38
|
*/
|
|
27
39
|
/**
|
|
28
40
|
* @typedef {object} PooledJobEntry
|
|
@@ -123,6 +135,24 @@ function isChildAcceptanceMessage(message) {
|
|
|
123
135
|
&& (record.runnerHostname === undefined || typeof record.runnerHostname === "string")
|
|
124
136
|
}
|
|
125
137
|
|
|
138
|
+
/**
|
|
139
|
+
* Checks whether an IPC value is a forked runner's job-outcome report. The forked
|
|
140
|
+
* child sends exactly one of these, once its own terminal report has settled, and
|
|
141
|
+
* says whether that report was acknowledged — so the worker can tell a runner that
|
|
142
|
+
* has finished its job from one that is still running it.
|
|
143
|
+
* @param {ReturnType<typeof JSON.parse>} message - IPC message.
|
|
144
|
+
* @returns {message is {acknowledged: boolean, error?: string, jobId?: string, type: "job-reported"}} - Whether this is a job-outcome report.
|
|
145
|
+
*/
|
|
146
|
+
function isForkedChildJobReport(message) {
|
|
147
|
+
if (!message || typeof message !== "object") return false
|
|
148
|
+
const record = /** @type {{acknowledged?: ReturnType<typeof JSON.parse>, error?: ReturnType<typeof JSON.parse>, jobId?: ReturnType<typeof JSON.parse>, type?: ReturnType<typeof JSON.parse>}} */ (message)
|
|
149
|
+
|
|
150
|
+
return record.type === "job-reported"
|
|
151
|
+
&& typeof record.acknowledged === "boolean"
|
|
152
|
+
&& (record.jobId === undefined || typeof record.jobId === "string")
|
|
153
|
+
&& (record.error === undefined || typeof record.error === "string")
|
|
154
|
+
}
|
|
155
|
+
|
|
126
156
|
/**
|
|
127
157
|
* Checks whether an IPC value is the child's bounded pre-teardown observation.
|
|
128
158
|
* @param {ReturnType<typeof JSON.parse>} message - IPC message.
|
|
@@ -366,6 +396,13 @@ export default class BackgroundJobsWorker {
|
|
|
366
396
|
* @type {Set<import("node:child_process").ChildProcess>}
|
|
367
397
|
*/
|
|
368
398
|
this.inflightProcessChildren = new Set()
|
|
399
|
+
/**
|
|
400
|
+
* Which job each live runner child was handed. A shutdown that runs out of
|
|
401
|
+
* drain time names the exact process it is still waiting on (pid, job, how
|
|
402
|
+
* long it has been held) from this map instead of reporting only a count.
|
|
403
|
+
* @type {Map<import("node:child_process").ChildProcess, ProcessJobHolder>}
|
|
404
|
+
*/
|
|
405
|
+
this.processChildJobs = new Map()
|
|
369
406
|
/** @type {Set<Promise<void>>} */
|
|
370
407
|
this.inflightPooledJobs = new Set()
|
|
371
408
|
/** @type {Map<string, Array<import("./types.js").BackgroundJobPayload & {id: string}>>} */
|
|
@@ -551,9 +588,9 @@ export default class BackgroundJobsWorker {
|
|
|
551
588
|
}
|
|
552
589
|
}
|
|
553
590
|
|
|
554
|
-
await this._drainInflight(this.inflightInlineJobs, timeoutMs)
|
|
555
|
-
await this._drainInflight(this.inflightPooledJobs, timeoutMs)
|
|
556
|
-
await this._drainInflight(this.inflightProcessJobs, timeoutMs)
|
|
591
|
+
await this._drainInflight(this.inflightInlineJobs, timeoutMs, {phase: "inline-jobs"})
|
|
592
|
+
await this._drainInflight(this.inflightPooledJobs, timeoutMs, {phase: "pooled-jobs"})
|
|
593
|
+
await this._drainInflight(this.inflightProcessJobs, timeoutMs, {phase: "process-jobs", describeHolders: () => this._describeProcessJobHolders()})
|
|
557
594
|
await this._terminateProcessChildren()
|
|
558
595
|
// Give in-flight result reports (now decoupled from job slots) a bounded
|
|
559
596
|
// chance to land before the socket closes.
|
|
@@ -645,12 +682,17 @@ export default class BackgroundJobsWorker {
|
|
|
645
682
|
|
|
646
683
|
/**
|
|
647
684
|
* Waits for a set of in-flight job promises to settle, optionally bounded by
|
|
648
|
-
* `timeoutMs`.
|
|
685
|
+
* `timeoutMs`. When a bounded window runs out with entries still held, the drain
|
|
686
|
+
* names what it is waiting on instead of returning silently: an unexplained wait
|
|
687
|
+
* is what turned one wedged worker into a 28-minute investigation.
|
|
649
688
|
* @param {Set<Promise<void>>} inflight - In-flight job promises.
|
|
650
689
|
* @param {number} [timeoutMs] - Max wait in ms; unbounded when omitted.
|
|
690
|
+
* @param {object} [options] - Drain options.
|
|
691
|
+
* @param {string} [options.phase] - Drain phase named in a timeout report.
|
|
692
|
+
* @param {(() => ProcessJobHolder[])} [options.describeHolders] - Names the live processes this phase is waiting on.
|
|
651
693
|
* @returns {Promise<void>} - Resolves when settled or the timeout elapses.
|
|
652
694
|
*/
|
|
653
|
-
async _drainInflight(inflight, timeoutMs) {
|
|
695
|
+
async _drainInflight(inflight, timeoutMs, {phase = "inflight", describeHolders} = {}) {
|
|
654
696
|
if (inflight.size === 0) return
|
|
655
697
|
|
|
656
698
|
const drain = Promise.allSettled([...inflight])
|
|
@@ -661,11 +703,51 @@ export default class BackgroundJobsWorker {
|
|
|
661
703
|
|
|
662
704
|
await Promise.race([drain, timeout])
|
|
663
705
|
clearTimeout(timer)
|
|
706
|
+
|
|
707
|
+
if (inflight.size > 0) {
|
|
708
|
+
this._logDrainTimeout({phase, timeoutMs, pendingCount: inflight.size, holders: describeHolders ? describeHolders() : []})
|
|
709
|
+
}
|
|
664
710
|
} else {
|
|
665
711
|
await drain
|
|
666
712
|
}
|
|
667
713
|
}
|
|
668
714
|
|
|
715
|
+
/**
|
|
716
|
+
* Names every live runner child the worker still holds a job for, so a drain
|
|
717
|
+
* that ran out of time says which process it is waiting on, for which job, and
|
|
718
|
+
* for how long.
|
|
719
|
+
* @returns {ProcessJobHolder[]} - Held job descriptors, oldest first.
|
|
720
|
+
*/
|
|
721
|
+
_describeProcessJobHolders() {
|
|
722
|
+
return [...this.processChildJobs.values()].sort((left, right) => left.startedAtMs - right.startedAtMs)
|
|
723
|
+
}
|
|
724
|
+
|
|
725
|
+
/**
|
|
726
|
+
* Logs one compact drain-timeout event to the worker's own stderr (the surface
|
|
727
|
+
* that reaches the production log), naming the held processes and how long each
|
|
728
|
+
* has been in flight.
|
|
729
|
+
* @param {object} args - Event fields.
|
|
730
|
+
* @param {string} args.phase - Drain phase that ran out of time.
|
|
731
|
+
* @param {number} args.timeoutMs - The bound that expired.
|
|
732
|
+
* @param {number} args.pendingCount - Entries still unsettled when it expired.
|
|
733
|
+
* @param {ProcessJobHolder[]} args.holders - Held processes named by the phase.
|
|
734
|
+
* @returns {void}
|
|
735
|
+
*/
|
|
736
|
+
_logDrainTimeout({phase, timeoutMs, pendingCount, holders}) {
|
|
737
|
+
const nowMs = Date.now()
|
|
738
|
+
|
|
739
|
+
console.error(JSON.stringify({
|
|
740
|
+
event: "worker-shutdown-drain-timeout",
|
|
741
|
+
phase,
|
|
742
|
+
timeoutMs,
|
|
743
|
+
pendingCount,
|
|
744
|
+
held: holders.map((holder) => ({
|
|
745
|
+
jobId: holder.jobId,
|
|
746
|
+
heldMs: nowMs - holder.startedAtMs
|
|
747
|
+
}))
|
|
748
|
+
}))
|
|
749
|
+
}
|
|
750
|
+
|
|
669
751
|
/**
|
|
670
752
|
* Terminates any process runner children still alive after the drain window so
|
|
671
753
|
* they don't outlive the worker as orphans. SIGTERM lets the runner close its
|
|
@@ -2147,12 +2229,13 @@ export default class BackgroundJobsWorker {
|
|
|
2147
2229
|
/**
|
|
2148
2230
|
* Runs fork job.
|
|
2149
2231
|
* @param {import("./types.js").BackgroundJobPayload & {id: string}} payload - Payload.
|
|
2150
|
-
* @returns {Promise<void>} - Resolves when the
|
|
2232
|
+
* @returns {Promise<void>} - Resolves when the runner reports its job outcome or the fork fails.
|
|
2151
2233
|
*/
|
|
2152
2234
|
_forkJob(payload) {
|
|
2153
2235
|
const child = this._createForkedChild()
|
|
2154
2236
|
|
|
2155
2237
|
this.inflightProcessChildren.add(child)
|
|
2238
|
+
this.processChildJobs.set(child, {jobId: payload.id, startedAtMs: Date.now()})
|
|
2156
2239
|
|
|
2157
2240
|
const finished = this._waitForForkedChild({child, payload})
|
|
2158
2241
|
|
|
@@ -2183,46 +2266,108 @@ export default class BackgroundJobsWorker {
|
|
|
2183
2266
|
* @param {object} args - Options.
|
|
2184
2267
|
* @param {import("node:child_process").ChildProcess} args.child - Forked child process.
|
|
2185
2268
|
* @param {import("./types.js").BackgroundJobPayload & {id: string}} args.payload - Payload.
|
|
2186
|
-
* @returns {Promise<void>} - Resolves when the child exits.
|
|
2269
|
+
* @returns {Promise<void>} - Resolves when the runner reports its job outcome or the child exits.
|
|
2187
2270
|
*/
|
|
2188
2271
|
_waitForForkedChild({child, payload}) {
|
|
2189
|
-
|
|
2272
|
+
/** @type {ForkedJobState} */
|
|
2273
|
+
const state = {
|
|
2274
|
+
timedOut: false,
|
|
2275
|
+
timeoutMs: null,
|
|
2276
|
+
timer: null,
|
|
2277
|
+
sigkillTimer: null,
|
|
2278
|
+
outcomeReported: false,
|
|
2279
|
+
outcomeAcknowledged: false,
|
|
2280
|
+
settled: false,
|
|
2281
|
+
resolve: () => {}
|
|
2282
|
+
}
|
|
2190
2283
|
|
|
2191
2284
|
return new Promise((resolve) => {
|
|
2285
|
+
state.resolve = resolve
|
|
2286
|
+
this._armForkedJobTimeout({child, payload, state})
|
|
2287
|
+
|
|
2288
|
+
child.once("message", (message) => {
|
|
2289
|
+
this._handleForkedChildJobReport({child, message, payload, state})
|
|
2290
|
+
})
|
|
2192
2291
|
child.once("exit", (code, signal) => {
|
|
2193
|
-
this._clearForkedJobTimeout(
|
|
2194
|
-
this._handleForkedChildExit({child, code, signal, payload,
|
|
2292
|
+
this._clearForkedJobTimeout(state)
|
|
2293
|
+
this._handleForkedChildExit({child, code, signal, payload, state})
|
|
2195
2294
|
})
|
|
2196
2295
|
child.once("error", (error) => {
|
|
2197
|
-
this._clearForkedJobTimeout(
|
|
2198
|
-
this._handleForkedChildError({child, error, payload,
|
|
2296
|
+
this._clearForkedJobTimeout(state)
|
|
2297
|
+
this._handleForkedChildError({child, error, payload, state})
|
|
2199
2298
|
})
|
|
2200
2299
|
})
|
|
2201
2300
|
}
|
|
2202
2301
|
|
|
2302
|
+
/**
|
|
2303
|
+
* Handles one forked runner's job-outcome report. The runner has finished the job
|
|
2304
|
+
* it was handed, so the worker frees the slot here instead of waiting for the
|
|
2305
|
+
* process to exit: a runner that holds no job has no reason to exit, and a
|
|
2306
|
+
* shutdown drain that waits for one never finishes. The wall-clock backstop stays
|
|
2307
|
+
* armed, so a runner that reports and then lingers is still terminated.
|
|
2308
|
+
* @param {object} args - Options.
|
|
2309
|
+
* @param {import("node:child_process").ChildProcess} args.child - Forked child process.
|
|
2310
|
+
* @param {ReturnType<typeof JSON.parse>} args.message - IPC message.
|
|
2311
|
+
* @param {import("./types.js").BackgroundJobPayload & {id: string}} args.payload - Job payload.
|
|
2312
|
+
* @param {ForkedJobState} args.state - Shared bookkeeping for this runner.
|
|
2313
|
+
* @returns {void}
|
|
2314
|
+
*/
|
|
2315
|
+
_handleForkedChildJobReport({child, message, payload, state}) {
|
|
2316
|
+
if (!isForkedChildJobReport(message)) return
|
|
2317
|
+
|
|
2318
|
+
state.outcomeReported = true
|
|
2319
|
+
state.outcomeAcknowledged = message.acknowledged === true
|
|
2320
|
+
|
|
2321
|
+
if (!state.outcomeAcknowledged) {
|
|
2322
|
+
// The runner stayed alive but could not confirm its own terminal report, so
|
|
2323
|
+
// reclaim the job exactly as the pooled lane does before freeing the slot.
|
|
2324
|
+
this._reportForkedChildFailure({
|
|
2325
|
+
payload,
|
|
2326
|
+
error: new Error(message.error || "Forked background job runner terminal report was not acknowledged")
|
|
2327
|
+
})
|
|
2328
|
+
}
|
|
2329
|
+
|
|
2330
|
+
this._settleForkedJob({child, state})
|
|
2331
|
+
}
|
|
2332
|
+
|
|
2333
|
+
/**
|
|
2334
|
+
* Frees the worker's in-flight entry for one forked job exactly once, so neither
|
|
2335
|
+
* a job-outcome report nor a later exit can settle it twice.
|
|
2336
|
+
* @param {object} args - Options.
|
|
2337
|
+
* @param {import("node:child_process").ChildProcess} args.child - Forked child process.
|
|
2338
|
+
* @param {ForkedJobState} args.state - Shared bookkeeping for this runner.
|
|
2339
|
+
* @returns {void}
|
|
2340
|
+
*/
|
|
2341
|
+
_settleForkedJob({child, state}) {
|
|
2342
|
+
if (state.settled) return
|
|
2343
|
+
|
|
2344
|
+
state.settled = true
|
|
2345
|
+
this.processChildJobs.delete(child)
|
|
2346
|
+
state.resolve(undefined)
|
|
2347
|
+
}
|
|
2348
|
+
|
|
2203
2349
|
/**
|
|
2204
2350
|
* Arms a wall-clock backstop for a forked job runner. A forked job still
|
|
2205
2351
|
* running after `jobTimeoutMs` is terminated (SIGTERM, then SIGKILL after the
|
|
2206
2352
|
* grace) so a single genuinely-hung runner can't pin a draining worker — and
|
|
2207
|
-
* its full-app boot and database connections — indefinitely.
|
|
2208
|
-
*
|
|
2209
|
-
* timeout-specific failure. When no timeout is configured the
|
|
2210
|
-
* and behavior is unchanged.
|
|
2353
|
+
* its full-app boot and database connections — indefinitely. The armed timers are
|
|
2354
|
+
* recorded on the shared state the outcome/exit/error handlers use to cancel them
|
|
2355
|
+
* and to report a timeout-specific failure. When no timeout is configured the
|
|
2356
|
+
* timer stays null and behavior is unchanged.
|
|
2211
2357
|
* @param {object} args - Options.
|
|
2212
2358
|
* @param {import("node:child_process").ChildProcess} args.child - Forked child process.
|
|
2213
2359
|
* @param {import("./types.js").BackgroundJobPayload & {id: string}} args.payload - Job payload.
|
|
2214
|
-
* @
|
|
2360
|
+
* @param {ForkedJobState} args.state - Shared bookkeeping for this runner.
|
|
2361
|
+
* @returns {void}
|
|
2215
2362
|
*/
|
|
2216
|
-
_armForkedJobTimeout({child, payload}) {
|
|
2363
|
+
_armForkedJobTimeout({child, payload, state}) {
|
|
2217
2364
|
const timeoutMs = this._resolveJobTimeoutMs(payload.options)
|
|
2218
|
-
/** @type {ForkedJobTimeoutState} */
|
|
2219
|
-
const state = {timedOut: false, timeoutMs, timer: null, sigkillTimer: null}
|
|
2220
2365
|
|
|
2221
|
-
|
|
2366
|
+
state.timeoutMs = timeoutMs
|
|
2222
2367
|
|
|
2223
|
-
|
|
2368
|
+
if (!(typeof timeoutMs === "number" && timeoutMs > 0)) return
|
|
2224
2369
|
|
|
2225
|
-
|
|
2370
|
+
state.timer = setTimeout(() => this._onForkedJobTimeout({child, state}), timeoutMs)
|
|
2226
2371
|
}
|
|
2227
2372
|
|
|
2228
2373
|
/**
|
|
@@ -2251,11 +2396,13 @@ export default class BackgroundJobsWorker {
|
|
|
2251
2396
|
/**
|
|
2252
2397
|
* Fired when a forked runner overruns its timeout. Sends SIGTERM for a clean
|
|
2253
2398
|
* shutdown, then SIGKILL after the grace for a runner that ignores it. The
|
|
2254
|
-
* resulting non-clean exit flows through `_handleForkedChildExit`, which
|
|
2255
|
-
* the
|
|
2399
|
+
* resulting non-clean exit flows through `_handleForkedChildExit`, which reports
|
|
2400
|
+
* the job failed unless the runner had already recorded its outcome — the
|
|
2401
|
+
* backstop still reaps a runner that reported and then lingered, without turning
|
|
2402
|
+
* that reap into a second terminal result for a settled job.
|
|
2256
2403
|
* @param {object} args - Options.
|
|
2257
2404
|
* @param {import("node:child_process").ChildProcess} args.child - Forked child process.
|
|
2258
|
-
* @param {
|
|
2405
|
+
* @param {ForkedJobState} args.state - Shared bookkeeping for this runner.
|
|
2259
2406
|
* @returns {void}
|
|
2260
2407
|
*/
|
|
2261
2408
|
_onForkedJobTimeout({child, state}) {
|
|
@@ -2279,7 +2426,7 @@ export default class BackgroundJobsWorker {
|
|
|
2279
2426
|
/**
|
|
2280
2427
|
* Cancels any pending timeout/SIGKILL timers for a forked runner that has
|
|
2281
2428
|
* exited (or errored) so they never fire against a gone or reused child.
|
|
2282
|
-
* @param {
|
|
2429
|
+
* @param {ForkedJobState} state - Shared bookkeeping for this runner.
|
|
2283
2430
|
* @returns {void}
|
|
2284
2431
|
*/
|
|
2285
2432
|
_clearForkedJobTimeout(state) {
|
|
@@ -2301,22 +2448,25 @@ export default class BackgroundJobsWorker {
|
|
|
2301
2448
|
* @param {number | null} args.code - Exit code.
|
|
2302
2449
|
* @param {keyof typeof import("node:os").constants.signals | null} args.signal - Exit signal.
|
|
2303
2450
|
* @param {import("./types.js").BackgroundJobPayload & {id: string}} args.payload - Payload.
|
|
2304
|
-
* @param {
|
|
2305
|
-
* @param {ForkedJobTimeoutState} [args.timeoutState] - Timeout state, when the runner had a wall-clock backstop.
|
|
2451
|
+
* @param {ForkedJobState} args.state - Shared bookkeeping for this runner.
|
|
2306
2452
|
* @returns {void}
|
|
2307
2453
|
*/
|
|
2308
|
-
_handleForkedChildExit({child, code, signal, payload,
|
|
2454
|
+
_handleForkedChildExit({child, code, signal, payload, state}) {
|
|
2309
2455
|
this.inflightProcessChildren.delete(child)
|
|
2310
2456
|
|
|
2311
2457
|
// Free the worker slot as soon as the child is gone — never gate it on the
|
|
2312
2458
|
// failure report. A hung/slow report must not leak the slot; enough leaked
|
|
2313
2459
|
// slots drive `acceptsForked` to false and silently wedge the worker.
|
|
2314
|
-
|
|
2460
|
+
this._settleForkedJob({child, state})
|
|
2315
2461
|
|
|
2316
2462
|
if (this._forkedChildExitedCleanly({code, signal})) return
|
|
2463
|
+
// The runner has already reported this job's outcome, so the outcome handler
|
|
2464
|
+
// owned the terminal result (or reclaimed it). A later non-clean exit is
|
|
2465
|
+
// teardown noise, not a second terminal result for the same job.
|
|
2466
|
+
if (state.outcomeReported) return
|
|
2317
2467
|
|
|
2318
|
-
const error =
|
|
2319
|
-
? new Error(`Forked background job runner timed out after ${
|
|
2468
|
+
const error = state.timedOut
|
|
2469
|
+
? new Error(`Forked background job runner timed out after ${state.timeoutMs}ms and was terminated: code=${code} signal=${signal || "none"}`)
|
|
2320
2470
|
: new Error(`Forked background job runner exited before reporting: code=${code} signal=${signal || "none"}`)
|
|
2321
2471
|
|
|
2322
2472
|
this._reportForkedChildFailure({payload, error})
|
|
@@ -2339,14 +2489,17 @@ export default class BackgroundJobsWorker {
|
|
|
2339
2489
|
* @param {import("node:child_process").ChildProcess} args.child - Forked child process.
|
|
2340
2490
|
* @param {Error} args.error - Child process error.
|
|
2341
2491
|
* @param {import("./types.js").BackgroundJobPayload & {id: string}} args.payload - Payload.
|
|
2342
|
-
* @param {
|
|
2492
|
+
* @param {ForkedJobState} args.state - Shared bookkeeping for this runner.
|
|
2343
2493
|
* @returns {void}
|
|
2344
2494
|
*/
|
|
2345
|
-
_handleForkedChildError({child, error, payload,
|
|
2495
|
+
_handleForkedChildError({child, error, payload, state}) {
|
|
2346
2496
|
this.inflightProcessChildren.delete(child)
|
|
2347
2497
|
// Free the slot first (see _handleForkedChildExit) — reporting is best-effort.
|
|
2348
|
-
|
|
2498
|
+
this._settleForkedJob({child, state})
|
|
2349
2499
|
console.error("Background jobs forked runner error:", error)
|
|
2500
|
+
|
|
2501
|
+
if (state.outcomeReported) return
|
|
2502
|
+
|
|
2350
2503
|
this._reportForkedChildFailure({payload, error})
|
|
2351
2504
|
}
|
|
2352
2505
|
|
|
@@ -2386,7 +2539,7 @@ export default class BackgroundJobsWorker {
|
|
|
2386
2539
|
|
|
2387
2540
|
/**
|
|
2388
2541
|
* Runs spawn job.
|
|
2389
|
-
* @param {import("./types.js").BackgroundJobPayload} payload - Payload.
|
|
2542
|
+
* @param {import("./types.js").BackgroundJobPayload & {id: string}} payload - Payload.
|
|
2390
2543
|
* @returns {Promise<void>} - Resolves when the spawned runner exits or spawn fails.
|
|
2391
2544
|
*/
|
|
2392
2545
|
_spawnJob(payload) {
|
|
@@ -2405,14 +2558,17 @@ export default class BackgroundJobsWorker {
|
|
|
2405
2558
|
})
|
|
2406
2559
|
|
|
2407
2560
|
this.inflightProcessChildren.add(child)
|
|
2561
|
+
this.processChildJobs.set(child, {jobId: payload.id, startedAtMs: Date.now()})
|
|
2408
2562
|
|
|
2409
2563
|
const finished = new Promise((resolve) => {
|
|
2410
2564
|
child.once("exit", () => {
|
|
2411
2565
|
this.inflightProcessChildren.delete(child)
|
|
2566
|
+
this.processChildJobs.delete(child)
|
|
2412
2567
|
resolve(undefined)
|
|
2413
2568
|
})
|
|
2414
2569
|
child.once("error", (error) => {
|
|
2415
2570
|
this.inflightProcessChildren.delete(child)
|
|
2571
|
+
this.processChildJobs.delete(child)
|
|
2416
2572
|
console.error("Background jobs spawned runner error:", error)
|
|
2417
2573
|
resolve(undefined)
|
|
2418
2574
|
})
|