velocious 1.0.722 → 1.0.724

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -69,9 +69,10 @@ export default class SyncAppliedSequenceStore {
69
69
 
70
70
  /**
71
71
  * Ensures the backing table exists.
72
+ * @param {{operation?: import("../database/operation.js").default | null}} [args] - Optional caller-owned operation.
72
73
  * @returns {Promise<void>} Resolves when ready.
73
74
  */
74
- async ensureReady() {
75
+ async ensureReady({operation} = {}) {
75
76
  if (this._isReady) return
76
77
 
77
78
  if (this._usesMemoryStorage()) {
@@ -79,43 +80,43 @@ export default class SyncAppliedSequenceStore {
79
80
  return
80
81
  }
81
82
 
82
- await this._withDb(async (db) => await this._ensureReadyWithDb(db))
83
+ await this._withDb(async (db) => await this._ensureReadyWithDb(db), operation)
83
84
  }
84
85
 
85
86
  /**
86
87
  * Returns the highest server sequence already applied to a record.
87
- * @param {{resourceType: string, resourceId: string}} identity - Resource identity.
88
+ * @param {{resourceType: string, resourceId: string, operation?: import("../database/operation.js").default | null}} identity - Resource identity and optional caller-owned operation.
88
89
  * @returns {Promise<number | null>} Applied sequence, or null when none is recorded.
89
90
  */
90
- async lastAppliedSequence(identity) {
91
- await this.ensureReady()
91
+ async lastAppliedSequence({resourceId, resourceType, operation}) {
92
+ await this.ensureReady({operation})
92
93
 
93
94
  if (this._usesMemoryStorage()) {
94
- return this._memorySequences.get(memoryKey(identity)) ?? null
95
+ return this._memorySequences.get(memoryKey({resourceId, resourceType})) ?? null
95
96
  }
96
97
 
97
98
  return await this._withDb(async (db) => {
98
- const row = await this._rowByResource(db, identity)
99
+ const row = await this._rowByResource(db, {resourceId, resourceType})
99
100
 
100
101
  return row ? row.serverSequence : null
101
- })
102
+ }, operation)
102
103
  }
103
104
 
104
105
  /**
105
106
  * Records a successfully applied server sequence for a record, keeping only
106
107
  * the highest value. Writes for the same record are serialized in-process so
107
108
  * concurrent pull/realtime applies cannot move the watermark backwards.
108
- * @param {{resourceType: string, resourceId: string, serverSequence: number}} args - Applied change identity and sequence.
109
+ * @param {{resourceType: string, resourceId: string, serverSequence: number, operation?: import("../database/operation.js").default | null}} args - Applied change identity, sequence and optional caller-owned operation.
109
110
  * @returns {Promise<void>} Resolves when recorded.
110
111
  */
111
- async recordAppliedSequence({resourceId, resourceType, serverSequence}) {
112
- await this.ensureReady()
112
+ async recordAppliedSequence({resourceId, resourceType, serverSequence, operation}) {
113
+ await this.ensureReady({operation})
113
114
 
114
115
  const identity = {resourceId, resourceType}
115
116
  const key = memoryKey(identity)
116
117
  const previousWrite = this._writeQueues.get(key) || Promise.resolve()
117
118
  const write = previousWrite
118
- .then(async () => await this._recordSequential({identity, key, serverSequence}))
119
+ .then(async () => await this._recordSequential({identity, key, operation, serverSequence}))
119
120
  .finally(() => {
120
121
  if (this._writeQueues.get(key) === write) this._writeQueues.delete(key)
121
122
  })
@@ -126,11 +127,11 @@ export default class SyncAppliedSequenceStore {
126
127
 
127
128
  /**
128
129
  * Applies one serialized watermark write.
129
- * @param {{identity: {resourceType: string, resourceId: string}, key: string, serverSequence: number}} args - Write args.
130
+ * @param {{identity: {resourceType: string, resourceId: string}, key: string, serverSequence: number, operation?: import("../database/operation.js").default | null}} args - Write args.
130
131
  * @returns {Promise<void>} Resolves when written.
131
132
  */
132
- async _recordSequential({identity, key, serverSequence}) {
133
- const currentSequence = await this.lastAppliedSequence(identity)
133
+ async _recordSequential({identity, key, serverSequence, operation}) {
134
+ const currentSequence = await this.lastAppliedSequence({resourceType: identity.resourceType, resourceId: identity.resourceId, operation})
134
135
 
135
136
  if (currentSequence !== null && currentSequence >= serverSequence) return
136
137
 
@@ -153,7 +154,7 @@ export default class SyncAppliedSequenceStore {
153
154
  tableName: TABLE_NAME,
154
155
  updateColumns: ["server_sequence", "updated_at"]
155
156
  })
156
- })
157
+ }, operation)
157
158
  }
158
159
 
159
160
  /**
@@ -227,11 +228,23 @@ export default class SyncAppliedSequenceStore {
227
228
 
228
229
  /**
229
230
  * Runs a callback with a database connection.
231
+ *
232
+ * A caller that already owns an operation on this store's physical database
233
+ * must hand it in: taking a second checkout from the same physical database
234
+ * would queue behind the operation the caller is running inside and wait
235
+ * behind itself.
230
236
  * @template Result
231
237
  * @param {(db: import("../database/drivers/base.js").default) => Promise<Result>} callback - Database callback.
238
+ * @param {import("../database/operation.js").default | null} [operation] - Caller-owned operation to run on.
232
239
  * @returns {Promise<Result>} Callback result.
233
240
  */
234
- async _withDb(callback) {
241
+ async _withDb(callback, operation) {
242
+ if (operation) {
243
+ this.assertOperationOwnership(operation)
244
+
245
+ return await callback(operation.connection())
246
+ }
247
+
235
248
  if (this.tenantHandle) {
236
249
  return await this.tenantHandle.databaseOperation({databaseIdentifier: this.databaseIdentifier, name: "Tenant sync applied-sequence store"}, async (operation) => {
237
250
  return await callback(operation.connection())
@@ -247,6 +260,22 @@ export default class SyncAppliedSequenceStore {
247
260
  })
248
261
  }
249
262
 
263
+ /**
264
+ * Fails closed when a caller hands in an operation that does not belong to this store's database.
265
+ * @param {import("../database/operation.js").default} operation - Caller-owned operation.
266
+ * @returns {void} Nothing when the operation matches the store.
267
+ * @throws {Error} When the operation belongs to another database or another physical tenant database.
268
+ */
269
+ assertOperationOwnership(operation) {
270
+ if (operation.databaseIdentifier() !== this.databaseIdentifier) {
271
+ throw new Error(`Sync applied-sequence store operation belongs to another database: ${operation.databaseIdentifier()}`)
272
+ }
273
+
274
+ if (this.tenantHandle && operation.databaseIdentity() !== this.storeIdentity) {
275
+ throw new Error(`Sync applied-sequence store operation belongs to another physical database: ${operation.databaseIdentity()}`)
276
+ }
277
+ }
278
+
250
279
  /**
251
280
  * Ensures the applied-sequences table exists.
252
281
  * @param {import("../database/drivers/base.js").default} db - Database connection.
@@ -861,7 +861,7 @@ export default class SyncClient {
861
861
  const appliedSequences = this.appliedSequenceStore()
862
862
 
863
863
  if (serverSequence !== null) {
864
- const lastAppliedSequence = await appliedSequences.lastAppliedSequence({resourceId, resourceType})
864
+ const lastAppliedSequence = await appliedSequences.lastAppliedSequence({resourceId, resourceType, operation})
865
865
 
866
866
  if (lastAppliedSequence !== null && serverSequence <= lastAppliedSequence) {
867
867
  return {changed: false, resourceType}
@@ -888,7 +888,7 @@ export default class SyncClient {
888
888
  const result = await applier(sync)
889
889
 
890
890
  if (serverSequence !== null) {
891
- await appliedSequences.recordAppliedSequence({resourceId, resourceType, serverSequence})
891
+ await appliedSequences.recordAppliedSequence({resourceId, resourceType, serverSequence, operation})
892
892
  }
893
893
 
894
894
  return result
package/package.json CHANGED
@@ -3,7 +3,7 @@
3
3
  "velocious": "build/bin/velocious.js"
4
4
  },
5
5
  "name": "velocious",
6
- "version": "1.0.722",
6
+ "version": "1.0.724",
7
7
  "main": "build/index.js",
8
8
  "types": "build/index.d.ts",
9
9
  "files": [
@@ -1,9 +1,16 @@
1
1
  // @ts-check
2
2
 
3
- import runJobPayload from "./job-runner.js"
3
+ import runJobPayload, { BackgroundJobPerformedFailure } from "./job-runner.js"
4
4
  import { closeRunnerConnections, currentConfigurationOrNull } from "./runner-graceful-shutdown.js"
5
5
  import setRunnerProcessTitle from "./runner-process-title.js"
6
6
 
7
+ /**
8
+ * @typedef {object} ForkedJobOutcome
9
+ * @property {number} exitCode - Exit code the runner exits with once the outcome is reported.
10
+ * @property {boolean} acknowledged - Whether the job's terminal report was recorded.
11
+ * @property {string} [error] - Why the terminal report could not be recorded.
12
+ */
13
+
7
14
  // Name the process so `ps`/`top`/`htop` can identify forked job runners at a
8
15
  // glance instead of a wall of generic "node" entries. Updated to the specific
9
16
  // job name once one arrives (see runJobMessage), so operators can see exactly
@@ -47,30 +54,39 @@ function isJobMessage(message) {
47
54
 
48
55
  /**
49
56
  * Runs finish.
50
- * @param {number} exitCode - Process exit code.
57
+ *
58
+ * The report tells the parent whether this runner's job has its terminal result.
59
+ * The parent frees its in-flight entry on an acknowledged outcome instead of
60
+ * waiting for this process to exit: a runner that holds no job has no reason to
61
+ * exit, and a parent that waits for one never finishes its shutdown drain.
62
+ * @param {ForkedJobOutcome & {jobId?: string}} args - Outcome report.
51
63
  * @returns {Promise<void>}
52
64
  */
53
- async function finish(exitCode) {
65
+ async function finish(args) {
66
+ const {acknowledged, error, exitCode, jobId} = args
67
+
54
68
  if (process.send) {
55
- await new Promise((resolve) => process.send?.({type: "job-reported"}, () => resolve(undefined)))
69
+ /** @type {{acknowledged: boolean, error?: string, jobId?: string, type: "job-reported"}} */
70
+ const report = {type: "job-reported", acknowledged}
71
+ if (typeof jobId === "string") report.jobId = jobId
72
+ if (typeof error === "string") report.error = error
73
+
74
+ await new Promise((resolve) => process.send?.(report, () => resolve(undefined)))
56
75
  }
76
+
57
77
  await shutdownRunner(exitCode)
58
78
  }
59
79
 
60
80
  /**
61
81
  * Runs run job message.
62
- * @param {ReturnType<typeof JSON.parse>} message - IPC message.
82
+ * @param {import("./types.js").BackgroundJobPayload} payload - Job payload.
63
83
  * @returns {Promise<void>} - Resolves after the payload has run.
64
84
  */
65
- async function runJobMessage(message) {
66
- if (!isJobMessage(message)) {
67
- throw new Error("Forked background job runner received invalid payload")
68
- }
69
-
85
+ async function runJobMessage(payload) {
70
86
  // The per-job process title (and its restore) is set inside runJobPayload,
71
87
  // which reads the job class's `static processTitle`. This process boots with
72
88
  // the base "velocious background-jobs-runner" title set at module load above.
73
- await runJobPayload(message.payload, {
89
+ await runJobPayload(payload, {
74
90
  closeConnections: false,
75
91
  processType: "background-jobs-forked-runner"
76
92
  })
@@ -82,17 +98,29 @@ async function runJobMessage(message) {
82
98
  * @returns {Promise<void>} - Resolves after completion is reported.
83
99
  */
84
100
  async function handleJobMessage(message) {
85
- let exitCode
101
+ if (!isJobMessage(message)) {
102
+ await finish({acknowledged: false, error: "Forked background job runner received invalid payload", exitCode: 1})
103
+
104
+ return
105
+ }
106
+
107
+ /** @type {ForkedJobOutcome} */
108
+ let outcome
86
109
 
87
110
  try {
88
- await runJobMessage(message)
89
- exitCode = 0
111
+ await runJobMessage(message.payload)
112
+ outcome = {acknowledged: true, exitCode: 0}
90
113
  } catch (error) {
91
114
  console.error("Forked background job runner failed:", error)
92
- exitCode = 1
115
+ // A perform error whose terminal report was acknowledged is still an outcome:
116
+ // the job has its result. Only a failure that left the job with no
117
+ // acknowledged terminal report has to be reclaimed by the parent.
118
+ outcome = error instanceof BackgroundJobPerformedFailure
119
+ ? {acknowledged: true, exitCode: 1}
120
+ : {acknowledged: false, error: error instanceof Error ? error.message : String(error), exitCode: 1}
93
121
  }
94
122
 
95
- await finish(exitCode)
123
+ await finish({...outcome, jobId: message.payload.id})
96
124
  }
97
125
 
98
126
  for (const signal of ["SIGTERM", "SIGINT"]) {
@@ -17,12 +17,24 @@ import BackgroundJobsGenerationHandshakeTimeoutError, { DEFAULT_GENERATION_HANDS
17
17
  import { POOLED_RUNNER_INFLIGHT_JOB_ID_LIMIT, boundedPooledRunnerInflightJobIds, isPooledChildShutdownReason, isPooledChildShutdownSignal } from "./pooled-runner-shutdown.js"
18
18
 
19
19
  /**
20
- * Per-forked-child timeout bookkeeping.
21
- * @typedef {object} ForkedJobTimeoutState
20
+ * Per-forked-child bookkeeping, shared by the wall-clock backstop, the runner's
21
+ * own job-outcome report, and the exit/error handlers.
22
+ * @typedef {object} ForkedJobState
22
23
  * @property {boolean} timedOut - Whether the timeout fired and the child was terminated.
23
24
  * @property {number | null} timeoutMs - The armed timeout in ms, or null when disabled.
24
25
  * @property {ReturnType<typeof setTimeout> | null} timer - The pending timeout timer, cleared on exit.
25
26
  * @property {ReturnType<typeof setTimeout> | null} sigkillTimer - The pending SIGKILL grace timer, cleared on exit.
27
+ * @property {boolean} outcomeReported - Whether the runner reported its job outcome over IPC.
28
+ * @property {boolean} outcomeAcknowledged - Whether that report confirmed the outcome was recorded.
29
+ * @property {boolean} settled - Whether the worker's in-flight entry for this job has been freed.
30
+ * @property {(value: void) => void} resolve - Resolver that frees the in-flight entry; a placeholder until the promise that owns the entry is created.
31
+ */
32
+ /**
33
+ * One live runner child, kept so a shutdown that runs out of drain time can name
34
+ * the exact process it is still waiting on instead of a bare count.
35
+ * @typedef {object} ProcessJobHolder
36
+ * @property {string} jobId - Job the child was handed.
37
+ * @property {number} startedAtMs - When the worker handed it that job.
26
38
  */
27
39
  /**
28
40
  * @typedef {object} PooledJobEntry
@@ -123,6 +135,24 @@ function isChildAcceptanceMessage(message) {
123
135
  && (record.runnerHostname === undefined || typeof record.runnerHostname === "string")
124
136
  }
125
137
 
138
+ /**
139
+ * Checks whether an IPC value is a forked runner's job-outcome report. The forked
140
+ * child sends exactly one of these, once its own terminal report has settled, and
141
+ * says whether that report was acknowledged — so the worker can tell a runner that
142
+ * has finished its job from one that is still running it.
143
+ * @param {ReturnType<typeof JSON.parse>} message - IPC message.
144
+ * @returns {message is {acknowledged: boolean, error?: string, jobId?: string, type: "job-reported"}} - Whether this is a job-outcome report.
145
+ */
146
+ function isForkedChildJobReport(message) {
147
+ if (!message || typeof message !== "object") return false
148
+ const record = /** @type {{acknowledged?: ReturnType<typeof JSON.parse>, error?: ReturnType<typeof JSON.parse>, jobId?: ReturnType<typeof JSON.parse>, type?: ReturnType<typeof JSON.parse>}} */ (message)
149
+
150
+ return record.type === "job-reported"
151
+ && typeof record.acknowledged === "boolean"
152
+ && (record.jobId === undefined || typeof record.jobId === "string")
153
+ && (record.error === undefined || typeof record.error === "string")
154
+ }
155
+
126
156
  /**
127
157
  * Checks whether an IPC value is the child's bounded pre-teardown observation.
128
158
  * @param {ReturnType<typeof JSON.parse>} message - IPC message.
@@ -366,6 +396,13 @@ export default class BackgroundJobsWorker {
366
396
  * @type {Set<import("node:child_process").ChildProcess>}
367
397
  */
368
398
  this.inflightProcessChildren = new Set()
399
+ /**
400
+ * Which job each live runner child was handed. A shutdown that runs out of
401
+ * drain time names the exact process it is still waiting on (pid, job, how
402
+ * long it has been held) from this map instead of reporting only a count.
403
+ * @type {Map<import("node:child_process").ChildProcess, ProcessJobHolder>}
404
+ */
405
+ this.processChildJobs = new Map()
369
406
  /** @type {Set<Promise<void>>} */
370
407
  this.inflightPooledJobs = new Set()
371
408
  /** @type {Map<string, Array<import("./types.js").BackgroundJobPayload & {id: string}>>} */
@@ -551,9 +588,9 @@ export default class BackgroundJobsWorker {
551
588
  }
552
589
  }
553
590
 
554
- await this._drainInflight(this.inflightInlineJobs, timeoutMs)
555
- await this._drainInflight(this.inflightPooledJobs, timeoutMs)
556
- await this._drainInflight(this.inflightProcessJobs, timeoutMs)
591
+ await this._drainInflight(this.inflightInlineJobs, timeoutMs, {phase: "inline-jobs"})
592
+ await this._drainInflight(this.inflightPooledJobs, timeoutMs, {phase: "pooled-jobs"})
593
+ await this._drainInflight(this.inflightProcessJobs, timeoutMs, {phase: "process-jobs", describeHolders: () => this._describeProcessJobHolders()})
557
594
  await this._terminateProcessChildren()
558
595
  // Give in-flight result reports (now decoupled from job slots) a bounded
559
596
  // chance to land before the socket closes.
@@ -645,12 +682,17 @@ export default class BackgroundJobsWorker {
645
682
 
646
683
  /**
647
684
  * Waits for a set of in-flight job promises to settle, optionally bounded by
648
- * `timeoutMs`.
685
+ * `timeoutMs`. When a bounded window runs out with entries still held, the drain
686
+ * names what it is waiting on instead of returning silently: an unexplained wait
687
+ * is what turned one wedged worker into a 28-minute investigation.
649
688
  * @param {Set<Promise<void>>} inflight - In-flight job promises.
650
689
  * @param {number} [timeoutMs] - Max wait in ms; unbounded when omitted.
690
+ * @param {object} [options] - Drain options.
691
+ * @param {string} [options.phase] - Drain phase named in a timeout report.
692
+ * @param {(() => ProcessJobHolder[])} [options.describeHolders] - Names the live processes this phase is waiting on.
651
693
  * @returns {Promise<void>} - Resolves when settled or the timeout elapses.
652
694
  */
653
- async _drainInflight(inflight, timeoutMs) {
695
+ async _drainInflight(inflight, timeoutMs, {phase = "inflight", describeHolders} = {}) {
654
696
  if (inflight.size === 0) return
655
697
 
656
698
  const drain = Promise.allSettled([...inflight])
@@ -661,11 +703,51 @@ export default class BackgroundJobsWorker {
661
703
 
662
704
  await Promise.race([drain, timeout])
663
705
  clearTimeout(timer)
706
+
707
+ if (inflight.size > 0) {
708
+ this._logDrainTimeout({phase, timeoutMs, pendingCount: inflight.size, holders: describeHolders ? describeHolders() : []})
709
+ }
664
710
  } else {
665
711
  await drain
666
712
  }
667
713
  }
668
714
 
715
+ /**
716
+ * Names every live runner child the worker still holds a job for, so a drain
717
+ * that ran out of time says which process it is waiting on, for which job, and
718
+ * for how long.
719
+ * @returns {ProcessJobHolder[]} - Held job descriptors, oldest first.
720
+ */
721
+ _describeProcessJobHolders() {
722
+ return [...this.processChildJobs.values()].sort((left, right) => left.startedAtMs - right.startedAtMs)
723
+ }
724
+
725
+ /**
726
+ * Logs one compact drain-timeout event to the worker's own stderr (the surface
727
+ * that reaches the production log), naming the held processes and how long each
728
+ * has been in flight.
729
+ * @param {object} args - Event fields.
730
+ * @param {string} args.phase - Drain phase that ran out of time.
731
+ * @param {number} args.timeoutMs - The bound that expired.
732
+ * @param {number} args.pendingCount - Entries still unsettled when it expired.
733
+ * @param {ProcessJobHolder[]} args.holders - Held processes named by the phase.
734
+ * @returns {void}
735
+ */
736
+ _logDrainTimeout({phase, timeoutMs, pendingCount, holders}) {
737
+ const nowMs = Date.now()
738
+
739
+ console.error(JSON.stringify({
740
+ event: "worker-shutdown-drain-timeout",
741
+ phase,
742
+ timeoutMs,
743
+ pendingCount,
744
+ held: holders.map((holder) => ({
745
+ jobId: holder.jobId,
746
+ heldMs: nowMs - holder.startedAtMs
747
+ }))
748
+ }))
749
+ }
750
+
669
751
  /**
670
752
  * Terminates any process runner children still alive after the drain window so
671
753
  * they don't outlive the worker as orphans. SIGTERM lets the runner close its
@@ -2147,12 +2229,13 @@ export default class BackgroundJobsWorker {
2147
2229
  /**
2148
2230
  * Runs fork job.
2149
2231
  * @param {import("./types.js").BackgroundJobPayload & {id: string}} payload - Payload.
2150
- * @returns {Promise<void>} - Resolves when the forked runner exits or fork fails.
2232
+ * @returns {Promise<void>} - Resolves when the runner reports its job outcome or the fork fails.
2151
2233
  */
2152
2234
  _forkJob(payload) {
2153
2235
  const child = this._createForkedChild()
2154
2236
 
2155
2237
  this.inflightProcessChildren.add(child)
2238
+ this.processChildJobs.set(child, {jobId: payload.id, startedAtMs: Date.now()})
2156
2239
 
2157
2240
  const finished = this._waitForForkedChild({child, payload})
2158
2241
 
@@ -2183,46 +2266,108 @@ export default class BackgroundJobsWorker {
2183
2266
  * @param {object} args - Options.
2184
2267
  * @param {import("node:child_process").ChildProcess} args.child - Forked child process.
2185
2268
  * @param {import("./types.js").BackgroundJobPayload & {id: string}} args.payload - Payload.
2186
- * @returns {Promise<void>} - Resolves when the child exits.
2269
+ * @returns {Promise<void>} - Resolves when the runner reports its job outcome or the child exits.
2187
2270
  */
2188
2271
  _waitForForkedChild({child, payload}) {
2189
- const timeoutState = this._armForkedJobTimeout({child, payload})
2272
+ /** @type {ForkedJobState} */
2273
+ const state = {
2274
+ timedOut: false,
2275
+ timeoutMs: null,
2276
+ timer: null,
2277
+ sigkillTimer: null,
2278
+ outcomeReported: false,
2279
+ outcomeAcknowledged: false,
2280
+ settled: false,
2281
+ resolve: () => {}
2282
+ }
2190
2283
 
2191
2284
  return new Promise((resolve) => {
2285
+ state.resolve = resolve
2286
+ this._armForkedJobTimeout({child, payload, state})
2287
+
2288
+ child.once("message", (message) => {
2289
+ this._handleForkedChildJobReport({child, message, payload, state})
2290
+ })
2192
2291
  child.once("exit", (code, signal) => {
2193
- this._clearForkedJobTimeout(timeoutState)
2194
- this._handleForkedChildExit({child, code, signal, payload, resolve, timeoutState})
2292
+ this._clearForkedJobTimeout(state)
2293
+ this._handleForkedChildExit({child, code, signal, payload, state})
2195
2294
  })
2196
2295
  child.once("error", (error) => {
2197
- this._clearForkedJobTimeout(timeoutState)
2198
- this._handleForkedChildError({child, error, payload, resolve})
2296
+ this._clearForkedJobTimeout(state)
2297
+ this._handleForkedChildError({child, error, payload, state})
2199
2298
  })
2200
2299
  })
2201
2300
  }
2202
2301
 
2302
+ /**
2303
+ * Handles one forked runner's job-outcome report. The runner has finished the job
2304
+ * it was handed, so the worker frees the slot here instead of waiting for the
2305
+ * process to exit: a runner that holds no job has no reason to exit, and a
2306
+ * shutdown drain that waits for one never finishes. The wall-clock backstop stays
2307
+ * armed, so a runner that reports and then lingers is still terminated.
2308
+ * @param {object} args - Options.
2309
+ * @param {import("node:child_process").ChildProcess} args.child - Forked child process.
2310
+ * @param {ReturnType<typeof JSON.parse>} args.message - IPC message.
2311
+ * @param {import("./types.js").BackgroundJobPayload & {id: string}} args.payload - Job payload.
2312
+ * @param {ForkedJobState} args.state - Shared bookkeeping for this runner.
2313
+ * @returns {void}
2314
+ */
2315
+ _handleForkedChildJobReport({child, message, payload, state}) {
2316
+ if (!isForkedChildJobReport(message)) return
2317
+
2318
+ state.outcomeReported = true
2319
+ state.outcomeAcknowledged = message.acknowledged === true
2320
+
2321
+ if (!state.outcomeAcknowledged) {
2322
+ // The runner stayed alive but could not confirm its own terminal report, so
2323
+ // reclaim the job exactly as the pooled lane does before freeing the slot.
2324
+ this._reportForkedChildFailure({
2325
+ payload,
2326
+ error: new Error(message.error || "Forked background job runner terminal report was not acknowledged")
2327
+ })
2328
+ }
2329
+
2330
+ this._settleForkedJob({child, state})
2331
+ }
2332
+
2333
+ /**
2334
+ * Frees the worker's in-flight entry for one forked job exactly once, so neither
2335
+ * a job-outcome report nor a later exit can settle it twice.
2336
+ * @param {object} args - Options.
2337
+ * @param {import("node:child_process").ChildProcess} args.child - Forked child process.
2338
+ * @param {ForkedJobState} args.state - Shared bookkeeping for this runner.
2339
+ * @returns {void}
2340
+ */
2341
+ _settleForkedJob({child, state}) {
2342
+ if (state.settled) return
2343
+
2344
+ state.settled = true
2345
+ this.processChildJobs.delete(child)
2346
+ state.resolve(undefined)
2347
+ }
2348
+
2203
2349
  /**
2204
2350
  * Arms a wall-clock backstop for a forked job runner. A forked job still
2205
2351
  * running after `jobTimeoutMs` is terminated (SIGTERM, then SIGKILL after the
2206
2352
  * grace) so a single genuinely-hung runner can't pin a draining worker — and
2207
- * its full-app boot and database connections — indefinitely. Returns a state
2208
- * object the exit/error handlers use to cancel the timer and to report a
2209
- * timeout-specific failure. When no timeout is configured the timer is null
2210
- * and behavior is unchanged.
2353
+ * its full-app boot and database connections — indefinitely. The armed timers are
2354
+ * recorded on the shared state the outcome/exit/error handlers use to cancel them
2355
+ * and to report a timeout-specific failure. When no timeout is configured the
2356
+ * timer stays null and behavior is unchanged.
2211
2357
  * @param {object} args - Options.
2212
2358
  * @param {import("node:child_process").ChildProcess} args.child - Forked child process.
2213
2359
  * @param {import("./types.js").BackgroundJobPayload & {id: string}} args.payload - Job payload.
2214
- * @returns {ForkedJobTimeoutState} - Timeout state.
2360
+ * @param {ForkedJobState} args.state - Shared bookkeeping for this runner.
2361
+ * @returns {void}
2215
2362
  */
2216
- _armForkedJobTimeout({child, payload}) {
2363
+ _armForkedJobTimeout({child, payload, state}) {
2217
2364
  const timeoutMs = this._resolveJobTimeoutMs(payload.options)
2218
- /** @type {ForkedJobTimeoutState} */
2219
- const state = {timedOut: false, timeoutMs, timer: null, sigkillTimer: null}
2220
2365
 
2221
- if (!(typeof timeoutMs === "number" && timeoutMs > 0)) return state
2366
+ state.timeoutMs = timeoutMs
2222
2367
 
2223
- state.timer = setTimeout(() => this._onForkedJobTimeout({child, state}), timeoutMs)
2368
+ if (!(typeof timeoutMs === "number" && timeoutMs > 0)) return
2224
2369
 
2225
- return state
2370
+ state.timer = setTimeout(() => this._onForkedJobTimeout({child, state}), timeoutMs)
2226
2371
  }
2227
2372
 
2228
2373
  /**
@@ -2251,11 +2396,13 @@ export default class BackgroundJobsWorker {
2251
2396
  /**
2252
2397
  * Fired when a forked runner overruns its timeout. Sends SIGTERM for a clean
2253
2398
  * shutdown, then SIGKILL after the grace for a runner that ignores it. The
2254
- * resulting non-clean exit flows through `_handleForkedChildExit`, which frees
2255
- * the slot and reports the job failed.
2399
+ * resulting non-clean exit flows through `_handleForkedChildExit`, which reports
2400
+ * the job failed unless the runner had already recorded its outcome — the
2401
+ * backstop still reaps a runner that reported and then lingered, without turning
2402
+ * that reap into a second terminal result for a settled job.
2256
2403
  * @param {object} args - Options.
2257
2404
  * @param {import("node:child_process").ChildProcess} args.child - Forked child process.
2258
- * @param {ForkedJobTimeoutState} args.state - Timeout state.
2405
+ * @param {ForkedJobState} args.state - Shared bookkeeping for this runner.
2259
2406
  * @returns {void}
2260
2407
  */
2261
2408
  _onForkedJobTimeout({child, state}) {
@@ -2279,7 +2426,7 @@ export default class BackgroundJobsWorker {
2279
2426
  /**
2280
2427
  * Cancels any pending timeout/SIGKILL timers for a forked runner that has
2281
2428
  * exited (or errored) so they never fire against a gone or reused child.
2282
- * @param {ForkedJobTimeoutState} state - Timeout state.
2429
+ * @param {ForkedJobState} state - Shared bookkeeping for this runner.
2283
2430
  * @returns {void}
2284
2431
  */
2285
2432
  _clearForkedJobTimeout(state) {
@@ -2301,22 +2448,25 @@ export default class BackgroundJobsWorker {
2301
2448
  * @param {number | null} args.code - Exit code.
2302
2449
  * @param {keyof typeof import("node:os").constants.signals | null} args.signal - Exit signal.
2303
2450
  * @param {import("./types.js").BackgroundJobPayload & {id: string}} args.payload - Payload.
2304
- * @param {(value: void) => void} args.resolve - Promise resolver.
2305
- * @param {ForkedJobTimeoutState} [args.timeoutState] - Timeout state, when the runner had a wall-clock backstop.
2451
+ * @param {ForkedJobState} args.state - Shared bookkeeping for this runner.
2306
2452
  * @returns {void}
2307
2453
  */
2308
- _handleForkedChildExit({child, code, signal, payload, resolve, timeoutState}) {
2454
+ _handleForkedChildExit({child, code, signal, payload, state}) {
2309
2455
  this.inflightProcessChildren.delete(child)
2310
2456
 
2311
2457
  // Free the worker slot as soon as the child is gone — never gate it on the
2312
2458
  // failure report. A hung/slow report must not leak the slot; enough leaked
2313
2459
  // slots drive `acceptsForked` to false and silently wedge the worker.
2314
- resolve(undefined)
2460
+ this._settleForkedJob({child, state})
2315
2461
 
2316
2462
  if (this._forkedChildExitedCleanly({code, signal})) return
2463
+ // The runner has already reported this job's outcome, so the outcome handler
2464
+ // owned the terminal result (or reclaimed it). A later non-clean exit is
2465
+ // teardown noise, not a second terminal result for the same job.
2466
+ if (state.outcomeReported) return
2317
2467
 
2318
- const error = timeoutState?.timedOut
2319
- ? new Error(`Forked background job runner timed out after ${timeoutState.timeoutMs}ms and was terminated: code=${code} signal=${signal || "none"}`)
2468
+ const error = state.timedOut
2469
+ ? new Error(`Forked background job runner timed out after ${state.timeoutMs}ms and was terminated: code=${code} signal=${signal || "none"}`)
2320
2470
  : new Error(`Forked background job runner exited before reporting: code=${code} signal=${signal || "none"}`)
2321
2471
 
2322
2472
  this._reportForkedChildFailure({payload, error})
@@ -2339,14 +2489,17 @@ export default class BackgroundJobsWorker {
2339
2489
  * @param {import("node:child_process").ChildProcess} args.child - Forked child process.
2340
2490
  * @param {Error} args.error - Child process error.
2341
2491
  * @param {import("./types.js").BackgroundJobPayload & {id: string}} args.payload - Payload.
2342
- * @param {(value: void) => void} args.resolve - Promise resolver.
2492
+ * @param {ForkedJobState} args.state - Shared bookkeeping for this runner.
2343
2493
  * @returns {void}
2344
2494
  */
2345
- _handleForkedChildError({child, error, payload, resolve}) {
2495
+ _handleForkedChildError({child, error, payload, state}) {
2346
2496
  this.inflightProcessChildren.delete(child)
2347
2497
  // Free the slot first (see _handleForkedChildExit) — reporting is best-effort.
2348
- resolve(undefined)
2498
+ this._settleForkedJob({child, state})
2349
2499
  console.error("Background jobs forked runner error:", error)
2500
+
2501
+ if (state.outcomeReported) return
2502
+
2350
2503
  this._reportForkedChildFailure({payload, error})
2351
2504
  }
2352
2505
 
@@ -2386,7 +2539,7 @@ export default class BackgroundJobsWorker {
2386
2539
 
2387
2540
  /**
2388
2541
  * Runs spawn job.
2389
- * @param {import("./types.js").BackgroundJobPayload} payload - Payload.
2542
+ * @param {import("./types.js").BackgroundJobPayload & {id: string}} payload - Payload.
2390
2543
  * @returns {Promise<void>} - Resolves when the spawned runner exits or spawn fails.
2391
2544
  */
2392
2545
  _spawnJob(payload) {
@@ -2405,14 +2558,17 @@ export default class BackgroundJobsWorker {
2405
2558
  })
2406
2559
 
2407
2560
  this.inflightProcessChildren.add(child)
2561
+ this.processChildJobs.set(child, {jobId: payload.id, startedAtMs: Date.now()})
2408
2562
 
2409
2563
  const finished = new Promise((resolve) => {
2410
2564
  child.once("exit", () => {
2411
2565
  this.inflightProcessChildren.delete(child)
2566
+ this.processChildJobs.delete(child)
2412
2567
  resolve(undefined)
2413
2568
  })
2414
2569
  child.once("error", (error) => {
2415
2570
  this.inflightProcessChildren.delete(child)
2571
+ this.processChildJobs.delete(child)
2416
2572
  console.error("Background jobs spawned runner error:", error)
2417
2573
  resolve(undefined)
2418
2574
  })