velocious 1.0.753 → 1.0.755

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. package/README.md +1 -1
  2. package/build/background-jobs/adapter.js +3 -3
  3. package/build/background-jobs/client.js +6 -5
  4. package/build/background-jobs/local-adapter.js +3 -3
  5. package/build/background-jobs/local-store.js +10 -5
  6. package/build/background-jobs/main.js +87 -8
  7. package/build/background-jobs/refusal-error.js +87 -0
  8. package/build/background-jobs/store.js +109 -10
  9. package/build/background-jobs/types.js +5 -0
  10. package/build/background-jobs/worker.js +50 -10
  11. package/build/configuration-types.js +8 -0
  12. package/build/configuration.js +32 -1
  13. package/build/environment-handlers/node/cli/commands/background-jobs-worker.js +4 -27
  14. package/build/src/background-jobs/adapter.d.ts +15 -6
  15. package/build/src/background-jobs/adapter.d.ts.map +1 -1
  16. package/build/src/background-jobs/adapter.js +4 -4
  17. package/build/src/background-jobs/client.d.ts.map +1 -1
  18. package/build/src/background-jobs/client.js +7 -6
  19. package/build/src/background-jobs/local-adapter.d.ts +8 -4
  20. package/build/src/background-jobs/local-adapter.d.ts.map +1 -1
  21. package/build/src/background-jobs/local-adapter.js +4 -4
  22. package/build/src/background-jobs/local-store.d.ts +12 -5
  23. package/build/src/background-jobs/local-store.d.ts.map +1 -1
  24. package/build/src/background-jobs/local-store.js +10 -6
  25. package/build/src/background-jobs/main.d.ts +46 -2
  26. package/build/src/background-jobs/main.d.ts.map +1 -1
  27. package/build/src/background-jobs/main.js +85 -9
  28. package/build/src/background-jobs/refusal-error.d.ts +57 -0
  29. package/build/src/background-jobs/refusal-error.d.ts.map +1 -0
  30. package/build/src/background-jobs/refusal-error.js +80 -0
  31. package/build/src/background-jobs/store.d.ts +43 -9
  32. package/build/src/background-jobs/store.d.ts.map +1 -1
  33. package/build/src/background-jobs/store.js +99 -11
  34. package/build/src/background-jobs/types.d.ts +10 -0
  35. package/build/src/background-jobs/types.d.ts.map +1 -1
  36. package/build/src/background-jobs/types.js +6 -1
  37. package/build/src/background-jobs/worker.d.ts +30 -2
  38. package/build/src/background-jobs/worker.d.ts.map +1 -1
  39. package/build/src/background-jobs/worker.js +50 -11
  40. package/build/src/configuration-types.d.ts +19 -0
  41. package/build/src/configuration-types.d.ts.map +1 -1
  42. package/build/src/configuration-types.js +9 -1
  43. package/build/src/configuration.d.ts.map +1 -1
  44. package/build/src/configuration.js +31 -2
  45. package/build/src/environment-handlers/node/cli/commands/background-jobs-worker.d.ts.map +1 -1
  46. package/build/src/environment-handlers/node/cli/commands/background-jobs-worker.js +5 -24
  47. package/package.json +1 -1
  48. package/scripts/verify-docker-dev-environment.js +3 -3
  49. package/src/background-jobs/adapter.js +3 -3
  50. package/src/background-jobs/client.js +6 -5
  51. package/src/background-jobs/local-adapter.js +3 -3
  52. package/src/background-jobs/local-store.js +10 -5
  53. package/src/background-jobs/main.js +87 -8
  54. package/src/background-jobs/refusal-error.js +87 -0
  55. package/src/background-jobs/store.js +109 -10
  56. package/src/background-jobs/types.js +5 -0
  57. package/src/background-jobs/worker.js +50 -10
  58. package/src/configuration-types.js +8 -0
  59. package/src/configuration.js +32 -1
  60. package/src/environment-handlers/node/cli/commands/background-jobs-worker.js +4 -27
@@ -1019,11 +1019,49 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
1019
1019
 
1020
1020
  if (!owned) {
1021
1021
  throw VelociousError.safe("Background job producer handoff is no longer owned.", {
1022
- code: "background-job-producer-handoff-not-owned"
1022
+ code: "background-job-producer-handoff-not-owned",
1023
+ details: this._producerHandoffLossDetails(producer, producerProof)
1023
1024
  })
1024
1025
  }
1025
1026
  }
1026
1027
 
1028
+ /**
1029
+ * Names, from the row just read inside the fence's own transaction, why the
1030
+ * producer proof no longer matches. The refusal reaches consumers over the
1031
+ * jobs-main socket as text only, so without this the same lifecycle loss is
1032
+ * indistinguishable from an unexplained enqueue failure.
1033
+ * @param {import("./types.js").BackgroundJobRow | null} producer - Producer row, or null when it is gone.
1034
+ * @param {import("./types.js").BackgroundJobProducerProof} producerProof - Proof the producer execution holds.
1035
+ * @returns {Record<string, string | number>} - Measured loss reason and the state it was measured from.
1036
+ */
1037
+ _producerHandoffLossDetails(producer, producerProof) {
1038
+ if (!producer) {
1039
+ return {
1040
+ producerJobId: producerProof.jobId,
1041
+ producerStatus: "missing",
1042
+ reason: "producer_row_missing"
1043
+ }
1044
+ }
1045
+
1046
+ if (producer.status !== "handed_off") {
1047
+ return {
1048
+ producerJobId: producerProof.jobId,
1049
+ producerStatus: producer.status,
1050
+ reason: `producer_${producer.status}`
1051
+ }
1052
+ }
1053
+
1054
+ return {
1055
+ producerJobId: producerProof.jobId,
1056
+ producerStatus: producer.status,
1057
+ producerWorkerId: producer.workerId || "",
1058
+ proofWorkerId: producerProof.workerId,
1059
+ reason: producer.handoffId === producerProof.handoffId
1060
+ ? "producer_lease_restarted"
1061
+ : "producer_lease_replaced"
1062
+ }
1063
+ }
1064
+
1027
1065
  /**
1028
1066
  * Replaces the queued owner of a stable schedule key with a new one-off job.
1029
1067
  * A handed-off owner is left running and reported truthfully.
@@ -1810,18 +1848,43 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
1810
1848
  }
1811
1849
 
1812
1850
  /**
1813
- * Runs mark returned to queue.
1851
+ * Returns an active handoff to the queue when its owning worker went away,
1852
+ * unless the recorded runner process is still provably alive and executing the
1853
+ * job. Absence of the worker's socket is not absence of the work: reclaiming a
1854
+ * lease a live runner still holds starts a second attempt while the first keeps
1855
+ * running, and every owned follow-up enqueue from the superseded attempt then
1856
+ * fails its producer fence. A deferred lease is settled by the dead-owner pass
1857
+ * once the runner actually exits.
1814
1858
  * @param {object} args - Options.
1859
+ * @param {boolean} [args.deferLiveOwners] - Whether a provably alive recorded runner keeps the lease instead of being returned to the queue.
1860
+ * @param {string} [args.handoffId] - Handoff lease id.
1861
+ * @param {string} [args.hostname] - Hostname whose local probe answers runner ownership, defaults to this process's hostname.
1815
1862
  * @param {string} args.jobId - Job id.
1816
- * @param {string} args.handoffId - Handoff lease id.
1817
- * @returns {Promise<void>} - Resolves when updated.
1863
+ * @param {import("./owner-liveness.js").BackgroundJobLiveChildIdentity[]} [args.liveChildren] - Runner identities this caller observes as live, for the pid-recycling check.
1864
+ * @param {(pid: number) => boolean} [args.probe] - Process probe for the live-owner check, injectable for tests.
1865
+ * @returns {Promise<boolean>} - Whether the lease was returned to the queue.
1818
1866
  */
1819
- async markReturnedToQueue({jobId, handoffId}) {
1867
+ async markReturnedToQueue({deferLiveOwners = false, handoffId, hostname, jobId, liveChildren = [], probe = isProcessAlive}) {
1820
1868
  await this.ensureReady()
1821
1869
 
1822
- await this._serializedCountMutation(async (db) => {
1870
+ return await this._serializedCountMutation(async (db) => {
1823
1871
  const job = await this._getJobRowById(db, jobId)
1824
- if (!job || job.handoffId !== handoffId || job.status !== "handed_off") return
1872
+
1873
+ if (!job || job.handoffId !== handoffId || job.status !== "handed_off") return false
1874
+
1875
+ if (deferLiveOwners) {
1876
+ const liveness = evaluateBackgroundJobOwnerLiveness({
1877
+ childInstanceId: job.childInstanceId,
1878
+ childPid: job.childPid,
1879
+ currentHostname: hostname ?? os.hostname(),
1880
+ liveChildren,
1881
+ probe,
1882
+ runnerHostname: job.runnerHostname
1883
+ })
1884
+
1885
+ if (liveness.status === "alive") return false
1886
+ }
1887
+
1825
1888
  await this._lockConcurrencyRows(db, this._jobConcurrencyKeys(job))
1826
1889
  const affectedRows = await this._updateAffectedRows(db, {
1827
1890
  tableName: JOBS_TABLE,
@@ -1838,7 +1901,11 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
1838
1901
  if (affectedRows === 1) {
1839
1902
  await this._releaseJobConcurrency(db, job)
1840
1903
  await this._recordStatusTransition(db, "handed_off", "queued")
1904
+
1905
+ return true
1841
1906
  }
1907
+
1908
+ return false
1842
1909
  })
1843
1910
  }
1844
1911
 
@@ -1914,14 +1981,21 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
1914
1981
  * snapshot. The ordinary orphan failure path owns retries, terminal status,
1915
1982
  * count transitions, schedule ownership, and concurrency release.
1916
1983
  * @param {object} args - Options.
1917
- * @param {import("./types.js").BackgroundJobHandoffSnapshot[]} args.handoffs - Exact startup snapshots.
1984
+ * @param {(handoffs: import("./types.js").BackgroundJobDeferredLiveOwnerHandoff[]) => void | Promise<void>} [args.onLiveOwnerDeferred] - Observation hook for leases a live runner still owns, called after the reclaim transaction settles.
1918
1985
  * @param {ReturnType<typeof JSON.parse>} args.error - Orphan reason.
1986
+ * @param {import("./types.js").BackgroundJobHandoffSnapshot[]} args.handoffs - Exact startup snapshots.
1987
+ * @param {string} [args.hostname] - Hostname whose local probe answers runner ownership, defaults to this process's hostname.
1988
+ * @param {boolean} [args.deferLiveOwners] - Whether a lease whose recorded runner process is provably alive stays with that runner instead of being reclaimed. Use this for every absence-based reclaim; leave it off once death is proven.
1989
+ * @param {import("./owner-liveness.js").BackgroundJobLiveChildIdentity[]} [args.liveChildren] - Runner identities this caller observes as live, for the pid-recycling check.
1990
+ * @param {(pid: number) => boolean} [args.probe] - Process probe for the live-owner check, injectable for tests.
1919
1991
  * @returns {Promise<import("./types.js").BackgroundJobRow[]>} - Accepted transitions.
1920
1992
  */
1921
- async markOrphanedHandoffs({handoffs, error}) {
1993
+ async markOrphanedHandoffs({deferLiveOwners = false, error, handoffs, hostname, liveChildren = [], onLiveOwnerDeferred, probe = isProcessAlive}) {
1922
1994
  await this.ensureReady()
1923
1995
 
1924
- return await this._serializedCountMutation(async (db) => {
1996
+ /** @type {import("./types.js").BackgroundJobDeferredLiveOwnerHandoff[]} */
1997
+ const deferredLiveOwnerHandoffs = []
1998
+ const orphanedJobs = await this._serializedCountMutation(async (db) => {
1925
1999
  /** @type {BackgroundJobOrphanSelection[]} */
1926
2000
  const selections = []
1927
2001
 
@@ -1933,6 +2007,27 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
1933
2007
  if (job.workerId !== handoff.workerId) continue
1934
2008
  if (job.handedOffAtMs !== handoff.handedOffAtMs) continue
1935
2009
 
2010
+ if (deferLiveOwners) {
2011
+ const liveness = evaluateBackgroundJobOwnerLiveness({
2012
+ childInstanceId: job.childInstanceId,
2013
+ childPid: job.childPid,
2014
+ currentHostname: hostname ?? os.hostname(),
2015
+ liveChildren,
2016
+ probe,
2017
+ runnerHostname: job.runnerHostname
2018
+ })
2019
+
2020
+ if (liveness.status === "alive") {
2021
+ deferredLiveOwnerHandoffs.push({
2022
+ ...handoff,
2023
+ childPid: liveness.childPid,
2024
+ livenessReason: liveness.reason,
2025
+ runnerHostname: liveness.runnerHostname
2026
+ })
2027
+ continue
2028
+ }
2029
+ }
2030
+
1936
2031
  selections.push({
1937
2032
  conditions: {
1938
2033
  handed_off_at_ms: handoff.handedOffAtMs,
@@ -1948,6 +2043,10 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
1948
2043
 
1949
2044
  return await this._markOrphanSelections({db, selections})
1950
2045
  })
2046
+
2047
+ if (deferredLiveOwnerHandoffs.length > 0 && onLiveOwnerDeferred) await onLiveOwnerDeferred(deferredLiveOwnerHandoffs)
2048
+
2049
+ return orphanedJobs
1951
2050
  }
1952
2051
 
1953
2052
  /**
@@ -188,6 +188,11 @@
188
188
  /**
189
189
  * @typedef {BackgroundJobHandoffSnapshot & {liveness: import("./owner-liveness.js").BackgroundJobOwnerLivenessResult}} BackgroundJobHandoffOwnershipVerdict
190
190
  */
191
+ /**
192
+ * One exact lease an absence-based reclaim left with its runner because the
193
+ * recorded runner process is provably alive.
194
+ * @typedef {BackgroundJobHandoffSnapshot & {childPid: number | null, livenessReason: string, runnerHostname: string | null}} BackgroundJobDeferredLiveOwnerHandoff
195
+ */
191
196
  /**
192
197
  * @typedef {object} BackgroundJobHandoffRequest
193
198
  * @property {string} jobId - Job to claim.
@@ -261,6 +261,7 @@ export default class BackgroundJobsWorker {
261
261
  * @param {number} [args.generationHandshakeTimeoutMs] - Maximum time to wait for generation acknowledgement (default: 4000).
262
262
  * @param {number} [args.reconnectDelayMs] - Delay before reconnecting an established worker connection (default: 1000).
263
263
  * @param {number} [args.jobTimeoutMs] - Override the wall-clock timeout for forked and pooled jobs from `configuration.getBackgroundJobsConfig()`. `0` disables it.
264
+ * @param {number | null} [args.workerShutdownTimeoutMs] - Override the wall-clock cap on one whole shutdown drain from `configuration.getBackgroundJobsConfig().workerShutdownTimeoutMs`. `null` drains without a bound.
264
265
  * @param {number} [args.pooledRunnerSilenceTimeoutMs] - Override the per-child IPC-silence bound for pooled children with in-flight jobs from `configuration.getBackgroundJobsConfig()` (default 120000ms). `0` disables the watchdog.
265
266
  * @param {boolean} [args.closeDatabaseConnectionsOnStop] - Whether stop owns closing the configuration's database pools (default true).
266
267
  * @param {() => void | Promise<void>} [args.onStopped] - Lifecycle hook invoked after the worker finishes stopping.
@@ -269,7 +270,7 @@ export default class BackgroundJobsWorker {
269
270
  * @param {(observation: import("./types.js").PooledChildMemoryObservation) => void | Promise<void>} [args.onPooledRunnerMemoryObservation] - Explicit pooled-child memory observation hook. Every validated observation (periodic while a child has in-flight jobs, plus on demand) is forwarded here, in addition to the worker's compact stderr log line, so an application can route memory diagnostics (e.g. to a bug reporter) without parsing logs.
270
271
  * @param {(payload: import("./types.js").BackgroundJobPayload) => void} [args.onJobAccepted] - Explicit job-acceptance observation hook. Fires the moment the worker accepts a dispatched job payload, before execution mode routing, so an observer can wait for the pickup event instead of polling in-flight sets.
271
272
  */
272
- constructor({configuration, host, port, generationId, workerInstanceId, maxConcurrentForkedJobs, maxConcurrentInlineJobs, pooledRunnerCount, pooledRunnerConcurrency, pooledRunnerMaxJobs, pooledRunnerMaxRssBytes, pooledRunnerMaxLifetimeMs, runnerMaxOldSpaceSizeMb, runnerMallocArenaMax, forkedChildSigkillGraceMs, heartbeatIntervalMs, generationHandshakeTimeoutMs = DEFAULT_GENERATION_HANDSHAKE_TIMEOUT_MS, reconnectDelayMs = 1000, jobTimeoutMs, pooledRunnerSilenceTimeoutMs, closeDatabaseConnectionsOnStop = true, onStopped, onGenerationAccepted, onRetireMessage, onPooledRunnerMemoryObservation, onJobAccepted} = {}) {
273
+ constructor({configuration, host, port, generationId, workerInstanceId, maxConcurrentForkedJobs, maxConcurrentInlineJobs, pooledRunnerCount, pooledRunnerConcurrency, pooledRunnerMaxJobs, pooledRunnerMaxRssBytes, pooledRunnerMaxLifetimeMs, runnerMaxOldSpaceSizeMb, runnerMallocArenaMax, forkedChildSigkillGraceMs, heartbeatIntervalMs, generationHandshakeTimeoutMs = DEFAULT_GENERATION_HANDSHAKE_TIMEOUT_MS, reconnectDelayMs = 1000, jobTimeoutMs, workerShutdownTimeoutMs, pooledRunnerSilenceTimeoutMs, closeDatabaseConnectionsOnStop = true, onStopped, onGenerationAccepted, onRetireMessage, onPooledRunnerMemoryObservation, onJobAccepted} = {}) {
273
274
  /**
274
275
  * Narrows the runtime value to the documented type.
275
276
  * @type {Promise<import("../configuration.js").default>} */
@@ -347,6 +348,20 @@ export default class BackgroundJobsWorker {
347
348
  * @type {number | undefined}
348
349
  */
349
350
  this.jobTimeoutMsOverride = typeof jobTimeoutMs === "number" ? jobTimeoutMs : undefined
351
+ /**
352
+ * Wall-clock cap on one whole shutdown drain, in ms, or `null` to drain
353
+ * without a bound. Constructor override wins; otherwise it is read from
354
+ * `configuration.getBackgroundJobsConfig().workerShutdownTimeoutMs` in
355
+ * `start()`. It bounds the generation-retirement drain as well as a SIGTERM
356
+ * stop, because a retired worker is the process that a deploy cannot wait on
357
+ * and must not have to wait on forever.
358
+ * @type {number | null | undefined}
359
+ */
360
+ this.workerShutdownTimeoutMsOverride = workerShutdownTimeoutMs === undefined
361
+ ? undefined
362
+ : (typeof workerShutdownTimeoutMs === "number" && Number.isInteger(workerShutdownTimeoutMs) && workerShutdownTimeoutMs > 0 ? workerShutdownTimeoutMs : null)
363
+ /** @type {number | null} */
364
+ this.workerShutdownTimeoutMs = this.workerShutdownTimeoutMsOverride ?? null
350
365
  this.shouldStop = false
351
366
  this.isRetiring = false
352
367
  /** @type {Promise<void> | undefined} */
@@ -509,6 +524,7 @@ export default class BackgroundJobsWorker {
509
524
  if (typeof this.pooledRunnerSilenceTimeoutMsOverride !== "number" && typeof poolConfig.pooledRunnerSilenceTimeoutMs === "number") this.pooledRunnerSilenceTimeoutMs = poolConfig.pooledRunnerSilenceTimeoutMs
510
525
  if (typeof this.runnerMaxOldSpaceSizeMbOverride !== "number" && typeof poolConfig.runnerMaxOldSpaceSizeMb === "number") this.runnerMaxOldSpaceSizeMb = poolConfig.runnerMaxOldSpaceSizeMb
511
526
  if (typeof this.runnerMallocArenaMaxOverride !== "number" && typeof poolConfig.runnerMallocArenaMax === "number") this.runnerMallocArenaMax = poolConfig.runnerMallocArenaMax
527
+ if (this.workerShutdownTimeoutMsOverride === undefined) this.workerShutdownTimeoutMs = poolConfig.workerShutdownTimeoutMs ?? null
512
528
 
513
529
  this.statusReporter = new BackgroundJobsStatusReporter({
514
530
  configuration: this.configuration,
@@ -586,10 +602,11 @@ export default class BackgroundJobsWorker {
586
602
  /**
587
603
  * Runs the worker shutdown lifecycle once.
588
604
  * @param {object} [args] - Options.
589
- * @param {number} [args.timeoutMs] - Max wait for in-flight jobs (per phase) in ms.
605
+ * @param {number} [args.timeoutMs] - Max wait for in-flight jobs (per phase) in ms. Defaults to the configured `workerShutdownTimeoutMs`.
590
606
  * @returns {Promise<void>} - Resolves when stopped.
591
607
  */
592
608
  async _stop({timeoutMs} = {}) {
609
+ const drainTimeoutMs = typeof timeoutMs === "number" ? timeoutMs : (this.workerShutdownTimeoutMs ?? undefined)
593
610
  this.shouldStop = true
594
611
  this.isRetiring = true
595
612
  this._stopHeartbeat()
@@ -611,14 +628,14 @@ export default class BackgroundJobsWorker {
611
628
  }
612
629
  }
613
630
 
614
- await this._drainInflight(this.inflightInlineJobs, timeoutMs, {phase: "inline-jobs"})
615
- await this._drainInflight(this.inflightPooledJobs, timeoutMs, {phase: "pooled-jobs"})
631
+ await this._drainInflight(this.inflightInlineJobs, drainTimeoutMs, {phase: "inline-jobs"})
632
+ await this._drainInflight(this.inflightPooledJobs, drainTimeoutMs, {phase: "pooled-jobs"})
616
633
  this._releaseProcessJobHoldersWithGoneChildren()
617
- await this._drainInflight(this.inflightProcessJobs, timeoutMs, {phase: "process-jobs", describeHolders: () => this._describeProcessJobHolders()})
634
+ await this._drainInflight(this.inflightProcessJobs, drainTimeoutMs, {phase: "process-jobs", describeHolders: () => this._describeProcessJobHolders()})
618
635
  await this._terminateProcessChildren()
619
636
  // Give in-flight result reports (now decoupled from job slots) a bounded
620
637
  // chance to land before the socket closes.
621
- await this._drainInflight(this.inflightReports, timeoutMs)
638
+ await this._drainInflight(this.inflightReports, drainTimeoutMs)
622
639
 
623
640
  if (this.jsonSocket) this.jsonSocket.close()
624
641
  if (!this.configuration) return
@@ -646,6 +663,12 @@ export default class BackgroundJobsWorker {
646
663
  /**
647
664
  * Drains accepted generation work while retaining the exact connection and
648
665
  * heartbeat, then performs the final terminating stop.
666
+ *
667
+ * The drain is bounded by the worker's own configured cap, as one deadline
668
+ * shared across every phase rather than one per phase: this is the path a
669
+ * deploy retires a generation through, and the supervisor's retirement budget
670
+ * is deliberately only slightly longer than the cap, so four separate windows
671
+ * would overrun it and hand the wedge straight back to the release.
649
672
  * @returns {Promise<void>} - Resolves after the worker has fully closed.
650
673
  */
651
674
  async _stopAfterGenerationDrain() {
@@ -657,11 +680,15 @@ export default class BackgroundJobsWorker {
657
680
  }
658
681
  }
659
682
 
660
- await this._drainInflight(this.inflightInlineJobs)
661
- await this._drainInflight(this.inflightPooledJobs)
683
+ const deadlineMs = typeof this.workerShutdownTimeoutMs === "number"
684
+ ? Date.now() + this.workerShutdownTimeoutMs
685
+ : undefined
686
+
687
+ await this._drainInflight(this.inflightInlineJobs, this._remainingDrainMs(deadlineMs), {phase: "inline-jobs"})
688
+ await this._drainInflight(this.inflightPooledJobs, this._remainingDrainMs(deadlineMs), {phase: "pooled-jobs"})
662
689
  this._releaseProcessJobHoldersWithGoneChildren()
663
- await this._drainInflight(this.inflightProcessJobs)
664
- await this._drainInflight(this.inflightReports)
690
+ await this._drainInflight(this.inflightProcessJobs, this._remainingDrainMs(deadlineMs), {phase: "process-jobs", describeHolders: () => this._describeProcessJobHolders()})
691
+ await this._drainInflight(this.inflightReports, this._remainingDrainMs(deadlineMs), {phase: "reports"})
665
692
 
666
693
  this.shouldStop = true
667
694
  this._stopHeartbeat()
@@ -682,6 +709,19 @@ export default class BackgroundJobsWorker {
682
709
  })
683
710
  }
684
711
 
712
+ /**
713
+ * Time left inside one shared drain deadline, or `undefined` for an unbounded
714
+ * drain. A deadline that has already passed yields `0`, which lets each later
715
+ * phase report what it is still holding instead of silently waiting.
716
+ * @param {number | undefined} deadlineMs - Absolute deadline, or undefined when unbounded.
717
+ * @returns {number | undefined} - Remaining milliseconds, or undefined for no bound.
718
+ */
719
+ _remainingDrainMs(deadlineMs) {
720
+ if (typeof deadlineMs !== "number") return undefined
721
+
722
+ return Math.max(0, deadlineMs - Date.now())
723
+ }
724
+
685
725
  /**
686
726
  * Closes application resources before framework resources when this worker owns them.
687
727
  * @returns {Promise<void>} - Resolves after every owned close succeeds.
@@ -274,6 +274,14 @@
274
274
  * tuning: set it well above the longest legitimate forked job (build runners,
275
275
  * long imports) so only genuinely-stuck runners are killed. Omit, `null`, or
276
276
  * `<= 0` to disable (default), which preserves the prior unbounded behavior.
277
+ * @property {number | null} [workerShutdownTimeoutMs] - Wall-clock cap, in ms, on
278
+ * one worker's whole shutdown drain, read from `VELOCIOUS_BACKGROUND_JOBS_WORKER_SHUTDOWN_TIMEOUT_MS`
279
+ * when the config omits it. It bounds the generation-retirement drain too, not
280
+ * only a SIGTERM stop: after the cap, whatever is still held is named in a
281
+ * `worker-shutdown-drain-timeout` event and the runner children are terminated
282
+ * (SIGTERM, then SIGKILL), so a retired generation closes itself instead of
283
+ * pinning its release. Omit, `null`, `0`, or the literal `"indefinite"` to drain
284
+ * without a bound (default), which keeps a live multi-hour job protected.
277
285
  * @property {BackgroundJobsRetentionConfiguration} [retention] - Retention/pruning
278
286
  * of terminal job rows. Without pruning the jobs table grows unbounded
279
287
  * (completed rows accumulate forever), which bloats storage and indexes and
@@ -140,6 +140,29 @@ function resolveBeaconUnreachableReportMs(value) {
140
140
  return 30_000
141
141
  }
142
142
 
143
+ /**
144
+ * Resolves the background-jobs worker's own drain cap from a raw configured value.
145
+ * `null` means "drain without a bound and never reap a live runner", which is what a
146
+ * multi-hour build needs; a positive integer is a deadline after which a runner still
147
+ * in flight is terminated instead of orphaned. The literal strings `indefinite` and
148
+ * `0` are the documented opt-out spellings — the production Rollbridge config writes
149
+ * `indefinite` — so they must not be parsed as numbers.
150
+ * @param {unknown} value - Configured value or raw environment value.
151
+ * @returns {number | null} - Milliseconds, or null for an unbounded drain.
152
+ */
153
+ function resolveWorkerShutdownTimeoutMs(value) {
154
+ if (value === null || value === undefined) return null
155
+
156
+ const normalized = typeof value === "string" ? value.trim().toLowerCase() : value
157
+
158
+ if (normalized === "" || normalized === "indefinite" || normalized === "0" || normalized === 0) return null
159
+ if (typeof normalized !== "string" && typeof normalized !== "number") return null
160
+
161
+ const parsed = Number(normalized)
162
+
163
+ return Number.isInteger(parsed) && parsed > 0 ? parsed : null
164
+ }
165
+
143
166
  const DEFAULT_WEBSOCKET_INBOUND_MAX_PENDING_BYTES = 16 * 1024 * 1024
144
167
  const DEFAULT_WEBSOCKET_INBOUND_MAX_PENDING_MESSAGES = 256
145
168
  const DEFAULT_WEBSOCKET_OUTBOUND_MAX_PENDING_BYTES = 16 * 1024 * 1024
@@ -1891,6 +1914,7 @@ export default class VelociousConfiguration {
1891
1914
  const envDispatchStrategy = processEnvironment?.VELOCIOUS_BACKGROUND_JOBS_DISPATCH_STRATEGY
1892
1915
  const envPollIntervalRaw = processEnvironment?.VELOCIOUS_BACKGROUND_JOBS_POLL_INTERVAL_MS
1893
1916
  const envJobTimeoutRaw = processEnvironment?.VELOCIOUS_BACKGROUND_JOBS_JOB_TIMEOUT_MS
1917
+ const envWorkerShutdownTimeoutRaw = processEnvironment?.VELOCIOUS_BACKGROUND_JOBS_WORKER_SHUTDOWN_TIMEOUT_MS
1894
1918
  const envPort = envPortRaw ? Number(envPortRaw) : undefined
1895
1919
  const envMaxConcurrentForked = envMaxConcurrentForkedRaw ? Number(envMaxConcurrentForkedRaw) : undefined
1896
1920
  const envMaxConcurrent = envMaxConcurrentRaw ? Number(envMaxConcurrentRaw) : undefined
@@ -1987,6 +2011,13 @@ export default class VelociousConfiguration {
1987
2011
  const jobTimeoutMs = "jobTimeoutMs" in configured
1988
2012
  ? (typeof configured.jobTimeoutMs === "number" && configured.jobTimeoutMs > 0 ? configured.jobTimeoutMs : null)
1989
2013
  : (typeof envJobTimeout === "number" && Number.isFinite(envJobTimeout) && envJobTimeout > 0 ? envJobTimeout : null)
2014
+ // The worker's own drain cap, read here so every shutdown path shares one
2015
+ // bound. `jobTimeoutMs` bounds one job; this bounds the whole retirement
2016
+ // drain, and it is the only thing that can end a drain whose remaining
2017
+ // work can no longer report its outcome.
2018
+ const workerShutdownTimeoutMs = "workerShutdownTimeoutMs" in configured
2019
+ ? resolveWorkerShutdownTimeoutMs(configured.workerShutdownTimeoutMs)
2020
+ : resolveWorkerShutdownTimeoutMs(envWorkerShutdownTimeoutRaw)
1990
2021
  const configuredRetention = configured.retention && typeof configured.retention === "object" ? configured.retention : {}
1991
2022
  const retention = {
1992
2023
  completedTtlMs: typeof configuredRetention.completedTtlMs === "number" || configuredRetention.completedTtlMs === null
@@ -2019,7 +2050,7 @@ export default class VelociousConfiguration {
2019
2050
  const jobClasses = this.getBackgroundJobClasses()
2020
2051
 
2021
2052
  const endpointPath = typeof configured.endpointPath === "string" ? configured.endpointPath : envEndpointPath
2022
- return {host, port, endpointPath, databaseIdentifier, maxConcurrentForkedJobs, maxConcurrentInlineJobs, mode, pooledRunnerCount, pooledRunnerConcurrency, pooledRunnerMaxJobs, pooledRunnerMaxRssBytes, pooledRunnerMaxLifetimeMs, pooledRunnerSilenceTimeoutMs, runnerMaxOldSpaceSizeMb, runnerMallocArenaMax, dispatchStrategy, pollIntervalMs, drainStoreOperationTimeoutMs, scheduledEnqueueTimeoutMs, queues, queuePriorityAgingUnitMs, jobClasses, jobTimeoutMs, retention, generationId, initialGenerationState, lifecycleSocketPath}
2053
+ return {host, port, endpointPath, databaseIdentifier, maxConcurrentForkedJobs, maxConcurrentInlineJobs, mode, pooledRunnerCount, pooledRunnerConcurrency, pooledRunnerMaxJobs, pooledRunnerMaxRssBytes, pooledRunnerMaxLifetimeMs, pooledRunnerSilenceTimeoutMs, runnerMaxOldSpaceSizeMb, runnerMallocArenaMax, dispatchStrategy, pollIntervalMs, drainStoreOperationTimeoutMs, scheduledEnqueueTimeoutMs, queues, queuePriorityAgingUnitMs, jobClasses, jobTimeoutMs, workerShutdownTimeoutMs, retention, generationId, initialGenerationState, lifecycleSocketPath}
2023
2054
  }
2024
2055
 
2025
2056
  /**
@@ -45,30 +45,6 @@ export async function waitForBackgroundJobsWorkerShutdown({onReady, processObjec
45
45
  }
46
46
  }
47
47
 
48
- /**
49
- * Resolves the shutdown drain timeout from
50
- * `VELOCIOUS_BACKGROUND_JOBS_WORKER_SHUTDOWN_TIMEOUT_MS`:
51
- * - unset / "indefinite" / "0" → indefinite: wait for in-flight jobs to
52
- * finish and never kill a process runner. This is the default so a graceful
53
- * stop (e.g. a deploy) does not interrupt long-running jobs such as builds.
54
- * - positive integer → that many milliseconds, after which any process runner
55
- * still in flight is terminated (SIGTERM, then SIGKILL) instead of orphaned.
56
- *
57
- * When a finite cap is used it must be shorter than the supervisor's
58
- * graceful-stop window so the worker reaps its own children before being
59
- * force-killed.
60
- * @returns {number | undefined} - Timeout in ms, or undefined for indefinite.
61
- */
62
- function resolveShutdownTimeoutMs() {
63
- const raw = (process.env.VELOCIOUS_BACKGROUND_JOBS_WORKER_SHUTDOWN_TIMEOUT_MS || "").trim().toLowerCase()
64
-
65
- if (!raw || raw === "indefinite" || raw === "0") return undefined
66
-
67
- const parsed = Number.parseInt(raw, 10)
68
-
69
- return Number.isInteger(parsed) && parsed > 0 ? parsed : undefined
70
- }
71
-
72
48
  export default class BackgroundJobsWorkerCommand extends BaseCommand {
73
49
  async execute() {
74
50
  // Identify this process in `ps`/`top` instead of a generic "node" entry.
@@ -80,11 +56,12 @@ export default class BackgroundJobsWorkerCommand extends BaseCommand {
80
56
  generationId: typeof args.generation === "string" ? args.generation : undefined
81
57
  })
82
58
 
83
- const timeoutMs = resolveShutdownTimeoutMs()
84
-
59
+ // The drain cap comes from the worker's own configuration
60
+ // (`VELOCIOUS_BACKGROUND_JOBS_WORKER_SHUTDOWN_TIMEOUT_MS`), not from here:
61
+ // a deploy retires this generation through the main's `retire` message, so a
62
+ // bound resolved only by this command could never reach that drain.
85
63
  await waitForBackgroundJobsWorkerShutdown({
86
64
  onReady: () => console.log("Background jobs worker connected"),
87
- timeoutMs,
88
65
  worker
89
66
  })
90
67
  }