velocious 1.0.753 → 1.0.755
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/build/background-jobs/adapter.js +3 -3
- package/build/background-jobs/client.js +6 -5
- package/build/background-jobs/local-adapter.js +3 -3
- package/build/background-jobs/local-store.js +10 -5
- package/build/background-jobs/main.js +87 -8
- package/build/background-jobs/refusal-error.js +87 -0
- package/build/background-jobs/store.js +109 -10
- package/build/background-jobs/types.js +5 -0
- package/build/background-jobs/worker.js +50 -10
- package/build/configuration-types.js +8 -0
- package/build/configuration.js +32 -1
- package/build/environment-handlers/node/cli/commands/background-jobs-worker.js +4 -27
- package/build/src/background-jobs/adapter.d.ts +15 -6
- package/build/src/background-jobs/adapter.d.ts.map +1 -1
- package/build/src/background-jobs/adapter.js +4 -4
- package/build/src/background-jobs/client.d.ts.map +1 -1
- package/build/src/background-jobs/client.js +7 -6
- package/build/src/background-jobs/local-adapter.d.ts +8 -4
- package/build/src/background-jobs/local-adapter.d.ts.map +1 -1
- package/build/src/background-jobs/local-adapter.js +4 -4
- package/build/src/background-jobs/local-store.d.ts +12 -5
- package/build/src/background-jobs/local-store.d.ts.map +1 -1
- package/build/src/background-jobs/local-store.js +10 -6
- package/build/src/background-jobs/main.d.ts +46 -2
- package/build/src/background-jobs/main.d.ts.map +1 -1
- package/build/src/background-jobs/main.js +85 -9
- package/build/src/background-jobs/refusal-error.d.ts +57 -0
- package/build/src/background-jobs/refusal-error.d.ts.map +1 -0
- package/build/src/background-jobs/refusal-error.js +80 -0
- package/build/src/background-jobs/store.d.ts +43 -9
- package/build/src/background-jobs/store.d.ts.map +1 -1
- package/build/src/background-jobs/store.js +99 -11
- package/build/src/background-jobs/types.d.ts +10 -0
- package/build/src/background-jobs/types.d.ts.map +1 -1
- package/build/src/background-jobs/types.js +6 -1
- package/build/src/background-jobs/worker.d.ts +30 -2
- package/build/src/background-jobs/worker.d.ts.map +1 -1
- package/build/src/background-jobs/worker.js +50 -11
- package/build/src/configuration-types.d.ts +19 -0
- package/build/src/configuration-types.d.ts.map +1 -1
- package/build/src/configuration-types.js +9 -1
- package/build/src/configuration.d.ts.map +1 -1
- package/build/src/configuration.js +31 -2
- package/build/src/environment-handlers/node/cli/commands/background-jobs-worker.d.ts.map +1 -1
- package/build/src/environment-handlers/node/cli/commands/background-jobs-worker.js +5 -24
- package/package.json +1 -1
- package/scripts/verify-docker-dev-environment.js +3 -3
- package/src/background-jobs/adapter.js +3 -3
- package/src/background-jobs/client.js +6 -5
- package/src/background-jobs/local-adapter.js +3 -3
- package/src/background-jobs/local-store.js +10 -5
- package/src/background-jobs/main.js +87 -8
- package/src/background-jobs/refusal-error.js +87 -0
- package/src/background-jobs/store.js +109 -10
- package/src/background-jobs/types.js +5 -0
- package/src/background-jobs/worker.js +50 -10
- package/src/configuration-types.js +8 -0
- package/src/configuration.js +32 -1
- package/src/environment-handlers/node/cli/commands/background-jobs-worker.js +4 -27
|
@@ -1019,11 +1019,49 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
|
|
|
1019
1019
|
|
|
1020
1020
|
if (!owned) {
|
|
1021
1021
|
throw VelociousError.safe("Background job producer handoff is no longer owned.", {
|
|
1022
|
-
code: "background-job-producer-handoff-not-owned"
|
|
1022
|
+
code: "background-job-producer-handoff-not-owned",
|
|
1023
|
+
details: this._producerHandoffLossDetails(producer, producerProof)
|
|
1023
1024
|
})
|
|
1024
1025
|
}
|
|
1025
1026
|
}
|
|
1026
1027
|
|
|
1028
|
+
/**
|
|
1029
|
+
* Names, from the row just read inside the fence's own transaction, why the
|
|
1030
|
+
* producer proof no longer matches. The refusal reaches consumers over the
|
|
1031
|
+
* jobs-main socket as text only, so without this the same lifecycle loss is
|
|
1032
|
+
* indistinguishable from an unexplained enqueue failure.
|
|
1033
|
+
* @param {import("./types.js").BackgroundJobRow | null} producer - Producer row, or null when it is gone.
|
|
1034
|
+
* @param {import("./types.js").BackgroundJobProducerProof} producerProof - Proof the producer execution holds.
|
|
1035
|
+
* @returns {Record<string, string | number>} - Measured loss reason and the state it was measured from.
|
|
1036
|
+
*/
|
|
1037
|
+
_producerHandoffLossDetails(producer, producerProof) {
|
|
1038
|
+
if (!producer) {
|
|
1039
|
+
return {
|
|
1040
|
+
producerJobId: producerProof.jobId,
|
|
1041
|
+
producerStatus: "missing",
|
|
1042
|
+
reason: "producer_row_missing"
|
|
1043
|
+
}
|
|
1044
|
+
}
|
|
1045
|
+
|
|
1046
|
+
if (producer.status !== "handed_off") {
|
|
1047
|
+
return {
|
|
1048
|
+
producerJobId: producerProof.jobId,
|
|
1049
|
+
producerStatus: producer.status,
|
|
1050
|
+
reason: `producer_${producer.status}`
|
|
1051
|
+
}
|
|
1052
|
+
}
|
|
1053
|
+
|
|
1054
|
+
return {
|
|
1055
|
+
producerJobId: producerProof.jobId,
|
|
1056
|
+
producerStatus: producer.status,
|
|
1057
|
+
producerWorkerId: producer.workerId || "",
|
|
1058
|
+
proofWorkerId: producerProof.workerId,
|
|
1059
|
+
reason: producer.handoffId === producerProof.handoffId
|
|
1060
|
+
? "producer_lease_restarted"
|
|
1061
|
+
: "producer_lease_replaced"
|
|
1062
|
+
}
|
|
1063
|
+
}
|
|
1064
|
+
|
|
1027
1065
|
/**
|
|
1028
1066
|
* Replaces the queued owner of a stable schedule key with a new one-off job.
|
|
1029
1067
|
* A handed-off owner is left running and reported truthfully.
|
|
@@ -1810,18 +1848,43 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
|
|
|
1810
1848
|
}
|
|
1811
1849
|
|
|
1812
1850
|
/**
|
|
1813
|
-
*
|
|
1851
|
+
* Returns an active handoff to the queue when its owning worker went away,
|
|
1852
|
+
* unless the recorded runner process is still provably alive and executing the
|
|
1853
|
+
* job. Absence of the worker's socket is not absence of the work: reclaiming a
|
|
1854
|
+
* lease a live runner still holds starts a second attempt while the first keeps
|
|
1855
|
+
* running, and every owned follow-up enqueue from the superseded attempt then
|
|
1856
|
+
* fails its producer fence. A deferred lease is settled by the dead-owner pass
|
|
1857
|
+
* once the runner actually exits.
|
|
1814
1858
|
* @param {object} args - Options.
|
|
1859
|
+
* @param {boolean} [args.deferLiveOwners] - Whether a provably alive recorded runner keeps the lease instead of being returned to the queue.
|
|
1860
|
+
* @param {string} [args.handoffId] - Handoff lease id.
|
|
1861
|
+
* @param {string} [args.hostname] - Hostname whose local probe answers runner ownership, defaults to this process's hostname.
|
|
1815
1862
|
* @param {string} args.jobId - Job id.
|
|
1816
|
-
* @param {
|
|
1817
|
-
* @
|
|
1863
|
+
* @param {import("./owner-liveness.js").BackgroundJobLiveChildIdentity[]} [args.liveChildren] - Runner identities this caller observes as live, for the pid-recycling check.
|
|
1864
|
+
* @param {(pid: number) => boolean} [args.probe] - Process probe for the live-owner check, injectable for tests.
|
|
1865
|
+
* @returns {Promise<boolean>} - Whether the lease was returned to the queue.
|
|
1818
1866
|
*/
|
|
1819
|
-
async markReturnedToQueue({jobId,
|
|
1867
|
+
async markReturnedToQueue({deferLiveOwners = false, handoffId, hostname, jobId, liveChildren = [], probe = isProcessAlive}) {
|
|
1820
1868
|
await this.ensureReady()
|
|
1821
1869
|
|
|
1822
|
-
await this._serializedCountMutation(async (db) => {
|
|
1870
|
+
return await this._serializedCountMutation(async (db) => {
|
|
1823
1871
|
const job = await this._getJobRowById(db, jobId)
|
|
1824
|
-
|
|
1872
|
+
|
|
1873
|
+
if (!job || job.handoffId !== handoffId || job.status !== "handed_off") return false
|
|
1874
|
+
|
|
1875
|
+
if (deferLiveOwners) {
|
|
1876
|
+
const liveness = evaluateBackgroundJobOwnerLiveness({
|
|
1877
|
+
childInstanceId: job.childInstanceId,
|
|
1878
|
+
childPid: job.childPid,
|
|
1879
|
+
currentHostname: hostname ?? os.hostname(),
|
|
1880
|
+
liveChildren,
|
|
1881
|
+
probe,
|
|
1882
|
+
runnerHostname: job.runnerHostname
|
|
1883
|
+
})
|
|
1884
|
+
|
|
1885
|
+
if (liveness.status === "alive") return false
|
|
1886
|
+
}
|
|
1887
|
+
|
|
1825
1888
|
await this._lockConcurrencyRows(db, this._jobConcurrencyKeys(job))
|
|
1826
1889
|
const affectedRows = await this._updateAffectedRows(db, {
|
|
1827
1890
|
tableName: JOBS_TABLE,
|
|
@@ -1838,7 +1901,11 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
|
|
|
1838
1901
|
if (affectedRows === 1) {
|
|
1839
1902
|
await this._releaseJobConcurrency(db, job)
|
|
1840
1903
|
await this._recordStatusTransition(db, "handed_off", "queued")
|
|
1904
|
+
|
|
1905
|
+
return true
|
|
1841
1906
|
}
|
|
1907
|
+
|
|
1908
|
+
return false
|
|
1842
1909
|
})
|
|
1843
1910
|
}
|
|
1844
1911
|
|
|
@@ -1914,14 +1981,21 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
|
|
|
1914
1981
|
* snapshot. The ordinary orphan failure path owns retries, terminal status,
|
|
1915
1982
|
* count transitions, schedule ownership, and concurrency release.
|
|
1916
1983
|
* @param {object} args - Options.
|
|
1917
|
-
* @param {import("./types.js").
|
|
1984
|
+
* @param {(handoffs: import("./types.js").BackgroundJobDeferredLiveOwnerHandoff[]) => void | Promise<void>} [args.onLiveOwnerDeferred] - Observation hook for leases a live runner still owns, called after the reclaim transaction settles.
|
|
1918
1985
|
* @param {ReturnType<typeof JSON.parse>} args.error - Orphan reason.
|
|
1986
|
+
* @param {import("./types.js").BackgroundJobHandoffSnapshot[]} args.handoffs - Exact startup snapshots.
|
|
1987
|
+
* @param {string} [args.hostname] - Hostname whose local probe answers runner ownership, defaults to this process's hostname.
|
|
1988
|
+
* @param {boolean} [args.deferLiveOwners] - Whether a lease whose recorded runner process is provably alive stays with that runner instead of being reclaimed. Use this for every absence-based reclaim; leave it off once death is proven.
|
|
1989
|
+
* @param {import("./owner-liveness.js").BackgroundJobLiveChildIdentity[]} [args.liveChildren] - Runner identities this caller observes as live, for the pid-recycling check.
|
|
1990
|
+
* @param {(pid: number) => boolean} [args.probe] - Process probe for the live-owner check, injectable for tests.
|
|
1919
1991
|
* @returns {Promise<import("./types.js").BackgroundJobRow[]>} - Accepted transitions.
|
|
1920
1992
|
*/
|
|
1921
|
-
async markOrphanedHandoffs({handoffs,
|
|
1993
|
+
async markOrphanedHandoffs({deferLiveOwners = false, error, handoffs, hostname, liveChildren = [], onLiveOwnerDeferred, probe = isProcessAlive}) {
|
|
1922
1994
|
await this.ensureReady()
|
|
1923
1995
|
|
|
1924
|
-
|
|
1996
|
+
/** @type {import("./types.js").BackgroundJobDeferredLiveOwnerHandoff[]} */
|
|
1997
|
+
const deferredLiveOwnerHandoffs = []
|
|
1998
|
+
const orphanedJobs = await this._serializedCountMutation(async (db) => {
|
|
1925
1999
|
/** @type {BackgroundJobOrphanSelection[]} */
|
|
1926
2000
|
const selections = []
|
|
1927
2001
|
|
|
@@ -1933,6 +2007,27 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
|
|
|
1933
2007
|
if (job.workerId !== handoff.workerId) continue
|
|
1934
2008
|
if (job.handedOffAtMs !== handoff.handedOffAtMs) continue
|
|
1935
2009
|
|
|
2010
|
+
if (deferLiveOwners) {
|
|
2011
|
+
const liveness = evaluateBackgroundJobOwnerLiveness({
|
|
2012
|
+
childInstanceId: job.childInstanceId,
|
|
2013
|
+
childPid: job.childPid,
|
|
2014
|
+
currentHostname: hostname ?? os.hostname(),
|
|
2015
|
+
liveChildren,
|
|
2016
|
+
probe,
|
|
2017
|
+
runnerHostname: job.runnerHostname
|
|
2018
|
+
})
|
|
2019
|
+
|
|
2020
|
+
if (liveness.status === "alive") {
|
|
2021
|
+
deferredLiveOwnerHandoffs.push({
|
|
2022
|
+
...handoff,
|
|
2023
|
+
childPid: liveness.childPid,
|
|
2024
|
+
livenessReason: liveness.reason,
|
|
2025
|
+
runnerHostname: liveness.runnerHostname
|
|
2026
|
+
})
|
|
2027
|
+
continue
|
|
2028
|
+
}
|
|
2029
|
+
}
|
|
2030
|
+
|
|
1936
2031
|
selections.push({
|
|
1937
2032
|
conditions: {
|
|
1938
2033
|
handed_off_at_ms: handoff.handedOffAtMs,
|
|
@@ -1948,6 +2043,10 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
|
|
|
1948
2043
|
|
|
1949
2044
|
return await this._markOrphanSelections({db, selections})
|
|
1950
2045
|
})
|
|
2046
|
+
|
|
2047
|
+
if (deferredLiveOwnerHandoffs.length > 0 && onLiveOwnerDeferred) await onLiveOwnerDeferred(deferredLiveOwnerHandoffs)
|
|
2048
|
+
|
|
2049
|
+
return orphanedJobs
|
|
1951
2050
|
}
|
|
1952
2051
|
|
|
1953
2052
|
/**
|
|
@@ -188,6 +188,11 @@
|
|
|
188
188
|
/**
|
|
189
189
|
* @typedef {BackgroundJobHandoffSnapshot & {liveness: import("./owner-liveness.js").BackgroundJobOwnerLivenessResult}} BackgroundJobHandoffOwnershipVerdict
|
|
190
190
|
*/
|
|
191
|
+
/**
|
|
192
|
+
* One exact lease an absence-based reclaim left with its runner because the
|
|
193
|
+
* recorded runner process is provably alive.
|
|
194
|
+
* @typedef {BackgroundJobHandoffSnapshot & {childPid: number | null, livenessReason: string, runnerHostname: string | null}} BackgroundJobDeferredLiveOwnerHandoff
|
|
195
|
+
*/
|
|
191
196
|
/**
|
|
192
197
|
* @typedef {object} BackgroundJobHandoffRequest
|
|
193
198
|
* @property {string} jobId - Job to claim.
|
|
@@ -261,6 +261,7 @@ export default class BackgroundJobsWorker {
|
|
|
261
261
|
* @param {number} [args.generationHandshakeTimeoutMs] - Maximum time to wait for generation acknowledgement (default: 4000).
|
|
262
262
|
* @param {number} [args.reconnectDelayMs] - Delay before reconnecting an established worker connection (default: 1000).
|
|
263
263
|
* @param {number} [args.jobTimeoutMs] - Override the wall-clock timeout for forked and pooled jobs from `configuration.getBackgroundJobsConfig()`. `0` disables it.
|
|
264
|
+
* @param {number | null} [args.workerShutdownTimeoutMs] - Override the wall-clock cap on one whole shutdown drain from `configuration.getBackgroundJobsConfig().workerShutdownTimeoutMs`. `null` drains without a bound.
|
|
264
265
|
* @param {number} [args.pooledRunnerSilenceTimeoutMs] - Override the per-child IPC-silence bound for pooled children with in-flight jobs from `configuration.getBackgroundJobsConfig()` (default 120000ms). `0` disables the watchdog.
|
|
265
266
|
* @param {boolean} [args.closeDatabaseConnectionsOnStop] - Whether stop owns closing the configuration's database pools (default true).
|
|
266
267
|
* @param {() => void | Promise<void>} [args.onStopped] - Lifecycle hook invoked after the worker finishes stopping.
|
|
@@ -269,7 +270,7 @@ export default class BackgroundJobsWorker {
|
|
|
269
270
|
* @param {(observation: import("./types.js").PooledChildMemoryObservation) => void | Promise<void>} [args.onPooledRunnerMemoryObservation] - Explicit pooled-child memory observation hook. Every validated observation (periodic while a child has in-flight jobs, plus on demand) is forwarded here, in addition to the worker's compact stderr log line, so an application can route memory diagnostics (e.g. to a bug reporter) without parsing logs.
|
|
270
271
|
* @param {(payload: import("./types.js").BackgroundJobPayload) => void} [args.onJobAccepted] - Explicit job-acceptance observation hook. Fires the moment the worker accepts a dispatched job payload, before execution mode routing, so an observer can wait for the pickup event instead of polling in-flight sets.
|
|
271
272
|
*/
|
|
272
|
-
constructor({configuration, host, port, generationId, workerInstanceId, maxConcurrentForkedJobs, maxConcurrentInlineJobs, pooledRunnerCount, pooledRunnerConcurrency, pooledRunnerMaxJobs, pooledRunnerMaxRssBytes, pooledRunnerMaxLifetimeMs, runnerMaxOldSpaceSizeMb, runnerMallocArenaMax, forkedChildSigkillGraceMs, heartbeatIntervalMs, generationHandshakeTimeoutMs = DEFAULT_GENERATION_HANDSHAKE_TIMEOUT_MS, reconnectDelayMs = 1000, jobTimeoutMs, pooledRunnerSilenceTimeoutMs, closeDatabaseConnectionsOnStop = true, onStopped, onGenerationAccepted, onRetireMessage, onPooledRunnerMemoryObservation, onJobAccepted} = {}) {
|
|
273
|
+
constructor({configuration, host, port, generationId, workerInstanceId, maxConcurrentForkedJobs, maxConcurrentInlineJobs, pooledRunnerCount, pooledRunnerConcurrency, pooledRunnerMaxJobs, pooledRunnerMaxRssBytes, pooledRunnerMaxLifetimeMs, runnerMaxOldSpaceSizeMb, runnerMallocArenaMax, forkedChildSigkillGraceMs, heartbeatIntervalMs, generationHandshakeTimeoutMs = DEFAULT_GENERATION_HANDSHAKE_TIMEOUT_MS, reconnectDelayMs = 1000, jobTimeoutMs, workerShutdownTimeoutMs, pooledRunnerSilenceTimeoutMs, closeDatabaseConnectionsOnStop = true, onStopped, onGenerationAccepted, onRetireMessage, onPooledRunnerMemoryObservation, onJobAccepted} = {}) {
|
|
273
274
|
/**
|
|
274
275
|
* Narrows the runtime value to the documented type.
|
|
275
276
|
* @type {Promise<import("../configuration.js").default>} */
|
|
@@ -347,6 +348,20 @@ export default class BackgroundJobsWorker {
|
|
|
347
348
|
* @type {number | undefined}
|
|
348
349
|
*/
|
|
349
350
|
this.jobTimeoutMsOverride = typeof jobTimeoutMs === "number" ? jobTimeoutMs : undefined
|
|
351
|
+
/**
|
|
352
|
+
* Wall-clock cap on one whole shutdown drain, in ms, or `null` to drain
|
|
353
|
+
* without a bound. Constructor override wins; otherwise it is read from
|
|
354
|
+
* `configuration.getBackgroundJobsConfig().workerShutdownTimeoutMs` in
|
|
355
|
+
* `start()`. It bounds the generation-retirement drain as well as a SIGTERM
|
|
356
|
+
* stop, because a retired worker is the process that a deploy cannot wait on
|
|
357
|
+
* and must not have to wait on forever.
|
|
358
|
+
* @type {number | null | undefined}
|
|
359
|
+
*/
|
|
360
|
+
this.workerShutdownTimeoutMsOverride = workerShutdownTimeoutMs === undefined
|
|
361
|
+
? undefined
|
|
362
|
+
: (typeof workerShutdownTimeoutMs === "number" && Number.isInteger(workerShutdownTimeoutMs) && workerShutdownTimeoutMs > 0 ? workerShutdownTimeoutMs : null)
|
|
363
|
+
/** @type {number | null} */
|
|
364
|
+
this.workerShutdownTimeoutMs = this.workerShutdownTimeoutMsOverride ?? null
|
|
350
365
|
this.shouldStop = false
|
|
351
366
|
this.isRetiring = false
|
|
352
367
|
/** @type {Promise<void> | undefined} */
|
|
@@ -509,6 +524,7 @@ export default class BackgroundJobsWorker {
|
|
|
509
524
|
if (typeof this.pooledRunnerSilenceTimeoutMsOverride !== "number" && typeof poolConfig.pooledRunnerSilenceTimeoutMs === "number") this.pooledRunnerSilenceTimeoutMs = poolConfig.pooledRunnerSilenceTimeoutMs
|
|
510
525
|
if (typeof this.runnerMaxOldSpaceSizeMbOverride !== "number" && typeof poolConfig.runnerMaxOldSpaceSizeMb === "number") this.runnerMaxOldSpaceSizeMb = poolConfig.runnerMaxOldSpaceSizeMb
|
|
511
526
|
if (typeof this.runnerMallocArenaMaxOverride !== "number" && typeof poolConfig.runnerMallocArenaMax === "number") this.runnerMallocArenaMax = poolConfig.runnerMallocArenaMax
|
|
527
|
+
if (this.workerShutdownTimeoutMsOverride === undefined) this.workerShutdownTimeoutMs = poolConfig.workerShutdownTimeoutMs ?? null
|
|
512
528
|
|
|
513
529
|
this.statusReporter = new BackgroundJobsStatusReporter({
|
|
514
530
|
configuration: this.configuration,
|
|
@@ -586,10 +602,11 @@ export default class BackgroundJobsWorker {
|
|
|
586
602
|
/**
|
|
587
603
|
* Runs the worker shutdown lifecycle once.
|
|
588
604
|
* @param {object} [args] - Options.
|
|
589
|
-
* @param {number} [args.timeoutMs] - Max wait for in-flight jobs (per phase) in ms.
|
|
605
|
+
* @param {number} [args.timeoutMs] - Max wait for in-flight jobs (per phase) in ms. Defaults to the configured `workerShutdownTimeoutMs`.
|
|
590
606
|
* @returns {Promise<void>} - Resolves when stopped.
|
|
591
607
|
*/
|
|
592
608
|
async _stop({timeoutMs} = {}) {
|
|
609
|
+
const drainTimeoutMs = typeof timeoutMs === "number" ? timeoutMs : (this.workerShutdownTimeoutMs ?? undefined)
|
|
593
610
|
this.shouldStop = true
|
|
594
611
|
this.isRetiring = true
|
|
595
612
|
this._stopHeartbeat()
|
|
@@ -611,14 +628,14 @@ export default class BackgroundJobsWorker {
|
|
|
611
628
|
}
|
|
612
629
|
}
|
|
613
630
|
|
|
614
|
-
await this._drainInflight(this.inflightInlineJobs,
|
|
615
|
-
await this._drainInflight(this.inflightPooledJobs,
|
|
631
|
+
await this._drainInflight(this.inflightInlineJobs, drainTimeoutMs, {phase: "inline-jobs"})
|
|
632
|
+
await this._drainInflight(this.inflightPooledJobs, drainTimeoutMs, {phase: "pooled-jobs"})
|
|
616
633
|
this._releaseProcessJobHoldersWithGoneChildren()
|
|
617
|
-
await this._drainInflight(this.inflightProcessJobs,
|
|
634
|
+
await this._drainInflight(this.inflightProcessJobs, drainTimeoutMs, {phase: "process-jobs", describeHolders: () => this._describeProcessJobHolders()})
|
|
618
635
|
await this._terminateProcessChildren()
|
|
619
636
|
// Give in-flight result reports (now decoupled from job slots) a bounded
|
|
620
637
|
// chance to land before the socket closes.
|
|
621
|
-
await this._drainInflight(this.inflightReports,
|
|
638
|
+
await this._drainInflight(this.inflightReports, drainTimeoutMs)
|
|
622
639
|
|
|
623
640
|
if (this.jsonSocket) this.jsonSocket.close()
|
|
624
641
|
if (!this.configuration) return
|
|
@@ -646,6 +663,12 @@ export default class BackgroundJobsWorker {
|
|
|
646
663
|
/**
|
|
647
664
|
* Drains accepted generation work while retaining the exact connection and
|
|
648
665
|
* heartbeat, then performs the final terminating stop.
|
|
666
|
+
*
|
|
667
|
+
* The drain is bounded by the worker's own configured cap, as one deadline
|
|
668
|
+
* shared across every phase rather than one per phase: this is the path a
|
|
669
|
+
* deploy retires a generation through, and the supervisor's retirement budget
|
|
670
|
+
* is deliberately only slightly longer than the cap, so four separate windows
|
|
671
|
+
* would overrun it and hand the wedge straight back to the release.
|
|
649
672
|
* @returns {Promise<void>} - Resolves after the worker has fully closed.
|
|
650
673
|
*/
|
|
651
674
|
async _stopAfterGenerationDrain() {
|
|
@@ -657,11 +680,15 @@ export default class BackgroundJobsWorker {
|
|
|
657
680
|
}
|
|
658
681
|
}
|
|
659
682
|
|
|
660
|
-
|
|
661
|
-
|
|
683
|
+
const deadlineMs = typeof this.workerShutdownTimeoutMs === "number"
|
|
684
|
+
? Date.now() + this.workerShutdownTimeoutMs
|
|
685
|
+
: undefined
|
|
686
|
+
|
|
687
|
+
await this._drainInflight(this.inflightInlineJobs, this._remainingDrainMs(deadlineMs), {phase: "inline-jobs"})
|
|
688
|
+
await this._drainInflight(this.inflightPooledJobs, this._remainingDrainMs(deadlineMs), {phase: "pooled-jobs"})
|
|
662
689
|
this._releaseProcessJobHoldersWithGoneChildren()
|
|
663
|
-
await this._drainInflight(this.inflightProcessJobs)
|
|
664
|
-
await this._drainInflight(this.inflightReports)
|
|
690
|
+
await this._drainInflight(this.inflightProcessJobs, this._remainingDrainMs(deadlineMs), {phase: "process-jobs", describeHolders: () => this._describeProcessJobHolders()})
|
|
691
|
+
await this._drainInflight(this.inflightReports, this._remainingDrainMs(deadlineMs), {phase: "reports"})
|
|
665
692
|
|
|
666
693
|
this.shouldStop = true
|
|
667
694
|
this._stopHeartbeat()
|
|
@@ -682,6 +709,19 @@ export default class BackgroundJobsWorker {
|
|
|
682
709
|
})
|
|
683
710
|
}
|
|
684
711
|
|
|
712
|
+
/**
|
|
713
|
+
* Time left inside one shared drain deadline, or `undefined` for an unbounded
|
|
714
|
+
* drain. A deadline that has already passed yields `0`, which lets each later
|
|
715
|
+
* phase report what it is still holding instead of silently waiting.
|
|
716
|
+
* @param {number | undefined} deadlineMs - Absolute deadline, or undefined when unbounded.
|
|
717
|
+
* @returns {number | undefined} - Remaining milliseconds, or undefined for no bound.
|
|
718
|
+
*/
|
|
719
|
+
_remainingDrainMs(deadlineMs) {
|
|
720
|
+
if (typeof deadlineMs !== "number") return undefined
|
|
721
|
+
|
|
722
|
+
return Math.max(0, deadlineMs - Date.now())
|
|
723
|
+
}
|
|
724
|
+
|
|
685
725
|
/**
|
|
686
726
|
* Closes application resources before framework resources when this worker owns them.
|
|
687
727
|
* @returns {Promise<void>} - Resolves after every owned close succeeds.
|
|
@@ -274,6 +274,14 @@
|
|
|
274
274
|
* tuning: set it well above the longest legitimate forked job (build runners,
|
|
275
275
|
* long imports) so only genuinely-stuck runners are killed. Omit, `null`, or
|
|
276
276
|
* `<= 0` to disable (default), which preserves the prior unbounded behavior.
|
|
277
|
+
* @property {number | null} [workerShutdownTimeoutMs] - Wall-clock cap, in ms, on
|
|
278
|
+
* one worker's whole shutdown drain, read from `VELOCIOUS_BACKGROUND_JOBS_WORKER_SHUTDOWN_TIMEOUT_MS`
|
|
279
|
+
* when the config omits it. It bounds the generation-retirement drain too, not
|
|
280
|
+
* only a SIGTERM stop: after the cap, whatever is still held is named in a
|
|
281
|
+
* `worker-shutdown-drain-timeout` event and the runner children are terminated
|
|
282
|
+
* (SIGTERM, then SIGKILL), so a retired generation closes itself instead of
|
|
283
|
+
* pinning its release. Omit, `null`, `0`, or the literal `"indefinite"` to drain
|
|
284
|
+
* without a bound (default), which keeps a live multi-hour job protected.
|
|
277
285
|
* @property {BackgroundJobsRetentionConfiguration} [retention] - Retention/pruning
|
|
278
286
|
* of terminal job rows. Without pruning the jobs table grows unbounded
|
|
279
287
|
* (completed rows accumulate forever), which bloats storage and indexes and
|
package/src/configuration.js
CHANGED
|
@@ -140,6 +140,29 @@ function resolveBeaconUnreachableReportMs(value) {
|
|
|
140
140
|
return 30_000
|
|
141
141
|
}
|
|
142
142
|
|
|
143
|
+
/**
|
|
144
|
+
* Resolves the background-jobs worker's own drain cap from a raw configured value.
|
|
145
|
+
* `null` means "drain without a bound and never reap a live runner", which is what a
|
|
146
|
+
* multi-hour build needs; a positive integer is a deadline after which a runner still
|
|
147
|
+
* in flight is terminated instead of orphaned. The literal strings `indefinite` and
|
|
148
|
+
* `0` are the documented opt-out spellings — the production Rollbridge config writes
|
|
149
|
+
* `indefinite` — so they must not be parsed as numbers.
|
|
150
|
+
* @param {unknown} value - Configured value or raw environment value.
|
|
151
|
+
* @returns {number | null} - Milliseconds, or null for an unbounded drain.
|
|
152
|
+
*/
|
|
153
|
+
function resolveWorkerShutdownTimeoutMs(value) {
|
|
154
|
+
if (value === null || value === undefined) return null
|
|
155
|
+
|
|
156
|
+
const normalized = typeof value === "string" ? value.trim().toLowerCase() : value
|
|
157
|
+
|
|
158
|
+
if (normalized === "" || normalized === "indefinite" || normalized === "0" || normalized === 0) return null
|
|
159
|
+
if (typeof normalized !== "string" && typeof normalized !== "number") return null
|
|
160
|
+
|
|
161
|
+
const parsed = Number(normalized)
|
|
162
|
+
|
|
163
|
+
return Number.isInteger(parsed) && parsed > 0 ? parsed : null
|
|
164
|
+
}
|
|
165
|
+
|
|
143
166
|
const DEFAULT_WEBSOCKET_INBOUND_MAX_PENDING_BYTES = 16 * 1024 * 1024
|
|
144
167
|
const DEFAULT_WEBSOCKET_INBOUND_MAX_PENDING_MESSAGES = 256
|
|
145
168
|
const DEFAULT_WEBSOCKET_OUTBOUND_MAX_PENDING_BYTES = 16 * 1024 * 1024
|
|
@@ -1891,6 +1914,7 @@ export default class VelociousConfiguration {
|
|
|
1891
1914
|
const envDispatchStrategy = processEnvironment?.VELOCIOUS_BACKGROUND_JOBS_DISPATCH_STRATEGY
|
|
1892
1915
|
const envPollIntervalRaw = processEnvironment?.VELOCIOUS_BACKGROUND_JOBS_POLL_INTERVAL_MS
|
|
1893
1916
|
const envJobTimeoutRaw = processEnvironment?.VELOCIOUS_BACKGROUND_JOBS_JOB_TIMEOUT_MS
|
|
1917
|
+
const envWorkerShutdownTimeoutRaw = processEnvironment?.VELOCIOUS_BACKGROUND_JOBS_WORKER_SHUTDOWN_TIMEOUT_MS
|
|
1894
1918
|
const envPort = envPortRaw ? Number(envPortRaw) : undefined
|
|
1895
1919
|
const envMaxConcurrentForked = envMaxConcurrentForkedRaw ? Number(envMaxConcurrentForkedRaw) : undefined
|
|
1896
1920
|
const envMaxConcurrent = envMaxConcurrentRaw ? Number(envMaxConcurrentRaw) : undefined
|
|
@@ -1987,6 +2011,13 @@ export default class VelociousConfiguration {
|
|
|
1987
2011
|
const jobTimeoutMs = "jobTimeoutMs" in configured
|
|
1988
2012
|
? (typeof configured.jobTimeoutMs === "number" && configured.jobTimeoutMs > 0 ? configured.jobTimeoutMs : null)
|
|
1989
2013
|
: (typeof envJobTimeout === "number" && Number.isFinite(envJobTimeout) && envJobTimeout > 0 ? envJobTimeout : null)
|
|
2014
|
+
// The worker's own drain cap, read here so every shutdown path shares one
|
|
2015
|
+
// bound. `jobTimeoutMs` bounds one job; this bounds the whole retirement
|
|
2016
|
+
// drain, and it is the only thing that can end a drain whose remaining
|
|
2017
|
+
// work can no longer report its outcome.
|
|
2018
|
+
const workerShutdownTimeoutMs = "workerShutdownTimeoutMs" in configured
|
|
2019
|
+
? resolveWorkerShutdownTimeoutMs(configured.workerShutdownTimeoutMs)
|
|
2020
|
+
: resolveWorkerShutdownTimeoutMs(envWorkerShutdownTimeoutRaw)
|
|
1990
2021
|
const configuredRetention = configured.retention && typeof configured.retention === "object" ? configured.retention : {}
|
|
1991
2022
|
const retention = {
|
|
1992
2023
|
completedTtlMs: typeof configuredRetention.completedTtlMs === "number" || configuredRetention.completedTtlMs === null
|
|
@@ -2019,7 +2050,7 @@ export default class VelociousConfiguration {
|
|
|
2019
2050
|
const jobClasses = this.getBackgroundJobClasses()
|
|
2020
2051
|
|
|
2021
2052
|
const endpointPath = typeof configured.endpointPath === "string" ? configured.endpointPath : envEndpointPath
|
|
2022
|
-
return {host, port, endpointPath, databaseIdentifier, maxConcurrentForkedJobs, maxConcurrentInlineJobs, mode, pooledRunnerCount, pooledRunnerConcurrency, pooledRunnerMaxJobs, pooledRunnerMaxRssBytes, pooledRunnerMaxLifetimeMs, pooledRunnerSilenceTimeoutMs, runnerMaxOldSpaceSizeMb, runnerMallocArenaMax, dispatchStrategy, pollIntervalMs, drainStoreOperationTimeoutMs, scheduledEnqueueTimeoutMs, queues, queuePriorityAgingUnitMs, jobClasses, jobTimeoutMs, retention, generationId, initialGenerationState, lifecycleSocketPath}
|
|
2053
|
+
return {host, port, endpointPath, databaseIdentifier, maxConcurrentForkedJobs, maxConcurrentInlineJobs, mode, pooledRunnerCount, pooledRunnerConcurrency, pooledRunnerMaxJobs, pooledRunnerMaxRssBytes, pooledRunnerMaxLifetimeMs, pooledRunnerSilenceTimeoutMs, runnerMaxOldSpaceSizeMb, runnerMallocArenaMax, dispatchStrategy, pollIntervalMs, drainStoreOperationTimeoutMs, scheduledEnqueueTimeoutMs, queues, queuePriorityAgingUnitMs, jobClasses, jobTimeoutMs, workerShutdownTimeoutMs, retention, generationId, initialGenerationState, lifecycleSocketPath}
|
|
2023
2054
|
}
|
|
2024
2055
|
|
|
2025
2056
|
/**
|
|
@@ -45,30 +45,6 @@ export async function waitForBackgroundJobsWorkerShutdown({onReady, processObjec
|
|
|
45
45
|
}
|
|
46
46
|
}
|
|
47
47
|
|
|
48
|
-
/**
|
|
49
|
-
* Resolves the shutdown drain timeout from
|
|
50
|
-
* `VELOCIOUS_BACKGROUND_JOBS_WORKER_SHUTDOWN_TIMEOUT_MS`:
|
|
51
|
-
* - unset / "indefinite" / "0" → indefinite: wait for in-flight jobs to
|
|
52
|
-
* finish and never kill a process runner. This is the default so a graceful
|
|
53
|
-
* stop (e.g. a deploy) does not interrupt long-running jobs such as builds.
|
|
54
|
-
* - positive integer → that many milliseconds, after which any process runner
|
|
55
|
-
* still in flight is terminated (SIGTERM, then SIGKILL) instead of orphaned.
|
|
56
|
-
*
|
|
57
|
-
* When a finite cap is used it must be shorter than the supervisor's
|
|
58
|
-
* graceful-stop window so the worker reaps its own children before being
|
|
59
|
-
* force-killed.
|
|
60
|
-
* @returns {number | undefined} - Timeout in ms, or undefined for indefinite.
|
|
61
|
-
*/
|
|
62
|
-
function resolveShutdownTimeoutMs() {
|
|
63
|
-
const raw = (process.env.VELOCIOUS_BACKGROUND_JOBS_WORKER_SHUTDOWN_TIMEOUT_MS || "").trim().toLowerCase()
|
|
64
|
-
|
|
65
|
-
if (!raw || raw === "indefinite" || raw === "0") return undefined
|
|
66
|
-
|
|
67
|
-
const parsed = Number.parseInt(raw, 10)
|
|
68
|
-
|
|
69
|
-
return Number.isInteger(parsed) && parsed > 0 ? parsed : undefined
|
|
70
|
-
}
|
|
71
|
-
|
|
72
48
|
export default class BackgroundJobsWorkerCommand extends BaseCommand {
|
|
73
49
|
async execute() {
|
|
74
50
|
// Identify this process in `ps`/`top` instead of a generic "node" entry.
|
|
@@ -80,11 +56,12 @@ export default class BackgroundJobsWorkerCommand extends BaseCommand {
|
|
|
80
56
|
generationId: typeof args.generation === "string" ? args.generation : undefined
|
|
81
57
|
})
|
|
82
58
|
|
|
83
|
-
|
|
84
|
-
|
|
59
|
+
// The drain cap comes from the worker's own configuration
|
|
60
|
+
// (`VELOCIOUS_BACKGROUND_JOBS_WORKER_SHUTDOWN_TIMEOUT_MS`), not from here:
|
|
61
|
+
// a deploy retires this generation through the main's `retire` message, so a
|
|
62
|
+
// bound resolved only by this command could never reach that drain.
|
|
85
63
|
await waitForBackgroundJobsWorkerShutdown({
|
|
86
64
|
onReady: () => console.log("Background jobs worker connected"),
|
|
87
|
-
timeoutMs,
|
|
88
65
|
worker
|
|
89
66
|
})
|
|
90
67
|
}
|