velocious 1.0.524 → 1.0.525
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -1
- package/build/background-jobs/worker.js +126 -7
- package/build/configuration-types.js +9 -0
- package/build/configuration.js +9 -1
- package/build/src/background-jobs/worker.d.ts +85 -2
- package/build/src/background-jobs/worker.d.ts.map +1 -1
- package/build/src/background-jobs/worker.js +116 -8
- package/build/src/configuration-types.d.ts +21 -0
- package/build/src/configuration-types.d.ts.map +1 -1
- package/build/src/configuration-types.js +10 -1
- package/build/src/configuration.d.ts.map +1 -1
- package/build/src/configuration.js +10 -2
- package/package.json +1 -1
- package/src/background-jobs/worker.js +126 -7
- package/src/configuration-types.js +9 -0
- package/src/configuration.js +9 -1
package/README.md
CHANGED
|
@@ -1975,7 +1975,8 @@ export default new Configuration({
|
|
|
1975
1975
|
databaseIdentifier: "default",
|
|
1976
1976
|
maxConcurrentForkedJobs: 4,
|
|
1977
1977
|
maxConcurrentInlineJobs: 4,
|
|
1978
|
-
dispatchStrategy: "beacon"
|
|
1978
|
+
dispatchStrategy: "beacon",
|
|
1979
|
+
jobTimeoutMs: null
|
|
1979
1980
|
}
|
|
1980
1981
|
})
|
|
1981
1982
|
```
|
|
@@ -1991,6 +1992,7 @@ VELOCIOUS_BACKGROUND_JOBS_MAX_CONCURRENT_INLINE_JOBS=4
|
|
|
1991
1992
|
VELOCIOUS_BACKGROUND_JOBS_DISPATCH_STRATEGY=beacon
|
|
1992
1993
|
VELOCIOUS_BACKGROUND_JOBS_POLL_INTERVAL_MS=1000
|
|
1993
1994
|
VELOCIOUS_BACKGROUND_JOBS_WORKER_SHUTDOWN_TIMEOUT_MS=indefinite
|
|
1995
|
+
VELOCIOUS_BACKGROUND_JOBS_JOB_TIMEOUT_MS=5400000
|
|
1994
1996
|
```
|
|
1995
1997
|
|
|
1996
1998
|
`VELOCIOUS_BACKGROUND_JOBS_WORKER_SHUTDOWN_TIMEOUT_MS` (default: `indefinite`) bounds how long a `background-jobs-worker` waits for in-flight jobs on `SIGTERM`/`SIGINT` before terminating any forked or spawned child runners still running, so they are not orphaned across a deploy. The default waits for jobs to finish and never interrupts a running job; set a positive number of milliseconds for a finite cap (keep it shorter than your process supervisor's graceful-stop window so the worker reaps its own runners first). See [docs/background-jobs.md](docs/background-jobs.md#worker-shutdown-and-process-job-draining).
|
|
@@ -1999,6 +2001,8 @@ VELOCIOUS_BACKGROUND_JOBS_WORKER_SHUTDOWN_TIMEOUT_MS=indefinite
|
|
|
1999
2001
|
|
|
2000
2002
|
`maxConcurrentForkedJobs` (default: `4`) caps how many out-of-process `executionMode: "forked"` or `executionMode: "spawned"` jobs one worker may keep in flight. Forked jobs use `child_process.fork()` with an attached IPC channel. After the main process acknowledges their durable status report, forked and spawned one-shot runners exit without waiting for graceful Beacon/database teardown; the OS closes their process-owned resources. A missing or rejected status acknowledgement makes the runner exit as failed instead of reporting clean success. Spawned jobs use the legacy `background-jobs-runner` CLI process via `child_process.spawn()` and are only for callers that intentionally want that spawned behavior.
|
|
2001
2003
|
|
|
2004
|
+
`jobTimeoutMs` (or `VELOCIOUS_BACKGROUND_JOBS_JOB_TIMEOUT_MS`, milliseconds; default: disabled) is a wall-clock backstop for a single `"forked"` runner. A forked job still running after the timeout is terminated (`SIGTERM`, then `SIGKILL` after the reaping grace) and reported `failed`, so a genuinely-hung runner can't pin a worker — and its whole-app boot and DB connections — indefinitely (notably a retired-release worker draining after a deploy). It's a coarse safety net, not per-job tuning: it applies to every forked runner, so set it well above the longest legitimate forked job. Omit it, or set `null`/`<= 0`, to disable. `"inline"` jobs are not covered — they share the worker's process and can't be killed without killing the worker. See [docs/background-jobs.md](docs/background-jobs.md#forked-job-timeout-hung-runner-backstop).
|
|
2005
|
+
|
|
2002
2006
|
### Dispatch strategy
|
|
2003
2007
|
|
|
2004
2008
|
`dispatchStrategy` controls how `background-jobs-main` detects new work.
|
|
@@ -11,6 +11,13 @@ import {fileURLToPath} from "node:url"
|
|
|
11
11
|
|
|
12
12
|
/** Grace period after SIGTERM before a lingering process runner is SIGKILLed. */
|
|
13
13
|
const FORKED_CHILD_SIGKILL_GRACE_MS = 5000
|
|
14
|
+
/**
|
|
15
|
+
* Largest delay Node's `setTimeout` accepts without overflowing to a 1ms delay
|
|
16
|
+
* (a 32-bit signed int of ms, ~24.8 days). A `jobTimeoutMs` above this — or a
|
|
17
|
+
* non-finite one like `Infinity` — is clamped/disabled rather than coerced to
|
|
18
|
+
* ~1ms, which would otherwise terminate every forked job almost immediately.
|
|
19
|
+
*/
|
|
20
|
+
const MAX_FORKED_JOB_TIMEOUT_MS = 2_147_483_647
|
|
14
21
|
const FORKED_RUNNER_ENTRY_PATH = fileURLToPath(new URL("./forked-runner-child.js", import.meta.url))
|
|
15
22
|
/** How often the worker sends a liveness heartbeat to the main. */
|
|
16
23
|
const HEARTBEAT_INTERVAL_MS = 15000
|
|
@@ -21,6 +28,15 @@ const SOCKET_KEEPALIVE_MS = 10000
|
|
|
21
28
|
* @type {import("./types.js").BackgroundJobExecutionMode[]} */
|
|
22
29
|
const EXECUTION_MODES = ["inline", "forked", "spawned"]
|
|
23
30
|
|
|
31
|
+
/**
|
|
32
|
+
* Per-forked-child timeout bookkeeping.
|
|
33
|
+
* @typedef {object} ForkedJobTimeoutState
|
|
34
|
+
* @property {boolean} timedOut - Whether the timeout fired and the child was terminated.
|
|
35
|
+
* @property {number | null} timeoutMs - The armed timeout in ms, or null when disabled.
|
|
36
|
+
* @property {ReturnType<typeof setTimeout> | null} timer - The pending timeout timer, cleared on exit.
|
|
37
|
+
* @property {ReturnType<typeof setTimeout> | null} sigkillTimer - The pending SIGKILL grace timer, cleared on exit.
|
|
38
|
+
*/
|
|
39
|
+
|
|
24
40
|
export default class BackgroundJobsWorker {
|
|
25
41
|
/**
|
|
26
42
|
* Runs constructor.
|
|
@@ -32,8 +48,9 @@ export default class BackgroundJobsWorker {
|
|
|
32
48
|
* @param {number} [args.maxConcurrentInlineJobs] - Override the inline-job concurrency cap from `configuration.getBackgroundJobsConfig()`.
|
|
33
49
|
* @param {number} [args.forkedChildSigkillGraceMs] - Override the grace period between SIGTERM and SIGKILL when reaping lingering process runners on stop.
|
|
34
50
|
* @param {number} [args.heartbeatIntervalMs] - Override the liveness heartbeat interval (default 15000ms).
|
|
51
|
+
* @param {number} [args.jobTimeoutMs] - Override the forked-job wall-clock timeout from `configuration.getBackgroundJobsConfig()`. `0` disables it.
|
|
35
52
|
*/
|
|
36
|
-
constructor({configuration, host, port, maxConcurrentForkedJobs, maxConcurrentInlineJobs, forkedChildSigkillGraceMs, heartbeatIntervalMs} = {}) {
|
|
53
|
+
constructor({configuration, host, port, maxConcurrentForkedJobs, maxConcurrentInlineJobs, forkedChildSigkillGraceMs, heartbeatIntervalMs, jobTimeoutMs} = {}) {
|
|
37
54
|
/**
|
|
38
55
|
* Narrows the runtime value to the documented type.
|
|
39
56
|
* @type {Promise<import("../configuration.js").default>} */
|
|
@@ -77,6 +94,13 @@ export default class BackgroundJobsWorker {
|
|
|
77
94
|
this.forkedChildSigkillGraceMs = typeof forkedChildSigkillGraceMs === "number" && forkedChildSigkillGraceMs >= 0
|
|
78
95
|
? forkedChildSigkillGraceMs
|
|
79
96
|
: FORKED_CHILD_SIGKILL_GRACE_MS
|
|
97
|
+
/**
|
|
98
|
+
* Constructor override for the forked-job wall-clock timeout. When unset the
|
|
99
|
+
* timeout is read from `configuration.getBackgroundJobsConfig().jobTimeoutMs`
|
|
100
|
+
* at fork time (default: disabled).
|
|
101
|
+
* @type {number | undefined}
|
|
102
|
+
*/
|
|
103
|
+
this.jobTimeoutMsOverride = typeof jobTimeoutMs === "number" ? jobTimeoutMs : undefined
|
|
80
104
|
this.shouldStop = false
|
|
81
105
|
this.workerId = randomUUID()
|
|
82
106
|
this.heartbeatIntervalMs = typeof heartbeatIntervalMs === "number" && heartbeatIntervalMs >= 1
|
|
@@ -590,16 +614,109 @@ export default class BackgroundJobsWorker {
|
|
|
590
614
|
* @returns {Promise<void>} - Resolves when the child exits.
|
|
591
615
|
*/
|
|
592
616
|
_waitForForkedChild({child, payload}) {
|
|
617
|
+
const timeoutState = this._armForkedJobTimeout({child})
|
|
618
|
+
|
|
593
619
|
return new Promise((resolve) => {
|
|
594
620
|
child.once("exit", (code, signal) => {
|
|
595
|
-
this.
|
|
621
|
+
this._clearForkedJobTimeout(timeoutState)
|
|
622
|
+
this._handleForkedChildExit({child, code, signal, payload, resolve, timeoutState})
|
|
596
623
|
})
|
|
597
624
|
child.once("error", (error) => {
|
|
625
|
+
this._clearForkedJobTimeout(timeoutState)
|
|
598
626
|
this._handleForkedChildError({child, error, payload, resolve})
|
|
599
627
|
})
|
|
600
628
|
})
|
|
601
629
|
}
|
|
602
630
|
|
|
631
|
+
/**
|
|
632
|
+
* Arms a wall-clock backstop for a forked job runner. A forked job still
|
|
633
|
+
* running after `jobTimeoutMs` is terminated (SIGTERM, then SIGKILL after the
|
|
634
|
+
* grace) so a single genuinely-hung runner can't pin a draining worker — and
|
|
635
|
+
* its full-app boot and database connections — indefinitely. Returns a state
|
|
636
|
+
* object the exit/error handlers use to cancel the timer and to report a
|
|
637
|
+
* timeout-specific failure. When no timeout is configured the timer is null
|
|
638
|
+
* and behavior is unchanged.
|
|
639
|
+
* @param {object} args - Options.
|
|
640
|
+
* @param {import("node:child_process").ChildProcess} args.child - Forked child process.
|
|
641
|
+
* @returns {ForkedJobTimeoutState} - Timeout state.
|
|
642
|
+
*/
|
|
643
|
+
_armForkedJobTimeout({child}) {
|
|
644
|
+
const timeoutMs = this._resolveForkedJobTimeoutMs()
|
|
645
|
+
/** @type {ForkedJobTimeoutState} */
|
|
646
|
+
const state = {timedOut: false, timeoutMs, timer: null, sigkillTimer: null}
|
|
647
|
+
|
|
648
|
+
if (!(typeof timeoutMs === "number" && timeoutMs > 0)) return state
|
|
649
|
+
|
|
650
|
+
state.timer = setTimeout(() => this._onForkedJobTimeout({child, state}), timeoutMs)
|
|
651
|
+
|
|
652
|
+
return state
|
|
653
|
+
}
|
|
654
|
+
|
|
655
|
+
/**
|
|
656
|
+
* Resolves the effective forked-job timeout in ms, or null when disabled. The
|
|
657
|
+
* constructor override wins; otherwise the value comes from the background-jobs
|
|
658
|
+
* configuration. A non-positive value disables the backstop.
|
|
659
|
+
* @returns {number | null} - Timeout in ms, or null when disabled.
|
|
660
|
+
*/
|
|
661
|
+
_resolveForkedJobTimeoutMs() {
|
|
662
|
+
const raw = typeof this.jobTimeoutMsOverride === "number"
|
|
663
|
+
? this.jobTimeoutMsOverride
|
|
664
|
+
: (this.configuration ? this.configuration.getBackgroundJobsConfig().jobTimeoutMs : null)
|
|
665
|
+
|
|
666
|
+
// A non-finite (e.g. Infinity) or non-positive value disables the backstop;
|
|
667
|
+
// a finite value beyond Node's timer range is clamped to the max rather than
|
|
668
|
+
// silently coerced to ~1ms (which would kill every forked job immediately).
|
|
669
|
+
if (typeof raw !== "number" || !Number.isFinite(raw) || raw <= 0) return null
|
|
670
|
+
|
|
671
|
+
return Math.min(raw, MAX_FORKED_JOB_TIMEOUT_MS)
|
|
672
|
+
}
|
|
673
|
+
|
|
674
|
+
/**
|
|
675
|
+
* Fired when a forked runner overruns its timeout. Sends SIGTERM for a clean
|
|
676
|
+
* shutdown, then SIGKILL after the grace for a runner that ignores it. The
|
|
677
|
+
* resulting non-clean exit flows through `_handleForkedChildExit`, which frees
|
|
678
|
+
* the slot and reports the job failed.
|
|
679
|
+
* @param {object} args - Options.
|
|
680
|
+
* @param {import("node:child_process").ChildProcess} args.child - Forked child process.
|
|
681
|
+
* @param {ForkedJobTimeoutState} args.state - Timeout state.
|
|
682
|
+
* @returns {void}
|
|
683
|
+
*/
|
|
684
|
+
_onForkedJobTimeout({child, state}) {
|
|
685
|
+
state.timedOut = true
|
|
686
|
+
|
|
687
|
+
try {
|
|
688
|
+
child.kill("SIGTERM")
|
|
689
|
+
} catch {
|
|
690
|
+
// Child already exited; nothing to do.
|
|
691
|
+
}
|
|
692
|
+
|
|
693
|
+
state.sigkillTimer = setTimeout(() => {
|
|
694
|
+
try {
|
|
695
|
+
child.kill("SIGKILL")
|
|
696
|
+
} catch {
|
|
697
|
+
// Child already exited; nothing to do.
|
|
698
|
+
}
|
|
699
|
+
}, this.forkedChildSigkillGraceMs)
|
|
700
|
+
}
|
|
701
|
+
|
|
702
|
+
/**
|
|
703
|
+
* Cancels any pending timeout/SIGKILL timers for a forked runner that has
|
|
704
|
+
* exited (or errored) so they never fire against a gone or reused child.
|
|
705
|
+
* @param {ForkedJobTimeoutState} state - Timeout state.
|
|
706
|
+
* @returns {void}
|
|
707
|
+
*/
|
|
708
|
+
_clearForkedJobTimeout(state) {
|
|
709
|
+
if (state.timer) {
|
|
710
|
+
clearTimeout(state.timer)
|
|
711
|
+
state.timer = null
|
|
712
|
+
}
|
|
713
|
+
|
|
714
|
+
if (state.sigkillTimer) {
|
|
715
|
+
clearTimeout(state.sigkillTimer)
|
|
716
|
+
state.sigkillTimer = null
|
|
717
|
+
}
|
|
718
|
+
}
|
|
719
|
+
|
|
603
720
|
/**
|
|
604
721
|
* Runs handle forked child exit.
|
|
605
722
|
* @param {object} args - Options.
|
|
@@ -608,9 +725,10 @@ export default class BackgroundJobsWorker {
|
|
|
608
725
|
* @param {keyof typeof import("node:os").constants.signals | null} args.signal - Exit signal.
|
|
609
726
|
* @param {import("./types.js").BackgroundJobPayload & {id: string}} args.payload - Payload.
|
|
610
727
|
* @param {(value: void) => void} args.resolve - Promise resolver.
|
|
728
|
+
* @param {ForkedJobTimeoutState} [args.timeoutState] - Timeout state, when the runner had a wall-clock backstop.
|
|
611
729
|
* @returns {void}
|
|
612
730
|
*/
|
|
613
|
-
_handleForkedChildExit({child, code, signal, payload, resolve}) {
|
|
731
|
+
_handleForkedChildExit({child, code, signal, payload, resolve, timeoutState}) {
|
|
614
732
|
this.inflightProcessChildren.delete(child)
|
|
615
733
|
|
|
616
734
|
// Free the worker slot as soon as the child is gone — never gate it on the
|
|
@@ -620,10 +738,11 @@ export default class BackgroundJobsWorker {
|
|
|
620
738
|
|
|
621
739
|
if (this._forkedChildExitedCleanly({code, signal})) return
|
|
622
740
|
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
741
|
+
const error = timeoutState?.timedOut
|
|
742
|
+
? new Error(`Forked background job runner timed out after ${timeoutState.timeoutMs}ms and was terminated: code=${code} signal=${signal || "none"}`)
|
|
743
|
+
: new Error(`Forked background job runner exited before reporting: code=${code} signal=${signal || "none"}`)
|
|
744
|
+
|
|
745
|
+
this._reportForkedChildFailure({payload, error})
|
|
627
746
|
}
|
|
628
747
|
|
|
629
748
|
/**
|
|
@@ -192,6 +192,15 @@
|
|
|
192
192
|
* to restore the legacy fixed-interval poll.
|
|
193
193
|
* @property {number} [pollIntervalMs] - Poll interval in milliseconds. Only used
|
|
194
194
|
* when `dispatchStrategy === "polling"`. Default: `1000`.
|
|
195
|
+
* @property {number | null} [jobTimeoutMs] - Wall-clock backstop, in ms, for a
|
|
196
|
+
* `"forked"` job runner. A forked job still running after this is terminated
|
|
197
|
+
* (SIGTERM, then SIGKILL after the reaping grace) and reported failed, so a
|
|
198
|
+
* single genuinely-hung runner can't pin a draining worker — and its full-app
|
|
199
|
+
* boot and DB connections — indefinitely (e.g. across a deploy where a retired
|
|
200
|
+
* worker drains in-flight jobs). This is a coarse safety net, not per-job
|
|
201
|
+
* tuning: set it well above the longest legitimate forked job (build runners,
|
|
202
|
+
* long imports) so only genuinely-stuck runners are killed. Omit, `null`, or
|
|
203
|
+
* `<= 0` to disable (default), which preserves the prior unbounded behavior.
|
|
195
204
|
* @property {BackgroundJobsRetentionConfiguration} [retention] - Retention/pruning
|
|
196
205
|
* of terminal job rows. Without pruning the jobs table grows unbounded
|
|
197
206
|
* (completed rows accumulate forever), which bloats storage and indexes and
|
package/build/configuration.js
CHANGED
|
@@ -1270,10 +1270,12 @@ export default class VelociousConfiguration {
|
|
|
1270
1270
|
const envMaxConcurrentRaw = process.env.VELOCIOUS_BACKGROUND_JOBS_MAX_CONCURRENT_INLINE_JOBS
|
|
1271
1271
|
const envDispatchStrategy = process.env.VELOCIOUS_BACKGROUND_JOBS_DISPATCH_STRATEGY
|
|
1272
1272
|
const envPollIntervalRaw = process.env.VELOCIOUS_BACKGROUND_JOBS_POLL_INTERVAL_MS
|
|
1273
|
+
const envJobTimeoutRaw = process.env.VELOCIOUS_BACKGROUND_JOBS_JOB_TIMEOUT_MS
|
|
1273
1274
|
const envPort = envPortRaw ? Number(envPortRaw) : undefined
|
|
1274
1275
|
const envMaxConcurrentForked = envMaxConcurrentForkedRaw ? Number(envMaxConcurrentForkedRaw) : undefined
|
|
1275
1276
|
const envMaxConcurrent = envMaxConcurrentRaw ? Number(envMaxConcurrentRaw) : undefined
|
|
1276
1277
|
const envPollInterval = envPollIntervalRaw ? Number(envPollIntervalRaw) : undefined
|
|
1278
|
+
const envJobTimeout = envJobTimeoutRaw ? Number(envJobTimeoutRaw) : undefined
|
|
1277
1279
|
const configured = this._backgroundJobs || {}
|
|
1278
1280
|
const host = configured.host || envHost || "127.0.0.1"
|
|
1279
1281
|
const port = typeof configured.port === "number"
|
|
@@ -1292,6 +1294,12 @@ export default class VelociousConfiguration {
|
|
|
1292
1294
|
? configured.pollIntervalMs
|
|
1293
1295
|
: (typeof envPollInterval === "number" && Number.isFinite(envPollInterval) && envPollInterval >= 1 ? envPollInterval : 1000)
|
|
1294
1296
|
const queues = configured.queues && typeof configured.queues === "object" ? configured.queues : {}
|
|
1297
|
+
// An explicit config value wins over the env var — including `null`/`0`,
|
|
1298
|
+
// which disable the backstop even when the environment sets a default.
|
|
1299
|
+
// Only fall through to the env var when config omits `jobTimeoutMs` entirely.
|
|
1300
|
+
const jobTimeoutMs = "jobTimeoutMs" in configured
|
|
1301
|
+
? (typeof configured.jobTimeoutMs === "number" && configured.jobTimeoutMs > 0 ? configured.jobTimeoutMs : null)
|
|
1302
|
+
: (typeof envJobTimeout === "number" && Number.isFinite(envJobTimeout) && envJobTimeout > 0 ? envJobTimeout : null)
|
|
1295
1303
|
const configuredRetention = configured.retention && typeof configured.retention === "object" ? configured.retention : {}
|
|
1296
1304
|
const retention = {
|
|
1297
1305
|
completedTtlMs: typeof configuredRetention.completedTtlMs === "number" || configuredRetention.completedTtlMs === null
|
|
@@ -1308,7 +1316,7 @@ export default class VelociousConfiguration {
|
|
|
1308
1316
|
: 60 * 60 * 1000
|
|
1309
1317
|
}
|
|
1310
1318
|
|
|
1311
|
-
return {host, port, databaseIdentifier, maxConcurrentForkedJobs, maxConcurrentInlineJobs, dispatchStrategy, pollIntervalMs, queues, retention}
|
|
1319
|
+
return {host, port, databaseIdentifier, maxConcurrentForkedJobs, maxConcurrentInlineJobs, dispatchStrategy, pollIntervalMs, queues, jobTimeoutMs, retention}
|
|
1312
1320
|
}
|
|
1313
1321
|
|
|
1314
1322
|
/**
|
|
@@ -1,3 +1,11 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Per-forked-child timeout bookkeeping.
|
|
3
|
+
* @typedef {object} ForkedJobTimeoutState
|
|
4
|
+
* @property {boolean} timedOut - Whether the timeout fired and the child was terminated.
|
|
5
|
+
* @property {number | null} timeoutMs - The armed timeout in ms, or null when disabled.
|
|
6
|
+
* @property {ReturnType<typeof setTimeout> | null} timer - The pending timeout timer, cleared on exit.
|
|
7
|
+
* @property {ReturnType<typeof setTimeout> | null} sigkillTimer - The pending SIGKILL grace timer, cleared on exit.
|
|
8
|
+
*/
|
|
1
9
|
export default class BackgroundJobsWorker {
|
|
2
10
|
/**
|
|
3
11
|
* Runs constructor.
|
|
@@ -9,8 +17,9 @@ export default class BackgroundJobsWorker {
|
|
|
9
17
|
* @param {number} [args.maxConcurrentInlineJobs] - Override the inline-job concurrency cap from `configuration.getBackgroundJobsConfig()`.
|
|
10
18
|
* @param {number} [args.forkedChildSigkillGraceMs] - Override the grace period between SIGTERM and SIGKILL when reaping lingering process runners on stop.
|
|
11
19
|
* @param {number} [args.heartbeatIntervalMs] - Override the liveness heartbeat interval (default 15000ms).
|
|
20
|
+
* @param {number} [args.jobTimeoutMs] - Override the forked-job wall-clock timeout from `configuration.getBackgroundJobsConfig()`. `0` disables it.
|
|
12
21
|
*/
|
|
13
|
-
constructor({ configuration, host, port, maxConcurrentForkedJobs, maxConcurrentInlineJobs, forkedChildSigkillGraceMs, heartbeatIntervalMs }?: {
|
|
22
|
+
constructor({ configuration, host, port, maxConcurrentForkedJobs, maxConcurrentInlineJobs, forkedChildSigkillGraceMs, heartbeatIntervalMs, jobTimeoutMs }?: {
|
|
14
23
|
configuration?: import("../configuration.js").default | undefined;
|
|
15
24
|
host?: string | undefined;
|
|
16
25
|
port?: number | undefined;
|
|
@@ -18,6 +27,7 @@ export default class BackgroundJobsWorker {
|
|
|
18
27
|
maxConcurrentInlineJobs?: number | undefined;
|
|
19
28
|
forkedChildSigkillGraceMs?: number | undefined;
|
|
20
29
|
heartbeatIntervalMs?: number | undefined;
|
|
30
|
+
jobTimeoutMs?: number | undefined;
|
|
21
31
|
});
|
|
22
32
|
/**
|
|
23
33
|
* Narrows the runtime value to the documented type.
|
|
@@ -56,6 +66,13 @@ export default class BackgroundJobsWorker {
|
|
|
56
66
|
* @type {number}
|
|
57
67
|
*/
|
|
58
68
|
forkedChildSigkillGraceMs: number;
|
|
69
|
+
/**
|
|
70
|
+
* Constructor override for the forked-job wall-clock timeout. When unset the
|
|
71
|
+
* timeout is read from `configuration.getBackgroundJobsConfig().jobTimeoutMs`
|
|
72
|
+
* at fork time (default: disabled).
|
|
73
|
+
* @type {number | undefined}
|
|
74
|
+
*/
|
|
75
|
+
jobTimeoutMsOverride: number | undefined;
|
|
59
76
|
shouldStop: boolean;
|
|
60
77
|
workerId: `${string}-${string}-${string}-${string}-${string}`;
|
|
61
78
|
heartbeatIntervalMs: number;
|
|
@@ -249,6 +266,49 @@ export default class BackgroundJobsWorker {
|
|
|
249
266
|
id: string;
|
|
250
267
|
};
|
|
251
268
|
}): Promise<void>;
|
|
269
|
+
/**
|
|
270
|
+
* Arms a wall-clock backstop for a forked job runner. A forked job still
|
|
271
|
+
* running after `jobTimeoutMs` is terminated (SIGTERM, then SIGKILL after the
|
|
272
|
+
* grace) so a single genuinely-hung runner can't pin a draining worker — and
|
|
273
|
+
* its full-app boot and database connections — indefinitely. Returns a state
|
|
274
|
+
* object the exit/error handlers use to cancel the timer and to report a
|
|
275
|
+
* timeout-specific failure. When no timeout is configured the timer is null
|
|
276
|
+
* and behavior is unchanged.
|
|
277
|
+
* @param {object} args - Options.
|
|
278
|
+
* @param {import("node:child_process").ChildProcess} args.child - Forked child process.
|
|
279
|
+
* @returns {ForkedJobTimeoutState} - Timeout state.
|
|
280
|
+
*/
|
|
281
|
+
_armForkedJobTimeout({ child }: {
|
|
282
|
+
child: import("node:child_process").ChildProcess;
|
|
283
|
+
}): ForkedJobTimeoutState;
|
|
284
|
+
/**
|
|
285
|
+
* Resolves the effective forked-job timeout in ms, or null when disabled. The
|
|
286
|
+
* constructor override wins; otherwise the value comes from the background-jobs
|
|
287
|
+
* configuration. A non-positive value disables the backstop.
|
|
288
|
+
* @returns {number | null} - Timeout in ms, or null when disabled.
|
|
289
|
+
*/
|
|
290
|
+
_resolveForkedJobTimeoutMs(): number | null;
|
|
291
|
+
/**
|
|
292
|
+
* Fired when a forked runner overruns its timeout. Sends SIGTERM for a clean
|
|
293
|
+
* shutdown, then SIGKILL after the grace for a runner that ignores it. The
|
|
294
|
+
* resulting non-clean exit flows through `_handleForkedChildExit`, which frees
|
|
295
|
+
* the slot and reports the job failed.
|
|
296
|
+
* @param {object} args - Options.
|
|
297
|
+
* @param {import("node:child_process").ChildProcess} args.child - Forked child process.
|
|
298
|
+
* @param {ForkedJobTimeoutState} args.state - Timeout state.
|
|
299
|
+
* @returns {void}
|
|
300
|
+
*/
|
|
301
|
+
_onForkedJobTimeout({ child, state }: {
|
|
302
|
+
child: import("node:child_process").ChildProcess;
|
|
303
|
+
state: ForkedJobTimeoutState;
|
|
304
|
+
}): void;
|
|
305
|
+
/**
|
|
306
|
+
* Cancels any pending timeout/SIGKILL timers for a forked runner that has
|
|
307
|
+
* exited (or errored) so they never fire against a gone or reused child.
|
|
308
|
+
* @param {ForkedJobTimeoutState} state - Timeout state.
|
|
309
|
+
* @returns {void}
|
|
310
|
+
*/
|
|
311
|
+
_clearForkedJobTimeout(state: ForkedJobTimeoutState): void;
|
|
252
312
|
/**
|
|
253
313
|
* Runs handle forked child exit.
|
|
254
314
|
* @param {object} args - Options.
|
|
@@ -257,9 +317,10 @@ export default class BackgroundJobsWorker {
|
|
|
257
317
|
* @param {keyof typeof import("node:os").constants.signals | null} args.signal - Exit signal.
|
|
258
318
|
* @param {import("./types.js").BackgroundJobPayload & {id: string}} args.payload - Payload.
|
|
259
319
|
* @param {(value: void) => void} args.resolve - Promise resolver.
|
|
320
|
+
* @param {ForkedJobTimeoutState} [args.timeoutState] - Timeout state, when the runner had a wall-clock backstop.
|
|
260
321
|
* @returns {void}
|
|
261
322
|
*/
|
|
262
|
-
_handleForkedChildExit({ child, code, signal, payload, resolve }: {
|
|
323
|
+
_handleForkedChildExit({ child, code, signal, payload, resolve, timeoutState }: {
|
|
263
324
|
child: import("node:child_process").ChildProcess;
|
|
264
325
|
code: number | null;
|
|
265
326
|
signal: keyof typeof import("node:os").constants.signals | null;
|
|
@@ -267,6 +328,7 @@ export default class BackgroundJobsWorker {
|
|
|
267
328
|
id: string;
|
|
268
329
|
};
|
|
269
330
|
resolve: (value: void) => void;
|
|
331
|
+
timeoutState?: ForkedJobTimeoutState | undefined;
|
|
270
332
|
}): void;
|
|
271
333
|
/**
|
|
272
334
|
* Runs forked child exited cleanly.
|
|
@@ -369,6 +431,27 @@ export default class BackgroundJobsWorker {
|
|
|
369
431
|
workerId?: string | undefined;
|
|
370
432
|
}): void;
|
|
371
433
|
}
|
|
434
|
+
/**
|
|
435
|
+
* Per-forked-child timeout bookkeeping.
|
|
436
|
+
*/
|
|
437
|
+
export type ForkedJobTimeoutState = {
|
|
438
|
+
/**
|
|
439
|
+
* - Whether the timeout fired and the child was terminated.
|
|
440
|
+
*/
|
|
441
|
+
timedOut: boolean;
|
|
442
|
+
/**
|
|
443
|
+
* - The armed timeout in ms, or null when disabled.
|
|
444
|
+
*/
|
|
445
|
+
timeoutMs: number | null;
|
|
446
|
+
/**
|
|
447
|
+
* - The pending timeout timer, cleared on exit.
|
|
448
|
+
*/
|
|
449
|
+
timer: ReturnType<typeof setTimeout> | null;
|
|
450
|
+
/**
|
|
451
|
+
* - The pending SIGKILL grace timer, cleared on exit.
|
|
452
|
+
*/
|
|
453
|
+
sigkillTimer: ReturnType<typeof setTimeout> | null;
|
|
454
|
+
};
|
|
372
455
|
import JsonSocket from "./json-socket.js";
|
|
373
456
|
import BackgroundJobsStatusReporter from "./status-reporter.js";
|
|
374
457
|
//# sourceMappingURL=worker.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"worker.d.ts","sourceRoot":"","sources":["../../../src/background-jobs/worker.js"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"worker.d.ts","sourceRoot":"","sources":["../../../src/background-jobs/worker.js"],"names":[],"mappings":"AA8BA;;;;;;;GAOG;AAEH;IACE;;;;;;;;;;;OAWG;IACH,4JATG;QAAqD,aAAa;QAC5C,IAAI;QACJ,IAAI;QACJ,uBAAuB;QACvB,uBAAuB;QACvB,yBAAyB;QACzB,mBAAmB;QACnB,YAAY;KACpC,EAmGA;IAjGC;;gEAE4D;IAC5D,sBADU,OAAO,CAAC,OAAO,qBAAqB,EAAE,OAAO,CAAC,CAC4C;IACpG;;mEAE+D;IAC/D,eADU,OAAO,qBAAqB,EAAE,OAAO,GAAG,SAAS,CAC7B;IAC9B,yBAAgB;IAChB,yBAAgB;IAChB;;;;;OAKG;IACH,iCAFU,MAAM,GAAG,SAAS,CAIf;IACb;;oCAEgC;IAChC,iCADU,MAAM,GAAG,SAAS,CAGf;IACb;;;;OAIG;IACH,yBAFU,MAAM,CAEwD;IACxE;;wBAEoB;IACpB,yBADU,MAAM,CACwD;IACxE;;;;OAIG;IACH,2BAFU,MAAM,CAIiB;IACjC;;;;;OAKG;IACH,sBAFU,MAAM,GAAG,SAAS,CAE2D;IACvF,oBAAuB;IACvB,8DAA4B;IAC5B,4BAEyB;IACzB;;4DAEwD;IACxD,iBADU,UAAU,CAAC,OAAO,WAAW,CAAC,GAAG,SAAS,CACpB;IAChC;;;;;;OAMG;IACH,iBAFU,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAEI;IAChC;;wCAEoC;IACpC,YADU,UAAU,GAAG,SAAS,CACL;IAC3B;;0DAEsD;IACtD,gBADU,4BAA4B,GAAG,SAAS,CACnB;IAC/B;;;;;;OAMG;IACH,oBAFU,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAEO;IACnC;;;;OAIG;IACH,qBAFU,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAEQ;IACpC;;;;;;OAMG;IACH,yBAFU,GAAG,CAAC,OAAO,oBAAoB,EAAE,YAAY,CAAC,CAEhB;IAG1C;;;OAGG;IACH,SAFa,OAAO,CAAC,IAAI,CAAC,CA0BzB;IAED;;;;;;;;;;;;;;OAcG;IACH,qBAHG;QAAsB,SAAS;KAC/B,GAAU,OAAO,CAAC,IAAI,CAAC,CAgCzB;IAED;;;;;;OAMG;IACH,yBAJW,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,cAClB,MAAM,GACJ,OAAO,CAAC,IAAI,CAAC,CAgBzB;IAED;;;;;OAKG;IACH,6BAFa,OAAO,CAAC,IAAI,CAAC,CAsBzB;IAED,0BAqCC;IAED;;;;;OAKG;IACH,mBAFa,IAAI,CAgBhB;IAED;;;OAGG;IACH,kBAFa,IAAI,CAOhB;IAED;;;;OAIG;IACH,oBAHW,OAAO,YAAY,EAAE,oBAAoB,GACvC,OAAO,CAAC,IAAI,CAAC,CAiBzB;IAED;;;;;;OAMG;IACH,6CAJG;QAA8D,aAAa,EAAnE,OAAO,YAAY,EAAE,0BAA0B;QACgB,OAAO,EAAtE,OAAO,YAAY,EAAE,oBAAoB,GAAG;YAAC,EAAE,EAAE,MAAM,CAAA;SAAC;KAChE,GAAU,OAAO,CAAC,IAAI,CAAC,CAMzB;IAED;;;;OAIG;IACH,0BAHW,OAAO,YAAY,EAAE,oBAAoB,GAAG;QAAC,EAAE,EAAE,MAAM,CAAA;KAAC,GACtD,IAAI,CAgChB;IAED;;;;OAIG;IACH,kCAHW,OAAO,YAAY,EAAE,oBAAoB,GACvC,OAAO,YAAY,EAAE,0BAA0B,CAS3D;IAED;;;;OAIG;IACH,uCAHW,MAAM,GACJ,OAAO,YAAY,EAAE,0BAA0B,CAQ3D;IAED;;;;OAIG;IACH,6BAHW,OAAO,CAAC,IAAI,CAAC,GACX,IAAI,CAyBhB;IAED;;;;OAIG;IACH,gCAHW,OAAO,YAAY,EAAE,oBAAoB,GAAG;QAAC,EAAE,EAAE,MAAM,CAAA;KAAC,GACtD,OAAO,CAAC,IAAI,CAAC,CAyBzB;IAED;;;;;OAKG;IACH,uBAFa,IAAI,CAUhB;IAED;;;OAGG;IACH,iBAFa,OAAO,YAAY,EAAE,0BAA0B,GAAG,IAAI,CAclE;IAED;;;;OAIG;IACH,uBAHW,OAAO,YAAY,EAAE,oBAAoB,GACvC,OAAO,CAAC,IAAI,CAAC,CAkBzB;IAED;;;;OAIG;IACH,kBAHW,OAAO,YAAY,EAAE,oBAAoB,GAAG;QAAC,EAAE,EAAE,MAAM,CAAA;KAAC,GACtD,OAAO,CAAC,IAAI,CAAC,CAYzB;IAED;;;OAGG;IACH,sBAFa,OAAO,oBAAoB,EAAE,YAAY,CAmBrD;IAED;;;;;;OAMG;IACH,wCAJG;QAAwD,KAAK,EAArD,OAAO,oBAAoB,EAAE,YAAY;QACsB,OAAO,EAAtE,OAAO,YAAY,EAAE,oBAAoB,GAAG;YAAC,EAAE,EAAE,MAAM,CAAA;SAAC;KAChE,GAAU,OAAO,CAAC,IAAI,CAAC,CAezB;IAED;;;;;;;;;;;OAWG;IACH,gCAHG;QAAwD,KAAK,EAArD,OAAO,oBAAoB,EAAE,YAAY;KACjD,GAAU,qBAAqB,CAYjC;IAED;;;;;OAKG;IACH,8BAFa,MAAM,GAAG,IAAI,CAazB;IAED;;;;;;;;;OASG;IACH,sCAJG;QAAwD,KAAK,EAArD,OAAO,oBAAoB,EAAE,YAAY;QACb,KAAK,EAAjC,qBAAqB;KAC7B,GAAU,IAAI,CAkBhB;IAED;;;;;OAKG;IACH,8BAHW,qBAAqB,GACnB,IAAI,CAYhB;IAED;;;;;;;;;;OAUG;IACH,gFARG;QAAwD,KAAK,EAArD,OAAO,oBAAoB,EAAE,YAAY;QACrB,IAAI,EAAxB,MAAM,GAAG,IAAI;QACiD,MAAM,EAApE,MAAM,cAAc,SAAS,EAAE,SAAS,CAAC,OAAO,GAAG,IAAI;QACQ,OAAO,EAAtE,OAAO,YAAY,EAAE,oBAAoB,GAAG;YAAC,EAAE,EAAE,MAAM,CAAA;SAAC;QAC5B,OAAO,EAAnC,CAAC,KAAK,EAAE,IAAI,KAAK,IAAI;QACQ,YAAY;KACjD,GAAU,IAAI,CAiBhB;IAED;;;;;;OAMG;IACH,4CAJG;QAA4B,IAAI,EAAxB,MAAM,GAAG,IAAI;QACiD,MAAM,EAApE,MAAM,cAAc,SAAS,EAAE,SAAS,CAAC,OAAO,GAAG,IAAI;KAC/D,GAAU,OAAO,CAInB;IAED;;;;;;;;OAQG;IACH,4DANG;QAAwD,KAAK,EAArD,OAAO,oBAAoB,EAAE,YAAY;QAC7B,KAAK,EAAjB,KAAK;QAC0D,OAAO,EAAtE,OAAO,YAAY,EAAE,oBAAoB,GAAG;YAAC,EAAE,EAAE,MAAM,CAAA;SAAC;QAC5B,OAAO,EAAnC,CAAC,KAAK,EAAE,IAAI,KAAK,IAAI;KAC7B,GAAU,IAAI,CAQhB;IAED;;;;;;OAMG;IACH,uCAJG;QAAwD,KAAK,EAArD,OAAO,oBAAoB,EAAE,YAAY;QACsB,OAAO,EAAtE,OAAO,YAAY,EAAE,oBAAoB,GAAG;YAAC,EAAE,EAAE,MAAM,CAAA;SAAC;KAChE,GAAU,IAAI,CAShB;IAED;;;;;;OAMG;IACH,8CAJG;QAAuE,OAAO,EAAtE,OAAO,YAAY,EAAE,oBAAoB,GAAG;YAAC,EAAE,EAAE,MAAM,CAAA;SAAC;QAChD,KAAK,EAAb,OAAC;KACT,GAAU,IAAI,CAWhB;IAED;;;;OAIG;IACH,mBAHW,OAAO,YAAY,EAAE,oBAAoB,GACvC,OAAO,CAAC,IAAI,CAAC,CAwCzB;IAED;;;;;;;;;;OAUG;IACH,+EARG;QAAqB,KAAK,EAAlB,MAAM;QACuB,MAAM,EAAnC,WAAW,GAAG,QAAQ;QACb,KAAK,GAAd,OAAC;QACa,SAAS;QACT,aAAa;QACb,QAAQ;KAC9B,GAAU,OAAO,CAAC,IAAI,CAAC,CAUzB;IAED;;;;;;;;;;;;OAYG;IACH,2FARG;QAAqB,KAAK,EAAlB,MAAM;QACuB,MAAM,EAAnC,WAAW,GAAG,QAAQ;QACb,KAAK,GAAd,OAAC;QACa,SAAS;QACT,aAAa;QACb,QAAQ;KAC9B,GAAU,IAAI,CAahB;CACF;;;;;;;;cAn2Ba,OAAO;;;;eACP,MAAM,GAAG,IAAI;;;;WACb,UAAU,CAAC,OAAO,UAAU,CAAC,GAAG,IAAI;;;;kBACpC,UAAU,CAAC,OAAO,UAAU,CAAC,GAAG,IAAI;;uBAhC3B,kBAAkB;yCAGA,sBAAsB"}
|