velocious 1.0.528 → 1.0.529
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -1
- package/build/background-jobs/forked-runner-child.js +2 -1
- package/build/background-jobs/job-runner.js +14 -2
- package/build/background-jobs/json-socket.js +2 -0
- package/build/background-jobs/main.js +47 -5
- package/build/background-jobs/pooled-runner-child.js +78 -0
- package/build/background-jobs/runner-process-title.js +6 -0
- package/build/background-jobs/scheduler.js +28 -5
- package/build/background-jobs/store.js +79 -15
- package/build/background-jobs/types.js +5 -5
- package/build/background-jobs/worker.js +184 -5
- package/build/configuration-types.js +4 -0
- package/build/configuration.js +21 -1
- package/build/src/background-jobs/forked-runner-child.js +3 -2
- package/build/src/background-jobs/job-runner.d.ts +7 -0
- package/build/src/background-jobs/job-runner.d.ts.map +1 -1
- package/build/src/background-jobs/job-runner.js +14 -3
- package/build/src/background-jobs/json-socket.d.ts +2 -0
- package/build/src/background-jobs/json-socket.d.ts.map +1 -1
- package/build/src/background-jobs/json-socket.js +3 -1
- package/build/src/background-jobs/main.d.ts +16 -0
- package/build/src/background-jobs/main.d.ts.map +1 -1
- package/build/src/background-jobs/main.js +46 -5
- package/build/src/background-jobs/pooled-runner-child.d.ts +2 -0
- package/build/src/background-jobs/pooled-runner-child.d.ts.map +1 -0
- package/build/src/background-jobs/pooled-runner-child.js +77 -0
- package/build/src/background-jobs/runner-process-title.d.ts +3 -0
- package/build/src/background-jobs/runner-process-title.d.ts.map +1 -0
- package/build/src/background-jobs/runner-process-title.js +6 -0
- package/build/src/background-jobs/scheduler.d.ts +16 -2
- package/build/src/background-jobs/scheduler.d.ts.map +1 -1
- package/build/src/background-jobs/scheduler.js +27 -6
- package/build/src/background-jobs/store.d.ts +24 -0
- package/build/src/background-jobs/store.d.ts.map +1 -1
- package/build/src/background-jobs/store.js +81 -15
- package/build/src/background-jobs/types.d.ts +10 -8
- package/build/src/background-jobs/types.d.ts.map +1 -1
- package/build/src/background-jobs/types.js +6 -6
- package/build/src/background-jobs/worker.d.ts +78 -1
- package/build/src/background-jobs/worker.d.ts.map +1 -1
- package/build/src/background-jobs/worker.js +193 -6
- package/build/src/configuration-types.d.ts +20 -0
- package/build/src/configuration-types.d.ts.map +1 -1
- package/build/src/configuration-types.js +5 -1
- package/build/src/configuration.d.ts.map +1 -1
- package/build/src/configuration.js +22 -2
- package/build/tsconfig.tsbuildinfo +1 -1
- package/package.json +1 -1
- package/src/background-jobs/forked-runner-child.js +2 -1
- package/src/background-jobs/job-runner.js +14 -2
- package/src/background-jobs/json-socket.js +2 -0
- package/src/background-jobs/main.js +47 -5
- package/src/background-jobs/pooled-runner-child.js +78 -0
- package/src/background-jobs/runner-process-title.js +6 -0
- package/src/background-jobs/scheduler.js +28 -5
- package/src/background-jobs/store.js +79 -15
- package/src/background-jobs/types.js +5 -5
- package/src/background-jobs/worker.js +184 -5
- package/src/configuration-types.js +4 -0
- package/src/configuration.js +21 -1
package/README.md
CHANGED
|
@@ -1976,6 +1976,10 @@ export default new Configuration({
|
|
|
1976
1976
|
databaseIdentifier: "default",
|
|
1977
1977
|
maxConcurrentForkedJobs: 4,
|
|
1978
1978
|
maxConcurrentInlineJobs: 4,
|
|
1979
|
+
pooledRunnerCount: 4,
|
|
1980
|
+
pooledRunnerMaxJobs: 100,
|
|
1981
|
+
pooledRunnerMaxRssBytes: 536870912,
|
|
1982
|
+
pooledRunnerMaxLifetimeMs: 3600000,
|
|
1979
1983
|
dispatchStrategy: "beacon",
|
|
1980
1984
|
jobTimeoutMs: null
|
|
1981
1985
|
}
|
|
@@ -1990,6 +1994,10 @@ VELOCIOUS_BACKGROUND_JOBS_PORT=7331
|
|
|
1990
1994
|
VELOCIOUS_BACKGROUND_JOBS_DATABASE_IDENTIFIER=default
|
|
1991
1995
|
VELOCIOUS_BACKGROUND_JOBS_MAX_CONCURRENT_FORKED_JOBS=4
|
|
1992
1996
|
VELOCIOUS_BACKGROUND_JOBS_MAX_CONCURRENT_INLINE_JOBS=4
|
|
1997
|
+
VELOCIOUS_BACKGROUND_JOBS_POOLED_RUNNER_COUNT=4
|
|
1998
|
+
VELOCIOUS_BACKGROUND_JOBS_POOLED_RUNNER_MAX_JOBS=100
|
|
1999
|
+
VELOCIOUS_BACKGROUND_JOBS_POOLED_RUNNER_MAX_RSS_BYTES=536870912
|
|
2000
|
+
VELOCIOUS_BACKGROUND_JOBS_POOLED_RUNNER_MAX_LIFETIME_MS=3600000
|
|
1993
2001
|
VELOCIOUS_BACKGROUND_JOBS_DISPATCH_STRATEGY=beacon
|
|
1994
2002
|
VELOCIOUS_BACKGROUND_JOBS_POLL_INTERVAL_MS=1000
|
|
1995
2003
|
VELOCIOUS_BACKGROUND_JOBS_WORKER_SHUTDOWN_TIMEOUT_MS=indefinite
|
|
@@ -2000,6 +2008,8 @@ VELOCIOUS_BACKGROUND_JOBS_JOB_TIMEOUT_MS=5400000
|
|
|
2000
2008
|
|
|
2001
2009
|
`maxConcurrentInlineJobs` (default: `4`) caps how many `executionMode: "inline"` jobs a single `background-jobs-worker` process runs in parallel. Concurrency is at the JS event-loop level: every job in flight shares the worker's process and DB connection pool, so the cap should fit the pool, not the CPU count. Forking remains the right tool when you need memory isolation across long-running jobs or want to use more cores. The older `forked: false` option still maps to inline mode.
|
|
2002
2010
|
|
|
2011
|
+
New jobs default to `executionMode: "pooled"`: a worker runs them serially in warm, reusable Node child runners. `pooledRunnerCount` (default: `4`) bounds this independent per-worker capacity. It and `pooledRunnerMaxJobs` must be finite positive integers; the RSS and lifetime limits must be finite positive numbers. A child is recycled after an acknowledged job when it reaches `pooledRunnerMaxJobs` (default: `100`), `pooledRunnerMaxRssBytes` (default: `536870912`, or 512 MiB), or `pooledRunnerMaxLifetimeMs` (default: `3600000`, or one hour). Pooled rows remain readable by older mains during rolling upgrades and run there with legacy one-shot fork behavior. `forked: true` remains an explicit alias for `"forked"`; `forked: false` remains inline. See [execution modes and pooled runners](docs/background-jobs.md#execution-modes-and-pooled-runners).
|
|
2012
|
+
|
|
2003
2013
|
`maxConcurrentForkedJobs` (default: `4`) caps how many out-of-process `executionMode: "forked"` or `executionMode: "spawned"` jobs one worker may keep in flight. Forked jobs use `child_process.fork()` with an attached IPC channel. After the main process acknowledges their durable status report, forked and spawned one-shot runners exit without waiting for graceful Beacon/database teardown; the OS closes their process-owned resources. A missing or rejected status acknowledgement makes the runner exit as failed instead of reporting clean success. Spawned jobs use the legacy `background-jobs-runner` CLI process via `child_process.spawn()` and are only for callers that intentionally want that spawned behavior.
|
|
2004
2014
|
|
|
2005
2015
|
`jobTimeoutMs` (or `VELOCIOUS_BACKGROUND_JOBS_JOB_TIMEOUT_MS`, milliseconds; default: disabled) is a wall-clock backstop for a single `"forked"` runner. A forked job still running after the timeout is terminated (`SIGTERM`, then `SIGKILL` after the reaping grace) and reported `failed`, so a genuinely-hung runner can't pin a worker — and its whole-app boot and DB connections — indefinitely (notably a retired-release worker draining after a deploy). It's a coarse safety net, not per-job tuning: it applies to every forked runner, so set it well above the longest legitimate forked job. Omit it, or set `null`/`<= 0`, to disable. `"inline"` jobs are not covered — they share the worker's process and can't be killed without killing the worker. See [docs/background-jobs.md](docs/background-jobs.md#forked-job-timeout-hung-runner-backstop).
|
|
@@ -2130,7 +2140,7 @@ Cron fields are: `minute hour day-of-month month day-of-week`. Supported syntax:
|
|
|
2130
2140
|
|
|
2131
2141
|
Each job must define exactly one of `every` or `cron`. Cron times are evaluated in the **server's local timezone**, at minute granularity.
|
|
2132
2142
|
|
|
2133
|
-
`background-jobs-main` owns the schedule and enqueues the configured jobs into the normal Velocious background-jobs queue. The HTTP server does not run scheduled jobs itself.
|
|
2143
|
+
`background-jobs-main` owns the schedule and enqueues the configured jobs into the normal Velocious background-jobs queue. The HTTP server does not run scheduled jobs itself. Graceful main shutdown waits for scheduled enqueues already in flight before closing database connections, so stopped schedulers cannot write into a later application or test lifecycle.
|
|
2134
2144
|
|
|
2135
2145
|
## Persistence and retries
|
|
2136
2146
|
|
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
// @ts-check
|
|
2
2
|
|
|
3
3
|
import runJobPayload from "./job-runner.js"
|
|
4
|
+
import setRunnerProcessTitle from "./runner-process-title.js"
|
|
4
5
|
|
|
5
6
|
// Name the process so `ps`/`top`/`htop` can identify forked job runners at a
|
|
6
7
|
// glance instead of a wall of generic "node" entries. Updated to the specific
|
|
7
8
|
// job name once one arrives (see runJobMessage), so operators can see exactly
|
|
8
9
|
// which jobs are running, how many of each, and which are eating resources.
|
|
9
|
-
|
|
10
|
+
setRunnerProcessTitle()
|
|
10
11
|
|
|
11
12
|
let finishing = false
|
|
12
13
|
|
|
@@ -6,6 +6,17 @@ import BackgroundJobsStatusReporter from "./status-reporter.js"
|
|
|
6
6
|
|
|
7
7
|
const BEACON_READY_TIMEOUT_MS = 5000
|
|
8
8
|
|
|
9
|
+
export class BackgroundJobPerformedFailure extends Error {
|
|
10
|
+
/**
|
|
11
|
+
* Creates a performed-job failure after its terminal report is acknowledged.
|
|
12
|
+
* @param {Error} cause - A job perform error whose failed terminal report was acknowledged.
|
|
13
|
+
*/
|
|
14
|
+
constructor(cause) {
|
|
15
|
+
super(cause.message, {cause})
|
|
16
|
+
this.name = "BackgroundJobPerformedFailure"
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
|
|
9
20
|
/**
|
|
10
21
|
* Runs report beacon ready error.
|
|
11
22
|
* @param {import("../configuration.js").default} configuration - Configuration.
|
|
@@ -96,11 +107,12 @@ export default async function runJobPayload(payload, {closeConnections = true} =
|
|
|
96
107
|
await perform.apply(jobInstance, payload.args || [])
|
|
97
108
|
})
|
|
98
109
|
} catch (error) {
|
|
110
|
+
const performedError = error instanceof Error ? error : new Error(String(error))
|
|
99
111
|
if (payload.id) {
|
|
100
112
|
await reporter.reportWithRetry({
|
|
101
113
|
jobId: payload.id,
|
|
102
114
|
status: "failed",
|
|
103
|
-
error,
|
|
115
|
+
error: performedError,
|
|
104
116
|
handoffId: payload.handoffId,
|
|
105
117
|
workerId: payload.workerId,
|
|
106
118
|
handedOffAtMs: payload.handedOffAtMs,
|
|
@@ -108,7 +120,7 @@ export default async function runJobPayload(payload, {closeConnections = true} =
|
|
|
108
120
|
})
|
|
109
121
|
}
|
|
110
122
|
|
|
111
|
-
throw
|
|
123
|
+
throw new BackgroundJobPerformedFailure(performedError)
|
|
112
124
|
}
|
|
113
125
|
|
|
114
126
|
if (payload.id) {
|
|
@@ -26,6 +26,8 @@ export default class JsonSocket extends EventEmitter {
|
|
|
26
26
|
* Narrows the runtime value to the documented type.
|
|
27
27
|
* @type {boolean} */
|
|
28
28
|
this.acceptsForkedJobs = true
|
|
29
|
+
/** @type {boolean} */
|
|
30
|
+
this.acceptsPooledJobs = false
|
|
29
31
|
/**
|
|
30
32
|
* Narrows the runtime value to the documented type.
|
|
31
33
|
* @type {boolean} */
|
|
@@ -39,6 +39,12 @@ const WORKER_LIVENESS_SWEEP_MS = 15000
|
|
|
39
39
|
const WORKER_EXECUTION_MODE_CAPABILITIES = [
|
|
40
40
|
{executionMode: "inline", accepts: (worker) => worker.acceptsInlineJobs !== false},
|
|
41
41
|
{executionMode: "forked", accepts: (worker) => worker.acceptsForkedJobs !== false},
|
|
42
|
+
// Pooled is opt-in: only workers that explicitly advertise `acceptsPooled`
|
|
43
|
+
// receive pooled jobs. The `=== true` (rather than `!== false`) check keeps a
|
|
44
|
+
// pre-pooled worker — which never sends the field — out of the pooled-capable
|
|
45
|
+
// set, so the main never dispatches a pooled job to a worker that cannot run
|
|
46
|
+
// one. This is the conservative half of the extended readiness protocol.
|
|
47
|
+
{executionMode: "pooled", accepts: (worker) => worker.acceptsPooledJobs === true},
|
|
42
48
|
{executionMode: "spawned", accepts: (worker) => worker.acceptsSpawnedJobs !== false}
|
|
43
49
|
]
|
|
44
50
|
const WORKER_EXECUTION_MODE_CAPABILITIES_BY_MODE = new Map(
|
|
@@ -81,6 +87,11 @@ export default class BackgroundJobsMain {
|
|
|
81
87
|
* Active durable handoffs keyed by the exact worker socket that received them.
|
|
82
88
|
* @type {Map<JsonSocket, Map<string, string>>} */
|
|
83
89
|
this.workerHandoffs = new Map()
|
|
90
|
+
/**
|
|
91
|
+
* Handoff-adoption queries started by worker hello messages. Shutdown must
|
|
92
|
+
* wait for these before closing the configuration's database pools.
|
|
93
|
+
* @type {Set<Promise<void>>} */
|
|
94
|
+
this.inflightWorkerHandoffAdoptions = new Set()
|
|
84
95
|
/**
|
|
85
96
|
* Narrows the runtime value to the documented type.
|
|
86
97
|
* @type {net.Server | undefined} */
|
|
@@ -209,9 +220,12 @@ export default class BackgroundJobsMain {
|
|
|
209
220
|
this._closeWorkers()
|
|
210
221
|
this._clearTimers()
|
|
211
222
|
this._disconnectBeaconHandlers()
|
|
212
|
-
this.scheduler?.stop()
|
|
213
|
-
|
|
214
|
-
|
|
223
|
+
await this.scheduler?.stop()
|
|
224
|
+
try {
|
|
225
|
+
await this._drainWorkerHandoffAdoptions()
|
|
226
|
+
} finally {
|
|
227
|
+
await this._stopBeaconAndServer()
|
|
228
|
+
}
|
|
215
229
|
}
|
|
216
230
|
|
|
217
231
|
/**
|
|
@@ -417,18 +431,45 @@ export default class BackgroundJobsMain {
|
|
|
417
431
|
if (message?.type !== "hello") return null
|
|
418
432
|
|
|
419
433
|
if (message.role === "worker") {
|
|
434
|
+
if (this._stopped) {
|
|
435
|
+
jsonSocket.close()
|
|
436
|
+
return message.role
|
|
437
|
+
}
|
|
438
|
+
|
|
420
439
|
jsonSocket.workerId = message.workerId
|
|
421
440
|
jsonSocket.supportsHandoffIdReporting = message.supportsHandoffIdReporting === true
|
|
422
441
|
jsonSocket.supportsHeartbeat = message.supportsHeartbeat === true
|
|
423
442
|
jsonSocket.lastSeenAt = Date.now()
|
|
424
443
|
this.workers.add(jsonSocket)
|
|
425
444
|
this.workerHandoffs.set(jsonSocket, new Map())
|
|
426
|
-
|
|
445
|
+
this._trackWorkerHandoffAdoption(jsonSocket)
|
|
427
446
|
}
|
|
428
447
|
|
|
429
448
|
return message.role
|
|
430
449
|
}
|
|
431
450
|
|
|
451
|
+
/**
|
|
452
|
+
* Tracks a worker handoff-adoption query through shutdown.
|
|
453
|
+
* @param {JsonSocket} jsonSocket - Reconnecting worker socket.
|
|
454
|
+
* @returns {void}
|
|
455
|
+
*/
|
|
456
|
+
_trackWorkerHandoffAdoption(jsonSocket) {
|
|
457
|
+
const adoption = this._adoptWorkerHandoffs(jsonSocket)
|
|
458
|
+
this.inflightWorkerHandoffAdoptions.add(adoption)
|
|
459
|
+
const removeAdoption = () => this.inflightWorkerHandoffAdoptions.delete(adoption)
|
|
460
|
+
void adoption.then(removeAdoption, removeAdoption)
|
|
461
|
+
}
|
|
462
|
+
|
|
463
|
+
/**
|
|
464
|
+
* Waits for worker handoff-adoption queries to finish.
|
|
465
|
+
* @returns {Promise<void>} - Resolves when no adoption query remains.
|
|
466
|
+
*/
|
|
467
|
+
async _drainWorkerHandoffAdoptions() {
|
|
468
|
+
while (this.inflightWorkerHandoffAdoptions.size > 0) {
|
|
469
|
+
await Promise.all([...this.inflightWorkerHandoffAdoptions])
|
|
470
|
+
}
|
|
471
|
+
}
|
|
472
|
+
|
|
432
473
|
/**
|
|
433
474
|
* Adopts a reconnecting worker's still-active `handed_off` jobs into its new
|
|
434
475
|
* socket's handoff map. A fresh main (e.g. after a deploy restart) holds no
|
|
@@ -534,6 +575,7 @@ export default class BackgroundJobsMain {
|
|
|
534
575
|
_handleWorkerReady({jsonSocket, message}) {
|
|
535
576
|
jsonSocket.acceptsSpawnedJobs = message.acceptsSpawned !== false && message.acceptsForked !== false
|
|
536
577
|
jsonSocket.acceptsForkedJobs = message.acceptsForked !== false
|
|
578
|
+
jsonSocket.acceptsPooledJobs = message.acceptsPooled === true
|
|
537
579
|
jsonSocket.acceptsInlineJobs = message.acceptsInline !== false
|
|
538
580
|
if (jsonSocket.supportsHandoffIdReporting) {
|
|
539
581
|
this.readyWorkers.add(jsonSocket)
|
|
@@ -1125,7 +1167,7 @@ export default class BackgroundJobsMain {
|
|
|
1125
1167
|
const executionModes = this.readyWorkerExecutionModes()
|
|
1126
1168
|
|
|
1127
1169
|
if (executionModes.length === 0) return null
|
|
1128
|
-
if (executionModes.length ===
|
|
1170
|
+
if (executionModes.length === WORKER_EXECUTION_MODE_CAPABILITIES.length) return await this.store.nextAvailableJob()
|
|
1129
1171
|
|
|
1130
1172
|
return await this.store.nextAvailableJob({executionMode: executionModes})
|
|
1131
1173
|
}
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
// @ts-check
|
|
2
|
+
|
|
3
|
+
import runJobPayload, { BackgroundJobPerformedFailure } from "./job-runner.js"
|
|
4
|
+
import setRunnerProcessTitle from "./runner-process-title.js"
|
|
5
|
+
|
|
6
|
+
setRunnerProcessTitle()
|
|
7
|
+
let running = false
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Checks whether an IPC value is a runnable pooled job message.
|
|
11
|
+
* @param {?} message - IPC message.
|
|
12
|
+
* @returns {message is {type: "job", payload: import("./types.js").BackgroundJobPayload & {id: string}}} - Whether this is a valid job message.
|
|
13
|
+
*/
|
|
14
|
+
function isJobMessage(message) {
|
|
15
|
+
if (!message || typeof message !== "object") return false
|
|
16
|
+
const record = /** @type {{type?: ?, payload?: ?}} */ (message)
|
|
17
|
+
|
|
18
|
+
return record.type === "job" && !!record.payload && typeof record.payload === "object" && typeof record.payload.id === "string"
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* Sends the terminal outcome after the main/DB report has been acknowledged or rejected.
|
|
23
|
+
* @param {object} args - Outcome.
|
|
24
|
+
* @param {string} args.jobId - Job id.
|
|
25
|
+
* @param {boolean} args.acknowledged - Whether the terminal report was acknowledged.
|
|
26
|
+
* @param {"completed" | "failed"} [args.status] - Acknowledged terminal status.
|
|
27
|
+
* @param {Error} [args.error] - Reporting error when acknowledgement was not obtained.
|
|
28
|
+
* @returns {Promise<void>} - Resolves after IPC accepts the message.
|
|
29
|
+
*/
|
|
30
|
+
function sendOutcome({jobId, acknowledged, status, error}) {
|
|
31
|
+
return new Promise((resolve) => {
|
|
32
|
+
if (!process.send) {
|
|
33
|
+
resolve(undefined)
|
|
34
|
+
return
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
process.send({
|
|
38
|
+
type: "job-outcome",
|
|
39
|
+
jobId,
|
|
40
|
+
acknowledged,
|
|
41
|
+
status,
|
|
42
|
+
rssBytes: process.memoryUsage().rss,
|
|
43
|
+
error: error?.message
|
|
44
|
+
}, () => resolve(undefined))
|
|
45
|
+
})
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Handles one serial job message.
|
|
50
|
+
* @param {?} message - IPC message.
|
|
51
|
+
* @returns {Promise<void>} - Resolves after reporting.
|
|
52
|
+
*/
|
|
53
|
+
async function handleMessage(message) {
|
|
54
|
+
if (running || !isJobMessage(message)) return
|
|
55
|
+
|
|
56
|
+
running = true
|
|
57
|
+
try {
|
|
58
|
+
await runJobPayload(message.payload, {closeConnections: false})
|
|
59
|
+
await sendOutcome({jobId: message.payload.id, acknowledged: true, status: "completed"})
|
|
60
|
+
} catch (error) {
|
|
61
|
+
if (error instanceof BackgroundJobPerformedFailure) {
|
|
62
|
+
await sendOutcome({jobId: message.payload.id, acknowledged: true, status: "failed"})
|
|
63
|
+
} else {
|
|
64
|
+
const reportError = error instanceof Error ? error : new Error(String(error))
|
|
65
|
+
console.error("Pooled background job runner failed before terminal acknowledgement:", reportError)
|
|
66
|
+
await sendOutcome({jobId: message.payload.id, acknowledged: false, error: reportError})
|
|
67
|
+
process.exitCode = 1
|
|
68
|
+
if (process.disconnect) process.disconnect()
|
|
69
|
+
setImmediate(() => process.exit(1))
|
|
70
|
+
}
|
|
71
|
+
} finally {
|
|
72
|
+
running = false
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
process.on("message", (message) => { void handleMessage(message) })
|
|
77
|
+
process.once("disconnect", () => process.exit(0))
|
|
78
|
+
for (const signal of ["SIGTERM", "SIGINT"]) process.once(signal, () => process.exit(1))
|
|
@@ -81,6 +81,8 @@ export default class BackgroundJobsScheduler {
|
|
|
81
81
|
* Narrows the runtime value to the documented type.
|
|
82
82
|
* @type {Array<ReturnType<typeof setTimeout>>} */
|
|
83
83
|
this.timeoutIds = []
|
|
84
|
+
/** @type {Set<Promise<void>>} - Scheduled enqueues that shutdown must drain. */
|
|
85
|
+
this.pendingEnqueues = new Set()
|
|
84
86
|
/**
|
|
85
87
|
* Narrows the runtime value to the documented type.
|
|
86
88
|
* @type {boolean} - True between stop() and the next start(); cron self-rescheduler checks this so a stop() during an in-flight enqueue doesn't immediately re-arm.
|
|
@@ -113,8 +115,9 @@ export default class BackgroundJobsScheduler {
|
|
|
113
115
|
|
|
114
116
|
/**
|
|
115
117
|
* Runs stop.
|
|
116
|
-
* @returns {void}
|
|
117
|
-
|
|
118
|
+
* @returns {Promise<void>} Resolves after in-flight scheduled enqueues finish.
|
|
119
|
+
*/
|
|
120
|
+
async stop() {
|
|
118
121
|
this.stopped = true
|
|
119
122
|
|
|
120
123
|
for (const intervalId of this.intervalIds) {
|
|
@@ -127,6 +130,8 @@ export default class BackgroundJobsScheduler {
|
|
|
127
130
|
|
|
128
131
|
this.intervalIds = []
|
|
129
132
|
this.timeoutIds = []
|
|
133
|
+
|
|
134
|
+
await Promise.all(this.pendingEnqueues)
|
|
130
135
|
}
|
|
131
136
|
|
|
132
137
|
/**
|
|
@@ -176,10 +181,10 @@ export default class BackgroundJobsScheduler {
|
|
|
176
181
|
}
|
|
177
182
|
|
|
178
183
|
const timeoutId = setTimeout(() => {
|
|
179
|
-
void this.
|
|
184
|
+
void this.runScheduledJob({jobConfiguration, jobKey})
|
|
180
185
|
|
|
181
186
|
const intervalId = setInterval(() => {
|
|
182
|
-
void this.
|
|
187
|
+
void this.runScheduledJob({jobConfiguration, jobKey})
|
|
183
188
|
}, intervalMs)
|
|
184
189
|
|
|
185
190
|
this.intervalIds.push(intervalId)
|
|
@@ -214,7 +219,7 @@ export default class BackgroundJobsScheduler {
|
|
|
214
219
|
const timeoutId = setTimeout(async () => {
|
|
215
220
|
if (this.stopped) return
|
|
216
221
|
|
|
217
|
-
await this.
|
|
222
|
+
await this.runScheduledJob({jobConfiguration, jobKey})
|
|
218
223
|
|
|
219
224
|
// The await above can yield to a stop() call. Re-check before
|
|
220
225
|
// re-arming so we don't keep firing after shutdown.
|
|
@@ -229,6 +234,24 @@ export default class BackgroundJobsScheduler {
|
|
|
229
234
|
scheduleNext()
|
|
230
235
|
}
|
|
231
236
|
|
|
237
|
+
/**
|
|
238
|
+
* Tracks a scheduled enqueue so stop() cannot return while it can still mutate the store.
|
|
239
|
+
* @param {object} args - Options.
|
|
240
|
+
* @param {import("../configuration-types.js").ScheduledBackgroundJobConfiguration} args.jobConfiguration - Job configuration.
|
|
241
|
+
* @param {string} args.jobKey - Job key.
|
|
242
|
+
* @returns {Promise<void>} - Resolves after the enqueue attempt finishes.
|
|
243
|
+
*/
|
|
244
|
+
async runScheduledJob({jobConfiguration, jobKey}) {
|
|
245
|
+
const pendingEnqueue = this.enqueueScheduledJob({jobConfiguration, jobKey})
|
|
246
|
+
this.pendingEnqueues.add(pendingEnqueue)
|
|
247
|
+
|
|
248
|
+
try {
|
|
249
|
+
await pendingEnqueue
|
|
250
|
+
} finally {
|
|
251
|
+
this.pendingEnqueues.delete(pendingEnqueue)
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
|
|
232
255
|
/**
|
|
233
256
|
* Runs enqueue scheduled job.
|
|
234
257
|
* @param {object} args - Options.
|
|
@@ -18,8 +18,25 @@ const ORPHANED_AFTER_MS = 2 * 60 * 60 * 1000
|
|
|
18
18
|
/**
|
|
19
19
|
* Execution modes.
|
|
20
20
|
* @type {import("./types.js").BackgroundJobExecutionMode[]} */
|
|
21
|
-
const EXECUTION_MODES = ["inline", "forked", "spawned"]
|
|
22
|
-
|
|
21
|
+
const EXECUTION_MODES = ["inline", "forked", "pooled", "spawned"]
|
|
22
|
+
/**
|
|
23
|
+
* Execution mode for a new enqueue that names neither `executionMode` nor the
|
|
24
|
+
* legacy `forked` flag. Pooled routes the job to a warm, reused local runner
|
|
25
|
+
* process — the same isolation as forked without paying a fresh process per job.
|
|
26
|
+
* @type {import("./types.js").BackgroundJobExecutionMode} */
|
|
27
|
+
const DEFAULT_EXECUTION_MODE = "pooled"
|
|
28
|
+
/**
|
|
29
|
+
* Execution mode a legacy `forked = true` row (persisted before the
|
|
30
|
+
* `execution_mode` column existed) backfills to. It must stay `"forked"` so an
|
|
31
|
+
* upgrade never silently reinterprets already-persisted forked jobs as pooled —
|
|
32
|
+
* distinct from {@link DEFAULT_EXECUTION_MODE}, which only governs new enqueues.
|
|
33
|
+
* @type {import("./types.js").BackgroundJobExecutionMode} */
|
|
34
|
+
const LEGACY_FORKED_EXECUTION_MODE = "forked"
|
|
35
|
+
// Pooled jobs persist with the legacy-safe `forked` mode so a pre-pool main can
|
|
36
|
+
// deserialize and run them during a rolling upgrade. Old versions ignore the
|
|
37
|
+
// nullable handoff id on queued rows, making it a backward-compatible marker.
|
|
38
|
+
const POOLED_HANDOFF_ID_PREFIX = "velocious-pooled:"
|
|
39
|
+
const POOLED_QUEUED_HANDOFF_ID = `${POOLED_HANDOFF_ID_PREFIX}queued`
|
|
23
40
|
const DEFAULT_QUEUE = "default"
|
|
24
41
|
// Queue-derived durable concurrency keys are namespaced so they can't collide
|
|
25
42
|
// with explicit caller-supplied concurrencyKeys.
|
|
@@ -138,7 +155,7 @@ export default class BackgroundJobsStore {
|
|
|
138
155
|
job_name: jobName,
|
|
139
156
|
args_json: argsJson,
|
|
140
157
|
forked: executionMode !== "inline",
|
|
141
|
-
execution_mode: executionMode,
|
|
158
|
+
execution_mode: executionMode === "pooled" ? LEGACY_FORKED_EXECUTION_MODE : executionMode,
|
|
142
159
|
queue,
|
|
143
160
|
max_retries: maxRetries,
|
|
144
161
|
attempts: 0,
|
|
@@ -146,7 +163,8 @@ export default class BackgroundJobsStore {
|
|
|
146
163
|
scheduled_at_ms: scheduledAtMs,
|
|
147
164
|
created_at_ms: now,
|
|
148
165
|
concurrency_key: concurrency?.concurrencyKey || null,
|
|
149
|
-
max_concurrency: concurrency?.maxConcurrency || null
|
|
166
|
+
max_concurrency: concurrency?.maxConcurrency || null,
|
|
167
|
+
handoff_id: executionMode === "pooled" ? POOLED_QUEUED_HANDOFF_ID : null
|
|
150
168
|
}
|
|
151
169
|
})
|
|
152
170
|
})
|
|
@@ -221,11 +239,7 @@ export default class BackgroundJobsStore {
|
|
|
221
239
|
if (typeof forked === "boolean") {
|
|
222
240
|
query = query.where({forked})
|
|
223
241
|
}
|
|
224
|
-
if (executionMode) {
|
|
225
|
-
const executionModes = Array.isArray(executionMode) ? executionMode : [executionMode]
|
|
226
|
-
|
|
227
|
-
query = query.where({execution_mode: executionModes})
|
|
228
|
-
}
|
|
242
|
+
if (executionMode) query = this._whereExecutionMode({db, executionMode, query})
|
|
229
243
|
|
|
230
244
|
if (scheduledAtOperator === "<=") {
|
|
231
245
|
const priorityOrder = this._queuePriorityOrderSql(db)
|
|
@@ -401,12 +415,14 @@ export default class BackgroundJobsStore {
|
|
|
401
415
|
await this.ensureReady()
|
|
402
416
|
|
|
403
417
|
const handedOffAtMs = Date.now()
|
|
404
|
-
const handoffId = randomUUID()
|
|
405
418
|
|
|
406
419
|
return await this._withDb(async (db) => await this._transactionResult(db, async () => {
|
|
407
420
|
const queuedJob = await this._getJobRowById(db, jobId)
|
|
408
421
|
if (!queuedJob || queuedJob.status !== "queued") return null
|
|
409
422
|
if (queuedJob.concurrencyKey && !(await this._reserveConcurrency(db, queuedJob.concurrencyKey))) return null
|
|
423
|
+
const handoffId = queuedJob.executionMode === "pooled"
|
|
424
|
+
? `${POOLED_HANDOFF_ID_PREFIX}${randomUUID()}`
|
|
425
|
+
: randomUUID()
|
|
410
426
|
|
|
411
427
|
const affectedRows = await this._updateAffectedRows(db, {
|
|
412
428
|
tableName: JOBS_TABLE,
|
|
@@ -480,7 +496,7 @@ export default class BackgroundJobsStore {
|
|
|
480
496
|
status: "queued",
|
|
481
497
|
scheduled_at_ms: Date.now(),
|
|
482
498
|
handed_off_at_ms: null,
|
|
483
|
-
handoff_id: null,
|
|
499
|
+
handoff_id: job.executionMode === "pooled" ? POOLED_QUEUED_HANDOFF_ID : null,
|
|
484
500
|
worker_id: null
|
|
485
501
|
},
|
|
486
502
|
conditions: {handoff_id: handoffId, id: jobId, status: "handed_off"}
|
|
@@ -990,7 +1006,7 @@ export default class BackgroundJobsStore {
|
|
|
990
1006
|
const executionModeColumnSql = db.quoteColumn("execution_mode")
|
|
991
1007
|
|
|
992
1008
|
await db.query(
|
|
993
|
-
`UPDATE ${tableNameSql} SET ${executionModeColumnSql} = ${db.quote(
|
|
1009
|
+
`UPDATE ${tableNameSql} SET ${executionModeColumnSql} = ${db.quote(LEGACY_FORKED_EXECUTION_MODE)} ` +
|
|
994
1010
|
`WHERE ${forkedColumnSql} = ${db.quote(true)} AND ${executionModeColumnSql} IS NULL`
|
|
995
1011
|
)
|
|
996
1012
|
await db.query(
|
|
@@ -1081,6 +1097,7 @@ export default class BackgroundJobsStore {
|
|
|
1081
1097
|
scheduledAt,
|
|
1082
1098
|
shouldRetry
|
|
1083
1099
|
})
|
|
1100
|
+
if (shouldRetry && job.executionMode === "pooled") update.handoff_id = POOLED_QUEUED_HANDOFF_ID
|
|
1084
1101
|
|
|
1085
1102
|
const affectedRows = await this._updateAffectedRows(db, {
|
|
1086
1103
|
tableName: JOBS_TABLE,
|
|
@@ -1189,8 +1206,10 @@ export default class BackgroundJobsStore {
|
|
|
1189
1206
|
* @returns {import("./types.js").BackgroundJobRow} - Normalized job row.
|
|
1190
1207
|
*/
|
|
1191
1208
|
_normalizeJobRow(row) {
|
|
1192
|
-
const
|
|
1193
|
-
|
|
1209
|
+
const persistedExecutionMode = row.execution_mode ? String(row.execution_mode) : null
|
|
1210
|
+
const handoffId = row.handoff_id ? String(row.handoff_id) : null
|
|
1211
|
+
const executionMode = persistedExecutionMode
|
|
1212
|
+
? this._normalizePersistedExecutionMode({executionMode: persistedExecutionMode, handoffId})
|
|
1194
1213
|
: this._normalizeExecutionMode({forked: this._normalizeBoolean(row.forked)})
|
|
1195
1214
|
|
|
1196
1215
|
return {
|
|
@@ -1206,7 +1225,7 @@ export default class BackgroundJobsStore {
|
|
|
1206
1225
|
scheduledAtMs: this._normalizeNumber(row.scheduled_at_ms),
|
|
1207
1226
|
createdAtMs: this._normalizeNumber(row.created_at_ms),
|
|
1208
1227
|
handedOffAtMs: this._normalizeNumber(row.handed_off_at_ms),
|
|
1209
|
-
handoffId
|
|
1228
|
+
handoffId,
|
|
1210
1229
|
completedAtMs: this._normalizeNumber(row.completed_at_ms),
|
|
1211
1230
|
failedAtMs: this._normalizeNumber(row.failed_at_ms),
|
|
1212
1231
|
orphanedAtMs: this._normalizeNumber(row.orphaned_at_ms),
|
|
@@ -1506,7 +1525,11 @@ export default class BackgroundJobsStore {
|
|
|
1506
1525
|
if (executionMode) {
|
|
1507
1526
|
return this._normalizeExecutionModeName(executionMode)
|
|
1508
1527
|
}
|
|
1528
|
+
// The legacy `forked` flag stays authoritative when supplied: `false` is
|
|
1529
|
+
// inline and `true` is forked (never the pooled default). Only an enqueue
|
|
1530
|
+
// that names neither option falls through to the pooled default.
|
|
1509
1531
|
if (options?.forked === false) return "inline"
|
|
1532
|
+
if (options?.forked === true) return LEGACY_FORKED_EXECUTION_MODE
|
|
1510
1533
|
|
|
1511
1534
|
return DEFAULT_EXECUTION_MODE
|
|
1512
1535
|
}
|
|
@@ -1524,6 +1547,47 @@ export default class BackgroundJobsStore {
|
|
|
1524
1547
|
throw new Error(`Invalid background job executionMode: ${executionMode}`)
|
|
1525
1548
|
}
|
|
1526
1549
|
|
|
1550
|
+
/**
|
|
1551
|
+
* Normalizes a legacy-safe persisted execution mode.
|
|
1552
|
+
* @param {object} args - Options.
|
|
1553
|
+
* @param {string} args.executionMode - Persisted legacy execution mode.
|
|
1554
|
+
* @param {string | null} args.handoffId - Persisted handoff id or pooled marker.
|
|
1555
|
+
* @returns {import("./types.js").BackgroundJobExecutionMode} - Runtime execution mode.
|
|
1556
|
+
*/
|
|
1557
|
+
_normalizePersistedExecutionMode({executionMode, handoffId}) {
|
|
1558
|
+
if (executionMode === LEGACY_FORKED_EXECUTION_MODE && handoffId?.startsWith(POOLED_HANDOFF_ID_PREFIX)) return "pooled"
|
|
1559
|
+
|
|
1560
|
+
return this._normalizeExecutionModeName(executionMode)
|
|
1561
|
+
}
|
|
1562
|
+
|
|
1563
|
+
/**
|
|
1564
|
+
* Filters persisted legacy-safe execution modes.
|
|
1565
|
+
* @param {object} args - Options.
|
|
1566
|
+
* @param {import("../database/drivers/base.js").default} args.db - Database connection.
|
|
1567
|
+
* @param {import("./types.js").BackgroundJobExecutionMode | import("./types.js").BackgroundJobExecutionMode[]} args.executionMode - Runtime modes.
|
|
1568
|
+
* @param {import("../database/query/index.js").default} args.query - Query to filter.
|
|
1569
|
+
* @returns {import("../database/query/index.js").default} - Filtered query.
|
|
1570
|
+
*/
|
|
1571
|
+
_whereExecutionMode({db, executionMode, query}) {
|
|
1572
|
+
const executionModes = Array.isArray(executionMode) ? executionMode : [executionMode]
|
|
1573
|
+
/** @type {string[]} */
|
|
1574
|
+
const conditions = []
|
|
1575
|
+
const executionModeColumn = db.quoteColumn("execution_mode")
|
|
1576
|
+
const handoffIdColumn = db.quoteColumn("handoff_id")
|
|
1577
|
+
|
|
1578
|
+
for (const mode of executionModes) {
|
|
1579
|
+
if (mode === "pooled") {
|
|
1580
|
+
conditions.push(`(${executionModeColumn} = ${db.quote(LEGACY_FORKED_EXECUTION_MODE)} AND ${handoffIdColumn} = ${db.quote(POOLED_QUEUED_HANDOFF_ID)})`)
|
|
1581
|
+
} else if (mode === LEGACY_FORKED_EXECUTION_MODE) {
|
|
1582
|
+
conditions.push(`(${executionModeColumn} = ${db.quote(mode)} AND (${handoffIdColumn} IS NULL OR ${handoffIdColumn} <> ${db.quote(POOLED_QUEUED_HANDOFF_ID)}))`)
|
|
1583
|
+
} else {
|
|
1584
|
+
conditions.push(`${executionModeColumn} = ${db.quote(mode)}`)
|
|
1585
|
+
}
|
|
1586
|
+
}
|
|
1587
|
+
|
|
1588
|
+
return query.where(`(${conditions.join(" OR ")})`)
|
|
1589
|
+
}
|
|
1590
|
+
|
|
1527
1591
|
/**
|
|
1528
1592
|
* Runs parse args.
|
|
1529
1593
|
* @param {?} value - Input value.
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// @ts-check
|
|
2
2
|
|
|
3
3
|
/**
|
|
4
|
-
* @typedef {"inline" | "forked" | "spawned"} BackgroundJobExecutionMode
|
|
4
|
+
* @typedef {"inline" | "forked" | "pooled" | "spawned"} BackgroundJobExecutionMode
|
|
5
5
|
*/
|
|
6
6
|
/**
|
|
7
7
|
* @typedef {object} BackgroundJobHandoff
|
|
@@ -10,8 +10,8 @@
|
|
|
10
10
|
*/
|
|
11
11
|
/**
|
|
12
12
|
* @typedef {object} BackgroundJobOptions
|
|
13
|
-
* @property {BackgroundJobExecutionMode} [executionMode] - How the job should run. Defaults to `"forked"
|
|
14
|
-
* @property {boolean} [forked] - Compatibility alias: `false` maps to `"inline"` and `true` maps to `"forked"`.
|
|
13
|
+
* @property {BackgroundJobExecutionMode} [executionMode] - How the job should run. Defaults to `"pooled"` (a warm, reused local runner process). `"forked"` runs the job in a fresh `child_process.fork()` child, `"spawned"` in a detached CLI runner, and `"inline"` inside the worker process.
|
|
14
|
+
* @property {boolean} [forked] - Compatibility alias: `false` maps to `"inline"` and `true` maps to `"forked"`. Omitting both `forked` and `executionMode` uses the default `"pooled"` mode.
|
|
15
15
|
* @property {number} [maxRetries] - Max retries for a failed job before it is marked failed.
|
|
16
16
|
* @property {string} [queue] - Queue name. Defaults to `"default"`. When the queue has a configured cap in `backgroundJobs.queues`, that cap is enforced cluster-wide.
|
|
17
17
|
* @property {string} [concurrencyKey] - Opaque non-empty key used to share a concurrency cap. Overrides any queue-derived cap.
|
|
@@ -67,8 +67,8 @@
|
|
|
67
67
|
* @typedef {"worker" | "client" | "reporter"} BackgroundJobSocketRole
|
|
68
68
|
*/
|
|
69
69
|
/**
|
|
70
|
-
* @typedef {{type: "hello", role: BackgroundJobSocketRole, supportsHandoffIdReporting?: boolean, supportsHeartbeat?: boolean, workerId?: string}} BackgroundJobHelloMessage
|
|
71
|
-
* @typedef {{type: "ready", acceptsForked?: boolean, acceptsInline?: boolean, acceptsSpawned?: boolean}} BackgroundJobReadyMessage
|
|
70
|
+
* @typedef {{type: "hello", role: BackgroundJobSocketRole, supportsHandoffIdReporting?: boolean, supportsHeartbeat?: boolean, supportsPooled?: boolean, workerId?: string}} BackgroundJobHelloMessage
|
|
71
|
+
* @typedef {{type: "ready", acceptsForked?: boolean, acceptsInline?: boolean, acceptsPooled?: boolean, acceptsSpawned?: boolean}} BackgroundJobReadyMessage
|
|
72
72
|
* @typedef {{type: "draining"}} BackgroundJobDrainingMessage
|
|
73
73
|
* @typedef {{type: "heartbeat", workerId?: string}} BackgroundJobHeartbeatMessage
|
|
74
74
|
* @typedef {{type: "enqueue", jobName: string, args?: Array<?>, options?: BackgroundJobOptions}} BackgroundJobEnqueueMessage
|