velocious 1.0.523 → 1.0.525
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -2
- package/build/background-jobs/forked-runner-child.js +9 -0
- package/build/background-jobs/job-runner.js +23 -0
- package/build/background-jobs/job.js +12 -0
- package/build/background-jobs/main.js +53 -0
- package/build/background-jobs/store.js +32 -0
- package/build/background-jobs/worker.js +126 -7
- package/build/configuration-types.js +9 -0
- package/build/configuration.js +9 -1
- package/build/environment-handlers/node/cli/commands/background-jobs-main.js +3 -0
- package/build/environment-handlers/node/cli/commands/background-jobs-runner.js +4 -0
- package/build/environment-handlers/node/cli/commands/background-jobs-worker.js +3 -0
- package/build/environment-handlers/node/cli/commands/beacon.js +3 -0
- package/build/environment-handlers/node/cli/commands/server.js +3 -0
- package/build/src/background-jobs/forked-runner-child.js +9 -1
- package/build/src/background-jobs/job-runner.d.ts.map +1 -1
- package/build/src/background-jobs/job-runner.js +21 -1
- package/build/src/background-jobs/job.d.ts +11 -0
- package/build/src/background-jobs/job.d.ts.map +1 -1
- package/build/src/background-jobs/job.js +12 -1
- package/build/src/background-jobs/main.d.ts +23 -0
- package/build/src/background-jobs/main.d.ts.map +1 -1
- package/build/src/background-jobs/main.js +50 -1
- package/build/src/background-jobs/store.d.ts +19 -0
- package/build/src/background-jobs/store.d.ts.map +1 -1
- package/build/src/background-jobs/store.js +26 -1
- package/build/src/background-jobs/worker.d.ts +85 -2
- package/build/src/background-jobs/worker.d.ts.map +1 -1
- package/build/src/background-jobs/worker.js +116 -8
- package/build/src/configuration-types.d.ts +21 -0
- package/build/src/configuration-types.d.ts.map +1 -1
- package/build/src/configuration-types.js +10 -1
- package/build/src/configuration.d.ts.map +1 -1
- package/build/src/configuration.js +10 -2
- package/build/src/environment-handlers/node/cli/commands/background-jobs-main.d.ts.map +1 -1
- package/build/src/environment-handlers/node/cli/commands/background-jobs-main.js +3 -1
- package/build/src/environment-handlers/node/cli/commands/background-jobs-runner.d.ts.map +1 -1
- package/build/src/environment-handlers/node/cli/commands/background-jobs-runner.js +4 -1
- package/build/src/environment-handlers/node/cli/commands/background-jobs-worker.d.ts.map +1 -1
- package/build/src/environment-handlers/node/cli/commands/background-jobs-worker.js +3 -1
- package/build/src/environment-handlers/node/cli/commands/beacon.d.ts.map +1 -1
- package/build/src/environment-handlers/node/cli/commands/beacon.js +3 -1
- package/build/src/environment-handlers/node/cli/commands/server.d.ts.map +1 -1
- package/build/src/environment-handlers/node/cli/commands/server.js +3 -1
- package/package.json +1 -1
- package/src/background-jobs/forked-runner-child.js +9 -0
- package/src/background-jobs/job-runner.js +23 -0
- package/src/background-jobs/job.js +12 -0
- package/src/background-jobs/main.js +53 -0
- package/src/background-jobs/store.js +32 -0
- package/src/background-jobs/worker.js +126 -7
- package/src/configuration-types.js +9 -0
- package/src/configuration.js +9 -1
- package/src/environment-handlers/node/cli/commands/background-jobs-main.js +3 -0
- package/src/environment-handlers/node/cli/commands/background-jobs-runner.js +4 -0
- package/src/environment-handlers/node/cli/commands/background-jobs-worker.js +3 -0
- package/src/environment-handlers/node/cli/commands/beacon.js +3 -0
- package/src/environment-handlers/node/cli/commands/server.js +3 -0
package/README.md
CHANGED
|
@@ -1975,7 +1975,8 @@ export default new Configuration({
|
|
|
1975
1975
|
databaseIdentifier: "default",
|
|
1976
1976
|
maxConcurrentForkedJobs: 4,
|
|
1977
1977
|
maxConcurrentInlineJobs: 4,
|
|
1978
|
-
dispatchStrategy: "beacon"
|
|
1978
|
+
dispatchStrategy: "beacon",
|
|
1979
|
+
jobTimeoutMs: null
|
|
1979
1980
|
}
|
|
1980
1981
|
})
|
|
1981
1982
|
```
|
|
@@ -1991,6 +1992,7 @@ VELOCIOUS_BACKGROUND_JOBS_MAX_CONCURRENT_INLINE_JOBS=4
|
|
|
1991
1992
|
VELOCIOUS_BACKGROUND_JOBS_DISPATCH_STRATEGY=beacon
|
|
1992
1993
|
VELOCIOUS_BACKGROUND_JOBS_POLL_INTERVAL_MS=1000
|
|
1993
1994
|
VELOCIOUS_BACKGROUND_JOBS_WORKER_SHUTDOWN_TIMEOUT_MS=indefinite
|
|
1995
|
+
VELOCIOUS_BACKGROUND_JOBS_JOB_TIMEOUT_MS=5400000
|
|
1994
1996
|
```
|
|
1995
1997
|
|
|
1996
1998
|
`VELOCIOUS_BACKGROUND_JOBS_WORKER_SHUTDOWN_TIMEOUT_MS` (default: `indefinite`) bounds how long a `background-jobs-worker` waits for in-flight jobs on `SIGTERM`/`SIGINT` before terminating any forked or spawned child runners still running, so they are not orphaned across a deploy. The default waits for jobs to finish and never interrupts a running job; set a positive number of milliseconds for a finite cap (keep it shorter than your process supervisor's graceful-stop window so the worker reaps its own runners first). See [docs/background-jobs.md](docs/background-jobs.md#worker-shutdown-and-process-job-draining).
|
|
@@ -1999,6 +2001,8 @@ VELOCIOUS_BACKGROUND_JOBS_WORKER_SHUTDOWN_TIMEOUT_MS=indefinite
|
|
|
1999
2001
|
|
|
2000
2002
|
`maxConcurrentForkedJobs` (default: `4`) caps how many out-of-process `executionMode: "forked"` or `executionMode: "spawned"` jobs one worker may keep in flight. Forked jobs use `child_process.fork()` with an attached IPC channel. After the main process acknowledges their durable status report, forked and spawned one-shot runners exit without waiting for graceful Beacon/database teardown; the OS closes their process-owned resources. A missing or rejected status acknowledgement makes the runner exit as failed instead of reporting clean success. Spawned jobs use the legacy `background-jobs-runner` CLI process via `child_process.spawn()` and are only for callers that intentionally want that spawned behavior.
|
|
2001
2003
|
|
|
2004
|
+
`jobTimeoutMs` (or `VELOCIOUS_BACKGROUND_JOBS_JOB_TIMEOUT_MS`, milliseconds; default: disabled) is a wall-clock backstop for a single `"forked"` runner. A forked job still running after the timeout is terminated (`SIGTERM`, then `SIGKILL` after the reaping grace) and reported `failed`, so a genuinely-hung runner can't pin a worker — and its whole-app boot and DB connections — indefinitely (notably a retired-release worker draining after a deploy). It's a coarse safety net, not per-job tuning: it applies to every forked runner, so set it well above the longest legitimate forked job. Omit it, or set `null`/`<= 0`, to disable. `"inline"` jobs are not covered — they share the worker's process and can't be killed without killing the worker. See [docs/background-jobs.md](docs/background-jobs.md#forked-job-timeout-hung-runner-backstop).
|
|
2005
|
+
|
|
2002
2006
|
### Dispatch strategy
|
|
2003
2007
|
|
|
2004
2008
|
`dispatchStrategy` controls how `background-jobs-main` detects new work.
|
|
@@ -2118,7 +2122,7 @@ Each job must define exactly one of `every` or `cron`. Cron times are evaluated
|
|
|
2118
2122
|
|
|
2119
2123
|
## Persistence and retries
|
|
2120
2124
|
|
|
2121
|
-
Jobs are persisted in the configured database (`backgroundJobs.databaseIdentifier`) in an internal `background_jobs` table. When a worker picks a job, the job is marked as handed off with a unique lease id and the worker reports completion or failure back to the main process. If that worker socket disconnects unexpectedly, only the leases handed to that exact socket are immediately returned to the queue; late reports are fenced by lease id so they cannot mutate a newer attempt. This recovery is at-least-once and may repeat application side effects if the disconnected attempt had already started them. Gracefully draining workers keep their leases while they finish. For rolling upgrades, upgrade workers before the main process: a lease-aware main keeps legacy workers connected for old reports but dispatches new jobs only to workers advertising lease-reporting support. See [docs/background-jobs.md](docs/background-jobs.md#worker-disconnect-recovery).
|
|
2125
|
+
Jobs are persisted in the configured database (`backgroundJobs.databaseIdentifier`) in an internal `background_jobs` table. When a worker picks a job, the job is marked as handed off with a unique lease id and the worker reports completion or failure back to the main process. If that worker socket disconnects unexpectedly, only the leases handed to that exact socket are immediately returned to the queue; late reports are fenced by lease id so they cannot mutate a newer attempt. This recovery is at-least-once and may repeat application side effects if the disconnected attempt had already started them. Gracefully draining workers keep their leases while they finish. When the **main** itself restarts (every deploy), a reconnecting worker's still-active handoffs are adopted into its new socket on `hello` so a later disconnect releases them instead of leaving them stuck until the orphan sweep; the main never time-reclaims a disconnected worker's jobs, so a gracefully-draining old-release worker is not double-run. For rolling upgrades, upgrade workers before the main process: a lease-aware main keeps legacy workers connected for old reports but dispatches new jobs only to workers advertising lease-reporting support. See [docs/background-jobs.md](docs/background-jobs.md#worker-disconnect-recovery).
|
|
2122
2126
|
|
|
2123
2127
|
Failed jobs are re-queued with backoff and retried up to 10 times by default (10s, 1m, 10m, 1h, then +1h per retry). You can override the retry limit per job:
|
|
2124
2128
|
|
|
@@ -2133,6 +2137,8 @@ If a handed-off job does not report back within 2 hours, it is marked orphaned a
|
|
|
2133
2137
|
|
|
2134
2138
|
Workers also send periodic heartbeats (and use TCP keepalive) so the main can drop a wedged or half-open worker that never fires a socket `close` and release its leases, and job slots are freed independently of durable, background result reporting so a transient outage can't wedge a worker or lose a completion. See [docs/background-jobs.md](docs/background-jobs.md#worker-liveness).
|
|
2135
2139
|
|
|
2140
|
+
Every background-jobs process sets a descriptive `process.title` (`velocious background-jobs-main`/`-worker`/`server`/`beacon`), and each forked/spawned job runner is named after the job it runs (`velocious job-runner: <JobName>`, or a job class's `static processTitle`), so `ps`/`top` show which jobs are consuming resources. See [docs/background-jobs.md](docs/background-jobs.md#process-titles).
|
|
2141
|
+
|
|
2136
2142
|
## Queues
|
|
2137
2143
|
|
|
2138
2144
|
Give a job a queue with `static queue = "..."` (or a `{queue}` job option; the option wins) and cap how many of that queue's jobs run in flight across the whole cluster under `backgroundJobs.queues`:
|
|
@@ -2,6 +2,12 @@
|
|
|
2
2
|
|
|
3
3
|
import runJobPayload from "./job-runner.js"
|
|
4
4
|
|
|
5
|
+
// Name the process so `ps`/`top`/`htop` can identify forked job runners at a
|
|
6
|
+
// glance instead of a wall of generic "node" entries. Updated to the specific
|
|
7
|
+
// job name once one arrives (see runJobMessage), so operators can see exactly
|
|
8
|
+
// which jobs are running, how many of each, and which are eating resources.
|
|
9
|
+
process.title = "velocious background-jobs-runner"
|
|
10
|
+
|
|
5
11
|
let finishing = false
|
|
6
12
|
|
|
7
13
|
/**
|
|
@@ -51,6 +57,9 @@ async function runJobMessage(message) {
|
|
|
51
57
|
throw new Error("Forked background job runner received invalid payload")
|
|
52
58
|
}
|
|
53
59
|
|
|
60
|
+
// The per-job process title (and its restore) is set inside runJobPayload,
|
|
61
|
+
// which reads the job class's `static processTitle`. This process boots with
|
|
62
|
+
// the base "velocious background-jobs-runner" title set at module load above.
|
|
54
63
|
await runJobPayload(message.payload, {closeConnections: false})
|
|
55
64
|
}
|
|
56
65
|
|
|
@@ -47,6 +47,21 @@ async function connectBeacon(configuration) {
|
|
|
47
47
|
}
|
|
48
48
|
}
|
|
49
49
|
|
|
50
|
+
/**
|
|
51
|
+
* Resolves the process title to show while a job runs: the job class's declared
|
|
52
|
+
* `static processTitle`, else a `velocious job-runner: <JobName>` fallback.
|
|
53
|
+
* @param {typeof import("./job.js").default} JobClass - Resolved job class.
|
|
54
|
+
* @param {import("./types.js").BackgroundJobPayload} payload - Payload.
|
|
55
|
+
* @returns {string} - Process title.
|
|
56
|
+
*/
|
|
57
|
+
function runnerProcessTitle(JobClass, payload) {
|
|
58
|
+
const declared = JobClass.processTitle
|
|
59
|
+
|
|
60
|
+
if (typeof declared === "string" && declared.length > 0) return declared
|
|
61
|
+
|
|
62
|
+
return `velocious job-runner: ${payload.jobName}`
|
|
63
|
+
}
|
|
64
|
+
|
|
50
65
|
/**
|
|
51
66
|
* Runs run job payload.
|
|
52
67
|
* @param {import("./types.js").BackgroundJobPayload} payload - Payload.
|
|
@@ -70,6 +85,11 @@ export default async function runJobPayload(payload, {closeConnections = true} =
|
|
|
70
85
|
* @type {(...args: Array<?>) => Promise<void>} */
|
|
71
86
|
const perform = jobInstance.perform
|
|
72
87
|
|
|
88
|
+
// Name the process after the job it is running so `ps`/`top` show what each
|
|
89
|
+
// runner is doing; restored in the `finally` below when the job finishes.
|
|
90
|
+
const previousTitle = process.title
|
|
91
|
+
process.title = runnerProcessTitle(JobClass, payload)
|
|
92
|
+
|
|
73
93
|
try {
|
|
74
94
|
try {
|
|
75
95
|
await configuration.withConnections({name: `Background job runner: ${payload.jobName}`}, async () => {
|
|
@@ -102,6 +122,9 @@ export default async function runJobPayload(payload, {closeConnections = true} =
|
|
|
102
122
|
})
|
|
103
123
|
}
|
|
104
124
|
} finally {
|
|
125
|
+
// Restore the runner's base title so a lingering/idle runner (or a reused
|
|
126
|
+
// one) doesn't misreport a finished job as still running.
|
|
127
|
+
process.title = previousTitle
|
|
105
128
|
if (closeConnections) {
|
|
106
129
|
try {
|
|
107
130
|
await configuration.disconnectBeacon()
|
|
@@ -22,6 +22,18 @@ export default class VelociousJob {
|
|
|
22
22
|
*/
|
|
23
23
|
static queue = undefined
|
|
24
24
|
|
|
25
|
+
/**
|
|
26
|
+
* Optional process title shown for the runner while this job executes.
|
|
27
|
+
* Velocious sets `process.title` to this for the duration of the job — so
|
|
28
|
+
* `ps`/`top`/`htop` identify what a runner is doing — and restores the
|
|
29
|
+
* runner's base title when the job finishes. Left undefined, the runner falls
|
|
30
|
+
* back to `velocious job-runner: <JobName>`. Set e.g.
|
|
31
|
+
* `static processTitle = "velocious media transcoder"` to give a job a
|
|
32
|
+
* custom, human-readable title.
|
|
33
|
+
* @type {string | undefined}
|
|
34
|
+
*/
|
|
35
|
+
static processTitle = undefined
|
|
36
|
+
|
|
25
37
|
/**
|
|
26
38
|
* Runs job name.
|
|
27
39
|
* @returns {string} - Job name.
|
|
@@ -422,11 +422,47 @@ export default class BackgroundJobsMain {
|
|
|
422
422
|
jsonSocket.lastSeenAt = Date.now()
|
|
423
423
|
this.workers.add(jsonSocket)
|
|
424
424
|
this.workerHandoffs.set(jsonSocket, new Map())
|
|
425
|
+
void this._adoptWorkerHandoffs(jsonSocket)
|
|
425
426
|
}
|
|
426
427
|
|
|
427
428
|
return message.role
|
|
428
429
|
}
|
|
429
430
|
|
|
431
|
+
/**
|
|
432
|
+
* Adopts a reconnecting worker's still-active `handed_off` jobs into its new
|
|
433
|
+
* socket's handoff map. A fresh main (e.g. after a deploy restart) holds no
|
|
434
|
+
* in-memory leases, so a worker that reconnects with its stable id would
|
|
435
|
+
* otherwise have its pre-restart jobs tracked nowhere — if it then died, those
|
|
436
|
+
* leases (and their concurrency reservations) would sit stuck until the
|
|
437
|
+
* hours-long orphan sweep. Adopting them means `_handleWorkerSocketClosed`
|
|
438
|
+
* releases them on the worker's next disconnect, while a still-running worker
|
|
439
|
+
* (including one gracefully draining) keeps executing them untouched. No
|
|
440
|
+
* time-based reclaim is used, so a draining worker whose jobs outlive the old
|
|
441
|
+
* main is never wrongly requeued into a duplicate attempt.
|
|
442
|
+
* @param {JsonSocket} jsonSocket - The reconnected worker socket.
|
|
443
|
+
* @returns {Promise<void>}
|
|
444
|
+
*/
|
|
445
|
+
async _adoptWorkerHandoffs(jsonSocket) {
|
|
446
|
+
const workerId = jsonSocket.workerId
|
|
447
|
+
|
|
448
|
+
if (typeof workerId !== "string" || workerId.length === 0) return
|
|
449
|
+
|
|
450
|
+
try {
|
|
451
|
+
const handoffs = await this.store.handedOffJobsForWorker({workerId})
|
|
452
|
+
const map = this.workerHandoffs.get(jsonSocket)
|
|
453
|
+
|
|
454
|
+
// The socket may have closed while the query was in flight; its map is then
|
|
455
|
+
// gone and the jobs are left for the orphan sweep rather than resurrected.
|
|
456
|
+
if (!map || !this.workers.has(jsonSocket)) return
|
|
457
|
+
|
|
458
|
+
for (const {jobId, handoffId} of handoffs) {
|
|
459
|
+
map.set(jobId, handoffId)
|
|
460
|
+
}
|
|
461
|
+
} catch (error) {
|
|
462
|
+
this._reportHandoffAdoptError(error)
|
|
463
|
+
}
|
|
464
|
+
}
|
|
465
|
+
|
|
430
466
|
/**
|
|
431
467
|
* Runs handle client socket message.
|
|
432
468
|
* @param {object} args - Options.
|
|
@@ -611,6 +647,23 @@ export default class BackgroundJobsMain {
|
|
|
611
647
|
errorEvents.emit("all-error", {...payload, errorType: "framework-error"})
|
|
612
648
|
}
|
|
613
649
|
|
|
650
|
+
/**
|
|
651
|
+
* Reports an unexpected worker-handoff adoption failure on framework error
|
|
652
|
+
* channels. A failed adoption is not fatal (the worker's jobs remain and are
|
|
653
|
+
* reclaimed by the orphan sweep), but must surface rather than be swallowed.
|
|
654
|
+
* @param {?} error - Adoption failure.
|
|
655
|
+
* @returns {void}
|
|
656
|
+
*/
|
|
657
|
+
_reportHandoffAdoptError(error) {
|
|
658
|
+
const normalizedError = error instanceof Error ? error : new Error(String(error))
|
|
659
|
+
const payload = {context: {stage: "background-job-handoff-adopt"}, error: normalizedError}
|
|
660
|
+
const errorEvents = this.configuration.getErrorEvents()
|
|
661
|
+
|
|
662
|
+
this.logger.error(() => ["Failed to adopt reconnected worker handoffs:", normalizedError])
|
|
663
|
+
errorEvents.emit("framework-error", payload)
|
|
664
|
+
errorEvents.emit("all-error", {...payload, errorType: "framework-error"})
|
|
665
|
+
}
|
|
666
|
+
|
|
614
667
|
/**
|
|
615
668
|
* Runs handle enqueue.
|
|
616
669
|
* @param {object} args - Options.
|
|
@@ -487,6 +487,38 @@ export default class BackgroundJobsStore {
|
|
|
487
487
|
}))
|
|
488
488
|
}
|
|
489
489
|
|
|
490
|
+
/**
|
|
491
|
+
* Returns the active `handed_off` jobs (jobId + handoffId) held under a worker
|
|
492
|
+
* id. Used on worker reconnect: after a main restart a worker reconnects with
|
|
493
|
+
* its stable id, and the fresh main adopts these leases so they are tracked —
|
|
494
|
+
* and released if the reconnected worker later disconnects — instead of
|
|
495
|
+
* sitting stuck until the age-based orphan sweep. This never reclaims, so a
|
|
496
|
+
* gracefully-draining worker that keeps running its in-flight jobs is left
|
|
497
|
+
* untouched. Rows with a null handoff id (legacy) are skipped; the orphan
|
|
498
|
+
* sweep reclaims those via its `handed_off_at_ms` fence.
|
|
499
|
+
* @param {object} args - Options.
|
|
500
|
+
* @param {string} args.workerId - Worker id.
|
|
501
|
+
* @returns {Promise<Array<{jobId: string, handoffId: string}>>} - Active handoffs.
|
|
502
|
+
*/
|
|
503
|
+
async handedOffJobsForWorker({workerId}) {
|
|
504
|
+
await this.ensureReady()
|
|
505
|
+
|
|
506
|
+
const rows = await this._withDb(async (db) =>
|
|
507
|
+
await db.newQuery().from(JOBS_TABLE).where({status: "handed_off", worker_id: workerId}).results()
|
|
508
|
+
)
|
|
509
|
+
|
|
510
|
+
/** @type {Array<{jobId: string, handoffId: string}>} */
|
|
511
|
+
const handoffs = []
|
|
512
|
+
|
|
513
|
+
for (const row of rows) {
|
|
514
|
+
const job = this._normalizeJobRow(row)
|
|
515
|
+
|
|
516
|
+
if (job.handoffId) handoffs.push({jobId: job.id, handoffId: job.handoffId})
|
|
517
|
+
}
|
|
518
|
+
|
|
519
|
+
return handoffs
|
|
520
|
+
}
|
|
521
|
+
|
|
490
522
|
/**
|
|
491
523
|
* Runs mark failed.
|
|
492
524
|
* @param {object} args - Options.
|
|
@@ -11,6 +11,13 @@ import {fileURLToPath} from "node:url"
|
|
|
11
11
|
|
|
12
12
|
/** Grace period after SIGTERM before a lingering process runner is SIGKILLed. */
|
|
13
13
|
const FORKED_CHILD_SIGKILL_GRACE_MS = 5000
|
|
14
|
+
/**
|
|
15
|
+
* Largest delay Node's `setTimeout` accepts without overflowing to a 1ms delay
|
|
16
|
+
* (a 32-bit signed int of ms, ~24.8 days). A `jobTimeoutMs` above this — or a
|
|
17
|
+
* non-finite one like `Infinity` — is clamped/disabled rather than coerced to
|
|
18
|
+
* ~1ms, which would otherwise terminate every forked job almost immediately.
|
|
19
|
+
*/
|
|
20
|
+
const MAX_FORKED_JOB_TIMEOUT_MS = 2_147_483_647
|
|
14
21
|
const FORKED_RUNNER_ENTRY_PATH = fileURLToPath(new URL("./forked-runner-child.js", import.meta.url))
|
|
15
22
|
/** How often the worker sends a liveness heartbeat to the main. */
|
|
16
23
|
const HEARTBEAT_INTERVAL_MS = 15000
|
|
@@ -21,6 +28,15 @@ const SOCKET_KEEPALIVE_MS = 10000
|
|
|
21
28
|
* @type {import("./types.js").BackgroundJobExecutionMode[]} */
|
|
22
29
|
const EXECUTION_MODES = ["inline", "forked", "spawned"]
|
|
23
30
|
|
|
31
|
+
/**
|
|
32
|
+
* Per-forked-child timeout bookkeeping.
|
|
33
|
+
* @typedef {object} ForkedJobTimeoutState
|
|
34
|
+
* @property {boolean} timedOut - Whether the timeout fired and the child was terminated.
|
|
35
|
+
* @property {number | null} timeoutMs - The armed timeout in ms, or null when disabled.
|
|
36
|
+
* @property {ReturnType<typeof setTimeout> | null} timer - The pending timeout timer, cleared on exit.
|
|
37
|
+
* @property {ReturnType<typeof setTimeout> | null} sigkillTimer - The pending SIGKILL grace timer, cleared on exit.
|
|
38
|
+
*/
|
|
39
|
+
|
|
24
40
|
export default class BackgroundJobsWorker {
|
|
25
41
|
/**
|
|
26
42
|
* Runs constructor.
|
|
@@ -32,8 +48,9 @@ export default class BackgroundJobsWorker {
|
|
|
32
48
|
* @param {number} [args.maxConcurrentInlineJobs] - Override the inline-job concurrency cap from `configuration.getBackgroundJobsConfig()`.
|
|
33
49
|
* @param {number} [args.forkedChildSigkillGraceMs] - Override the grace period between SIGTERM and SIGKILL when reaping lingering process runners on stop.
|
|
34
50
|
* @param {number} [args.heartbeatIntervalMs] - Override the liveness heartbeat interval (default 15000ms).
|
|
51
|
+
* @param {number} [args.jobTimeoutMs] - Override the forked-job wall-clock timeout from `configuration.getBackgroundJobsConfig()`. `0` disables it.
|
|
35
52
|
*/
|
|
36
|
-
constructor({configuration, host, port, maxConcurrentForkedJobs, maxConcurrentInlineJobs, forkedChildSigkillGraceMs, heartbeatIntervalMs} = {}) {
|
|
53
|
+
constructor({configuration, host, port, maxConcurrentForkedJobs, maxConcurrentInlineJobs, forkedChildSigkillGraceMs, heartbeatIntervalMs, jobTimeoutMs} = {}) {
|
|
37
54
|
/**
|
|
38
55
|
* Narrows the runtime value to the documented type.
|
|
39
56
|
* @type {Promise<import("../configuration.js").default>} */
|
|
@@ -77,6 +94,13 @@ export default class BackgroundJobsWorker {
|
|
|
77
94
|
this.forkedChildSigkillGraceMs = typeof forkedChildSigkillGraceMs === "number" && forkedChildSigkillGraceMs >= 0
|
|
78
95
|
? forkedChildSigkillGraceMs
|
|
79
96
|
: FORKED_CHILD_SIGKILL_GRACE_MS
|
|
97
|
+
/**
|
|
98
|
+
* Constructor override for the forked-job wall-clock timeout. When unset the
|
|
99
|
+
* timeout is read from `configuration.getBackgroundJobsConfig().jobTimeoutMs`
|
|
100
|
+
* at fork time (default: disabled).
|
|
101
|
+
* @type {number | undefined}
|
|
102
|
+
*/
|
|
103
|
+
this.jobTimeoutMsOverride = typeof jobTimeoutMs === "number" ? jobTimeoutMs : undefined
|
|
80
104
|
this.shouldStop = false
|
|
81
105
|
this.workerId = randomUUID()
|
|
82
106
|
this.heartbeatIntervalMs = typeof heartbeatIntervalMs === "number" && heartbeatIntervalMs >= 1
|
|
@@ -590,16 +614,109 @@ export default class BackgroundJobsWorker {
|
|
|
590
614
|
* @returns {Promise<void>} - Resolves when the child exits.
|
|
591
615
|
*/
|
|
592
616
|
_waitForForkedChild({child, payload}) {
|
|
617
|
+
const timeoutState = this._armForkedJobTimeout({child})
|
|
618
|
+
|
|
593
619
|
return new Promise((resolve) => {
|
|
594
620
|
child.once("exit", (code, signal) => {
|
|
595
|
-
this.
|
|
621
|
+
this._clearForkedJobTimeout(timeoutState)
|
|
622
|
+
this._handleForkedChildExit({child, code, signal, payload, resolve, timeoutState})
|
|
596
623
|
})
|
|
597
624
|
child.once("error", (error) => {
|
|
625
|
+
this._clearForkedJobTimeout(timeoutState)
|
|
598
626
|
this._handleForkedChildError({child, error, payload, resolve})
|
|
599
627
|
})
|
|
600
628
|
})
|
|
601
629
|
}
|
|
602
630
|
|
|
631
|
+
/**
|
|
632
|
+
* Arms a wall-clock backstop for a forked job runner. A forked job still
|
|
633
|
+
* running after `jobTimeoutMs` is terminated (SIGTERM, then SIGKILL after the
|
|
634
|
+
* grace) so a single genuinely-hung runner can't pin a draining worker — and
|
|
635
|
+
* its full-app boot and database connections — indefinitely. Returns a state
|
|
636
|
+
* object the exit/error handlers use to cancel the timer and to report a
|
|
637
|
+
* timeout-specific failure. When no timeout is configured the timer is null
|
|
638
|
+
* and behavior is unchanged.
|
|
639
|
+
* @param {object} args - Options.
|
|
640
|
+
* @param {import("node:child_process").ChildProcess} args.child - Forked child process.
|
|
641
|
+
* @returns {ForkedJobTimeoutState} - Timeout state.
|
|
642
|
+
*/
|
|
643
|
+
_armForkedJobTimeout({child}) {
|
|
644
|
+
const timeoutMs = this._resolveForkedJobTimeoutMs()
|
|
645
|
+
/** @type {ForkedJobTimeoutState} */
|
|
646
|
+
const state = {timedOut: false, timeoutMs, timer: null, sigkillTimer: null}
|
|
647
|
+
|
|
648
|
+
if (!(typeof timeoutMs === "number" && timeoutMs > 0)) return state
|
|
649
|
+
|
|
650
|
+
state.timer = setTimeout(() => this._onForkedJobTimeout({child, state}), timeoutMs)
|
|
651
|
+
|
|
652
|
+
return state
|
|
653
|
+
}
|
|
654
|
+
|
|
655
|
+
/**
|
|
656
|
+
* Resolves the effective forked-job timeout in ms, or null when disabled. The
|
|
657
|
+
* constructor override wins; otherwise the value comes from the background-jobs
|
|
658
|
+
* configuration. A non-positive value disables the backstop.
|
|
659
|
+
* @returns {number | null} - Timeout in ms, or null when disabled.
|
|
660
|
+
*/
|
|
661
|
+
_resolveForkedJobTimeoutMs() {
|
|
662
|
+
const raw = typeof this.jobTimeoutMsOverride === "number"
|
|
663
|
+
? this.jobTimeoutMsOverride
|
|
664
|
+
: (this.configuration ? this.configuration.getBackgroundJobsConfig().jobTimeoutMs : null)
|
|
665
|
+
|
|
666
|
+
// A non-finite (e.g. Infinity) or non-positive value disables the backstop;
|
|
667
|
+
// a finite value beyond Node's timer range is clamped to the max rather than
|
|
668
|
+
// silently coerced to ~1ms (which would kill every forked job immediately).
|
|
669
|
+
if (typeof raw !== "number" || !Number.isFinite(raw) || raw <= 0) return null
|
|
670
|
+
|
|
671
|
+
return Math.min(raw, MAX_FORKED_JOB_TIMEOUT_MS)
|
|
672
|
+
}
|
|
673
|
+
|
|
674
|
+
/**
|
|
675
|
+
* Fired when a forked runner overruns its timeout. Sends SIGTERM for a clean
|
|
676
|
+
* shutdown, then SIGKILL after the grace for a runner that ignores it. The
|
|
677
|
+
* resulting non-clean exit flows through `_handleForkedChildExit`, which frees
|
|
678
|
+
* the slot and reports the job failed.
|
|
679
|
+
* @param {object} args - Options.
|
|
680
|
+
* @param {import("node:child_process").ChildProcess} args.child - Forked child process.
|
|
681
|
+
* @param {ForkedJobTimeoutState} args.state - Timeout state.
|
|
682
|
+
* @returns {void}
|
|
683
|
+
*/
|
|
684
|
+
_onForkedJobTimeout({child, state}) {
|
|
685
|
+
state.timedOut = true
|
|
686
|
+
|
|
687
|
+
try {
|
|
688
|
+
child.kill("SIGTERM")
|
|
689
|
+
} catch {
|
|
690
|
+
// Child already exited; nothing to do.
|
|
691
|
+
}
|
|
692
|
+
|
|
693
|
+
state.sigkillTimer = setTimeout(() => {
|
|
694
|
+
try {
|
|
695
|
+
child.kill("SIGKILL")
|
|
696
|
+
} catch {
|
|
697
|
+
// Child already exited; nothing to do.
|
|
698
|
+
}
|
|
699
|
+
}, this.forkedChildSigkillGraceMs)
|
|
700
|
+
}
|
|
701
|
+
|
|
702
|
+
/**
|
|
703
|
+
* Cancels any pending timeout/SIGKILL timers for a forked runner that has
|
|
704
|
+
* exited (or errored) so they never fire against a gone or reused child.
|
|
705
|
+
* @param {ForkedJobTimeoutState} state - Timeout state.
|
|
706
|
+
* @returns {void}
|
|
707
|
+
*/
|
|
708
|
+
_clearForkedJobTimeout(state) {
|
|
709
|
+
if (state.timer) {
|
|
710
|
+
clearTimeout(state.timer)
|
|
711
|
+
state.timer = null
|
|
712
|
+
}
|
|
713
|
+
|
|
714
|
+
if (state.sigkillTimer) {
|
|
715
|
+
clearTimeout(state.sigkillTimer)
|
|
716
|
+
state.sigkillTimer = null
|
|
717
|
+
}
|
|
718
|
+
}
|
|
719
|
+
|
|
603
720
|
/**
|
|
604
721
|
* Runs handle forked child exit.
|
|
605
722
|
* @param {object} args - Options.
|
|
@@ -608,9 +725,10 @@ export default class BackgroundJobsWorker {
|
|
|
608
725
|
* @param {keyof typeof import("node:os").constants.signals | null} args.signal - Exit signal.
|
|
609
726
|
* @param {import("./types.js").BackgroundJobPayload & {id: string}} args.payload - Payload.
|
|
610
727
|
* @param {(value: void) => void} args.resolve - Promise resolver.
|
|
728
|
+
* @param {ForkedJobTimeoutState} [args.timeoutState] - Timeout state, when the runner had a wall-clock backstop.
|
|
611
729
|
* @returns {void}
|
|
612
730
|
*/
|
|
613
|
-
_handleForkedChildExit({child, code, signal, payload, resolve}) {
|
|
731
|
+
_handleForkedChildExit({child, code, signal, payload, resolve, timeoutState}) {
|
|
614
732
|
this.inflightProcessChildren.delete(child)
|
|
615
733
|
|
|
616
734
|
// Free the worker slot as soon as the child is gone — never gate it on the
|
|
@@ -620,10 +738,11 @@ export default class BackgroundJobsWorker {
|
|
|
620
738
|
|
|
621
739
|
if (this._forkedChildExitedCleanly({code, signal})) return
|
|
622
740
|
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
741
|
+
const error = timeoutState?.timedOut
|
|
742
|
+
? new Error(`Forked background job runner timed out after ${timeoutState.timeoutMs}ms and was terminated: code=${code} signal=${signal || "none"}`)
|
|
743
|
+
: new Error(`Forked background job runner exited before reporting: code=${code} signal=${signal || "none"}`)
|
|
744
|
+
|
|
745
|
+
this._reportForkedChildFailure({payload, error})
|
|
627
746
|
}
|
|
628
747
|
|
|
629
748
|
/**
|
|
@@ -192,6 +192,15 @@
|
|
|
192
192
|
* to restore the legacy fixed-interval poll.
|
|
193
193
|
* @property {number} [pollIntervalMs] - Poll interval in milliseconds. Only used
|
|
194
194
|
* when `dispatchStrategy === "polling"`. Default: `1000`.
|
|
195
|
+
* @property {number | null} [jobTimeoutMs] - Wall-clock backstop, in ms, for a
|
|
196
|
+
* `"forked"` job runner. A forked job still running after this is terminated
|
|
197
|
+
* (SIGTERM, then SIGKILL after the reaping grace) and reported failed, so a
|
|
198
|
+
* single genuinely-hung runner can't pin a draining worker — and its full-app
|
|
199
|
+
* boot and DB connections — indefinitely (e.g. across a deploy where a retired
|
|
200
|
+
* worker drains in-flight jobs). This is a coarse safety net, not per-job
|
|
201
|
+
* tuning: set it well above the longest legitimate forked job (build runners,
|
|
202
|
+
* long imports) so only genuinely-stuck runners are killed. Omit, `null`, or
|
|
203
|
+
* `<= 0` to disable (default), which preserves the prior unbounded behavior.
|
|
195
204
|
* @property {BackgroundJobsRetentionConfiguration} [retention] - Retention/pruning
|
|
196
205
|
* of terminal job rows. Without pruning the jobs table grows unbounded
|
|
197
206
|
* (completed rows accumulate forever), which bloats storage and indexes and
|
package/build/configuration.js
CHANGED
|
@@ -1270,10 +1270,12 @@ export default class VelociousConfiguration {
|
|
|
1270
1270
|
const envMaxConcurrentRaw = process.env.VELOCIOUS_BACKGROUND_JOBS_MAX_CONCURRENT_INLINE_JOBS
|
|
1271
1271
|
const envDispatchStrategy = process.env.VELOCIOUS_BACKGROUND_JOBS_DISPATCH_STRATEGY
|
|
1272
1272
|
const envPollIntervalRaw = process.env.VELOCIOUS_BACKGROUND_JOBS_POLL_INTERVAL_MS
|
|
1273
|
+
const envJobTimeoutRaw = process.env.VELOCIOUS_BACKGROUND_JOBS_JOB_TIMEOUT_MS
|
|
1273
1274
|
const envPort = envPortRaw ? Number(envPortRaw) : undefined
|
|
1274
1275
|
const envMaxConcurrentForked = envMaxConcurrentForkedRaw ? Number(envMaxConcurrentForkedRaw) : undefined
|
|
1275
1276
|
const envMaxConcurrent = envMaxConcurrentRaw ? Number(envMaxConcurrentRaw) : undefined
|
|
1276
1277
|
const envPollInterval = envPollIntervalRaw ? Number(envPollIntervalRaw) : undefined
|
|
1278
|
+
const envJobTimeout = envJobTimeoutRaw ? Number(envJobTimeoutRaw) : undefined
|
|
1277
1279
|
const configured = this._backgroundJobs || {}
|
|
1278
1280
|
const host = configured.host || envHost || "127.0.0.1"
|
|
1279
1281
|
const port = typeof configured.port === "number"
|
|
@@ -1292,6 +1294,12 @@ export default class VelociousConfiguration {
|
|
|
1292
1294
|
? configured.pollIntervalMs
|
|
1293
1295
|
: (typeof envPollInterval === "number" && Number.isFinite(envPollInterval) && envPollInterval >= 1 ? envPollInterval : 1000)
|
|
1294
1296
|
const queues = configured.queues && typeof configured.queues === "object" ? configured.queues : {}
|
|
1297
|
+
// An explicit config value wins over the env var — including `null`/`0`,
|
|
1298
|
+
// which disable the backstop even when the environment sets a default.
|
|
1299
|
+
// Only fall through to the env var when config omits `jobTimeoutMs` entirely.
|
|
1300
|
+
const jobTimeoutMs = "jobTimeoutMs" in configured
|
|
1301
|
+
? (typeof configured.jobTimeoutMs === "number" && configured.jobTimeoutMs > 0 ? configured.jobTimeoutMs : null)
|
|
1302
|
+
: (typeof envJobTimeout === "number" && Number.isFinite(envJobTimeout) && envJobTimeout > 0 ? envJobTimeout : null)
|
|
1295
1303
|
const configuredRetention = configured.retention && typeof configured.retention === "object" ? configured.retention : {}
|
|
1296
1304
|
const retention = {
|
|
1297
1305
|
completedTtlMs: typeof configuredRetention.completedTtlMs === "number" || configuredRetention.completedTtlMs === null
|
|
@@ -1308,7 +1316,7 @@ export default class VelociousConfiguration {
|
|
|
1308
1316
|
: 60 * 60 * 1000
|
|
1309
1317
|
}
|
|
1310
1318
|
|
|
1311
|
-
return {host, port, databaseIdentifier, maxConcurrentForkedJobs, maxConcurrentInlineJobs, dispatchStrategy, pollIntervalMs, queues, retention}
|
|
1319
|
+
return {host, port, databaseIdentifier, maxConcurrentForkedJobs, maxConcurrentInlineJobs, dispatchStrategy, pollIntervalMs, queues, jobTimeoutMs, retention}
|
|
1312
1320
|
}
|
|
1313
1321
|
|
|
1314
1322
|
/**
|
|
@@ -3,6 +3,9 @@ import BackgroundJobsMain from "../../../../background-jobs/main.js"
|
|
|
3
3
|
|
|
4
4
|
export default class BackgroundJobsMainCommand extends BaseCommand {
|
|
5
5
|
async execute() {
|
|
6
|
+
// Identify this process in `ps`/`top` instead of a generic "node" entry.
|
|
7
|
+
process.title = "velocious background-jobs-main"
|
|
8
|
+
|
|
6
9
|
const main = new BackgroundJobsMain({configuration: this.getConfiguration()})
|
|
7
10
|
await main.start()
|
|
8
11
|
|
|
@@ -16,6 +16,10 @@ export default class BackgroundJobsRunnerCommand extends BaseCommand {
|
|
|
16
16
|
process.once(signal, () => process.exit(0))
|
|
17
17
|
}
|
|
18
18
|
|
|
19
|
+
// Base title; runJobPayload sets the per-job title (from the job class's
|
|
20
|
+
// `static processTitle`) for the duration of the job and restores this after.
|
|
21
|
+
process.title = "velocious background-jobs-runner"
|
|
22
|
+
|
|
19
23
|
const decoded = Buffer.from(payload, "base64").toString("utf8")
|
|
20
24
|
const jobPayload = JSON.parse(decoded)
|
|
21
25
|
|
|
@@ -27,6 +27,9 @@ function resolveShutdownTimeoutMs() {
|
|
|
27
27
|
|
|
28
28
|
export default class BackgroundJobsWorkerCommand extends BaseCommand {
|
|
29
29
|
async execute() {
|
|
30
|
+
// Identify this process in `ps`/`top` instead of a generic "node" entry.
|
|
31
|
+
process.title = "velocious background-jobs-worker"
|
|
32
|
+
|
|
30
33
|
const worker = new BackgroundJobsWorker({configuration: this.getConfiguration()})
|
|
31
34
|
await worker.start()
|
|
32
35
|
|
|
@@ -3,6 +3,9 @@ import BeaconServer from "../../../../beacon/server.js"
|
|
|
3
3
|
|
|
4
4
|
export default class BeaconCommand extends BaseCommand {
|
|
5
5
|
async execute() {
|
|
6
|
+
// Identify this process in `ps`/`top` instead of a generic "node" entry.
|
|
7
|
+
process.title = "velocious beacon"
|
|
8
|
+
|
|
6
9
|
const beacon = new BeaconServer({configuration: this.getConfiguration()})
|
|
7
10
|
await beacon.start()
|
|
8
11
|
|
|
@@ -133,6 +133,9 @@ export default class VelociousCliCommandsServer extends BaseCommand{
|
|
|
133
133
|
* @returns {Promise<void>} - Starts the HTTP server and waits until it stops.
|
|
134
134
|
*/
|
|
135
135
|
async execute() {
|
|
136
|
+
// Identify this process in `ps`/`top` instead of a generic "node" entry.
|
|
137
|
+
process.title = "velocious server"
|
|
138
|
+
|
|
136
139
|
const parsedProcessArgs = this.args?.parsedProcessArgs || {}
|
|
137
140
|
const configuration = this.getConfiguration()
|
|
138
141
|
const httpServer = httpServerConfigFromParsedArgs(parsedProcessArgs, configuration.httpServer)
|
|
@@ -1,5 +1,10 @@
|
|
|
1
1
|
// @ts-check
|
|
2
2
|
import runJobPayload from "./job-runner.js";
|
|
3
|
+
// Name the process so `ps`/`top`/`htop` can identify forked job runners at a
|
|
4
|
+
// glance instead of a wall of generic "node" entries. Updated to the specific
|
|
5
|
+
// job name once one arrives (see runJobMessage), so operators can see exactly
|
|
6
|
+
// which jobs are running, how many of each, and which are eating resources.
|
|
7
|
+
process.title = "velocious background-jobs-runner";
|
|
3
8
|
let finishing = false;
|
|
4
9
|
/**
|
|
5
10
|
* Runs is job message.
|
|
@@ -42,6 +47,9 @@ async function runJobMessage(message) {
|
|
|
42
47
|
if (!isJobMessage(message)) {
|
|
43
48
|
throw new Error("Forked background job runner received invalid payload");
|
|
44
49
|
}
|
|
50
|
+
// The per-job process title (and its restore) is set inside runJobPayload,
|
|
51
|
+
// which reads the job class's `static processTitle`. This process boots with
|
|
52
|
+
// the base "velocious background-jobs-runner" title set at module load above.
|
|
45
53
|
await runJobPayload(message.payload, { closeConnections: false });
|
|
46
54
|
}
|
|
47
55
|
/**
|
|
@@ -71,4 +79,4 @@ process.once("disconnect", () => {
|
|
|
71
79
|
process.once("message", (message) => {
|
|
72
80
|
void handleJobMessage(message);
|
|
73
81
|
});
|
|
74
|
-
//# sourceMappingURL=data:application/json;base64,
|
|
82
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiZm9ya2VkLXJ1bm5lci1jaGlsZC5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uLy4uL3NyYy9iYWNrZ3JvdW5kLWpvYnMvZm9ya2VkLXJ1bm5lci1jaGlsZC5qcyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSxZQUFZO0FBRVosT0FBTyxhQUFhLE1BQU0saUJBQWlCLENBQUE7QUFFM0MsNkVBQTZFO0FBQzdFLDhFQUE4RTtBQUM5RSw4RUFBOEU7QUFDOUUsNEVBQTRFO0FBQzVFLE9BQU8sQ0FBQyxLQUFLLEdBQUcsa0NBQWtDLENBQUE7QUFFbEQsSUFBSSxTQUFTLEdBQUcsS0FBSyxDQUFBO0FBRXJCOzs7O0dBSUc7QUFDSCxTQUFTLFlBQVksQ0FBQyxPQUFPO0lBQzNCLElBQUksQ0FBQyxPQUFPLElBQUksT0FBTyxPQUFPLEtBQUssUUFBUTtRQUFFLE9BQU8sS0FBSyxDQUFBO0lBRXpELE1BQU0sYUFBYSxHQUFHLHNDQUFzQyxDQUFDLENBQUMsT0FBTyxDQUFDLENBQUE7SUFFdEUsT0FBTyxhQUFhLENBQUMsSUFBSSxLQUFLLEtBQUssSUFBSSxNQUFNLENBQUMsTUFBTSxDQUFDLGFBQWEsRUFBRSxTQUFTLENBQUMsQ0FBQTtBQUNoRixDQUFDO0FBRUQ7Ozs7R0FJRztBQUNILFNBQVMsTUFBTSxDQUFDLFFBQVE7SUFDdEIsU0FBUyxHQUFHLElBQUksQ0FBQTtJQUNoQixPQUFPLENBQUMsUUFBUSxHQUFHLFFBQVEsQ0FBQTtJQUUzQixJQUFJLE9BQU8sQ0FBQyxTQUFTLElBQUksT0FBTyxDQUFDLFVBQVUsRUFBRSxDQUFDO1FBQzVDLE9BQU8sQ0FBQyxVQUFVLEVBQUUsQ0FBQTtJQUN0QixDQUFDO0lBRUQsWUFBWSxDQUFDLEdBQUcsRUFBRSxDQUFDLE9BQU8sQ0FBQyxJQUFJLENBQUMsUUFBUSxDQUFDLENBQUMsQ0FBQTtBQUM1QyxDQUFDO0FBRUQ7OztHQUdHO0FBQ0gsU0FBUyxpQkFBaUI7SUFDeEIsSUFBSSxPQUFPLENBQUMsSUFBSTtRQUFFLE9BQU8sQ0FBQyxJQUFJLENBQUMsRUFBQyxJQUFJLEVBQUUsY0FBYyxFQUFDLENBQUMsQ0FBQTtBQUN4RCxDQUFDO0FBRUQ7Ozs7R0FJRztBQUNILEtBQUssVUFBVSxhQUFhLENBQUMsT0FBTztJQUNsQyxJQUFJLENBQUMsWUFBWSxDQUFDLE9BQU8sQ0FBQyxFQUFFLENBQUM7UUFDM0IsTUFBTSxJQUFJLEtBQUssQ0FBQyx1REFBdUQsQ0FBQyxDQUFBO0lBQzFFLENBQUM7SUFFRCwyRUFBMkU7SUFDM0UsNkVBQTZFO0lBQzdFLDhFQUE4RTtJQUM5RSxNQUFNLGFBQWEsQ0FBQyxPQUFPLENBQUMsT0FBTyxFQUFFLEVBQUMsZ0JBQWdCLEVBQUUsS0FBSyxFQUFDLENBQUMsQ0FBQTtBQUNqRSxDQUFDO0FBRUQ7Ozs7R0FJRztBQUNILEtBQUssVUFBVSxnQkFBZ0IsQ0FBQyxPQUFPO0lBQ3JDLElBQUksQ0FBQztRQUNILE1BQU0sYUFBYSxDQUFDLE9BQU8sQ0FBQyxDQUFBO1FBQzVCLGlCQUFpQixFQUFFLENBQUE7UUFDbkIsTUFBTSxDQUFDLENBQUMsQ0FBQyxDQUFBO0lBQ1gsQ0FBQztJQUFDLE9BQU8sS0FBSyxFQUFFLENBQUM7UUFDZixpQkFBaUIsRUFBRSxDQUFBO1FBQ25CLE9BQU8sQ0FBQyxLQUFLLENBQUMsc0NBQXNDLEVBQUUsS0FBSyxDQUFDLENBQUE7UUFDNUQsTUFBTSxDQUFDLENBQUMsQ0FBQyxDQUFBO0lBQ1gsQ0FBQztBQUNILENBQUM7QUFFRCxLQUFLLE1BQU0sTUFBTSxJQUFJLENBQUMsU0FBUyxFQUFFLFFBQVEsQ0FBQyxFQUFFLENBQUM7SUFDM0MsT0FBTyxDQUFDLElBQUksQ0FBQyxNQUFNLEVBQUUsR0FBRyxFQUFFLENBQUMsT0FBTyxDQUFDLElBQUksQ0FBQyxDQUFDLENBQUMsQ0FBQyxDQUFBO0FBQzdDLENBQUM7QUFFRCxPQUFPLENBQUMsSUFBSSxDQUFDLFlBQVksRUFBRSxHQUFHLEVBQUU7SUFDOUIsSUFBSSxDQUFDLFNBQVM7UUFBRSxPQUFPLENBQUMsSUFBSSxDQUFDLENBQUMsQ0FBQyxDQUFBO0FBQ2pDLENBQUMsQ0FBQyxDQUFBO0FBRUYsT0FBTyxDQUFDLElBQUksQ0FBQyxTQUFTLEVBQUUsQ0FBQyxPQUFPLEVBQUUsRUFBRTtJQUNsQyxLQUFLLGdCQUFnQixDQUFDLE9BQU8sQ0FBQyxDQUFBO0FBQ2hDLENBQUMsQ0FBQyxDQUFBIiwic291cmNlc0NvbnRlbnQiOlsiLy8gQHRzLWNoZWNrXG5cbmltcG9ydCBydW5Kb2JQYXlsb2FkIGZyb20gXCIuL2pvYi1ydW5uZXIuanNcIlxuXG4vLyBOYW1lIHRoZSBwcm9jZXNzIHNvIGBwc2AvYHRvcGAvYGh0b3BgIGNhbiBpZGVudGlmeSBmb3JrZWQgam9iIHJ1bm5lcnMgYXQgYVxuLy8gZ2xhbmNlIGluc3RlYWQgb2YgYSB3YWxsIG9mIGdlbmVyaWMgXCJub2RlXCIgZW50cmllcy4gVXBkYXRlZCB0byB0aGUgc3BlY2lmaWNcbi8vIGpvYiBuYW1lIG9uY2Ugb25lIGFycml2ZXMgKHNlZSBydW5Kb2JNZXNzYWdlKSwgc28gb3BlcmF0b3JzIGNhbiBzZWUgZXhhY3RseVxuLy8gd2hpY2ggam9icyBhcmUgcnVubmluZywgaG93IG1hbnkgb2YgZWFjaCwgYW5kIHdoaWNoIGFyZSBlYXRpbmcgcmVzb3VyY2VzLlxucHJvY2Vzcy50aXRsZSA9IFwidmVsb2Npb3VzIGJhY2tncm91bmQtam9icy1ydW5uZXJcIlxuXG5sZXQgZmluaXNoaW5nID0gZmFsc2VcblxuLyoqXG4gKiBSdW5zIGlzIGpvYiBtZXNzYWdlLlxuICogQHBhcmFtIHs/fSBtZXNzYWdlIC0gSVBDIG1lc3NhZ2UuXG4gKiBAcmV0dXJucyB7bWVzc2FnZSBpcyB7dHlwZTogXCJqb2JcIiwgcGF5bG9hZDogaW1wb3J0KFwiLi90eXBlcy5qc1wiKS5CYWNrZ3JvdW5kSm9iUGF5bG9hZH19IC0gV2hldGhlciB0aGlzIGlzIGEgam9iIG1lc3NhZ2UuXG4gKi9cbmZ1bmN0aW9uIGlzSm9iTWVzc2FnZShtZXNzYWdlKSB7XG4gIGlmICghbWVzc2FnZSB8fCB0eXBlb2YgbWVzc2FnZSAhPT0gXCJvYmplY3RcIikgcmV0dXJuIGZhbHNlXG5cbiAgY29uc3QgbWVzc2FnZVJlY29yZCA9IC8qKiBAdHlwZSB7e3R5cGU/OiA/LCBwYXlsb2FkPzogP319ICovIChtZXNzYWdlKVxuXG4gIHJldHVybiBtZXNzYWdlUmVjb3JkLnR5cGUgPT09IFwiam9iXCIgJiYgT2JqZWN0Lmhhc093bihtZXNzYWdlUmVjb3JkLCBcInBheWxvYWRcIilcbn1cblxuLyoqXG4gKiBSdW5zIGZpbmlzaC5cbiAqIEBwYXJhbSB7bnVtYmVyfSBleGl0Q29kZSAtIFByb2Nlc3MgZXhpdCBjb2RlLlxuICogQHJldHVybnMge3ZvaWR9XG4gKi9cbmZ1bmN0aW9uIGZpbmlzaChleGl0Q29kZSkge1xuICBmaW5pc2hpbmcgPSB0cnVlXG4gIHByb2Nlc3MuZXhpdENvZGUgPSBleGl0Q29kZVxuXG4gIGlmIChwcm9jZXNzLmNvbm5lY3RlZCAmJiBwcm9jZXNzLmRpc2Nvbm5lY3QpIHtcbiAgICBwcm9jZXNzLmRpc2Nvbm5lY3QoKVxuICB9XG5cbiAgc2V0SW1tZWRpYXRlKCgpID0+IHByb2Nlc3MuZXhpdChleGl0Q29kZSkpXG59XG5cbi8qKlxuICogUnVucyByZXBvcnQgam9iIGZpbmlzaGVkLlxuICogQHJldHVybnMge3ZvaWR9XG4gKi9cbmZ1bmN0aW9uIHJlcG9ydEpvYkZpbmlzaGVkKCkge1xuICBpZiAocHJvY2Vzcy5zZW5kKSBwcm9jZXNzLnNlbmQoe3R5cGU6IFwiam9iLXJlcG9ydGVkXCJ9KVxufVxuXG4vKipcbiAqIFJ1bnMgcnVuIGpvYiBtZXNzYWdlLlxuICogQHBhcmFtIHs/fSBtZXNzYWdlIC0gSVBDIG1lc3NhZ2UuXG4gKiBAcmV0dXJucyB7UHJvbWlzZTx2b2lkPn0gLSBSZXNvbHZlcyBhZnRlciB0aGUgcGF5bG9hZCBoYXMgcnVuLlxuICovXG5hc3luYyBmdW5jdGlvbiBydW5Kb2JNZXNzYWdlKG1lc3NhZ2UpIHtcbiAgaWYgKCFpc0pvYk1lc3NhZ2UobWVzc2FnZSkpIHtcbiAgICB0aHJvdyBuZXcgRXJyb3IoXCJGb3JrZWQgYmFja2dyb3VuZCBqb2IgcnVubmVyIHJlY2VpdmVkIGludmFsaWQgcGF5bG9hZFwiKVxuICB9XG5cbiAgLy8gVGhlIHBlci1qb2IgcHJvY2VzcyB0aXRsZSAoYW5kIGl0cyByZXN0b3JlKSBpcyBzZXQgaW5zaWRlIHJ1bkpvYlBheWxvYWQsXG4gIC8vIHdoaWNoIHJlYWRzIHRoZSBqb2IgY2xhc3MncyBgc3RhdGljIHByb2Nlc3NUaXRsZWAuIFRoaXMgcHJvY2VzcyBib290cyB3aXRoXG4gIC8vIHRoZSBiYXNlIFwidmVsb2Npb3VzIGJhY2tncm91bmQtam9icy1ydW5uZXJcIiB0aXRsZSBzZXQgYXQgbW9kdWxlIGxvYWQgYWJvdmUuXG4gIGF3YWl0IHJ1bkpvYlBheWxvYWQobWVzc2FnZS5wYXlsb2FkLCB7Y2xvc2VDb25uZWN0aW9uczogZmFsc2V9KVxufVxuXG4vKipcbiAqIFJ1bnMgaGFuZGxlIGpvYiBtZXNzYWdlLlxuICogQHBhcmFtIHs/fSBtZXNzYWdlIC0gSVBDIG1lc3NhZ2UuXG4gKiBAcmV0dXJucyB7UHJvbWlzZTx2b2lkPn0gLSBSZXNvbHZlcyBhZnRlciBjb21wbGV0aW9uIGlzIHJlcG9ydGVkLlxuICovXG5hc3luYyBmdW5jdGlvbiBoYW5kbGVKb2JNZXNzYWdlKG1lc3NhZ2UpIHtcbiAgdHJ5IHtcbiAgICBhd2FpdCBydW5Kb2JNZXNzYWdlKG1lc3NhZ2UpXG4gICAgcmVwb3J0Sm9iRmluaXNoZWQoKVxuICAgIGZpbmlzaCgwKVxuICB9IGNhdGNoIChlcnJvcikge1xuICAgIHJlcG9ydEpvYkZpbmlzaGVkKClcbiAgICBjb25zb2xlLmVycm9yKFwiRm9ya2VkIGJhY2tncm91bmQgam9iIHJ1bm5lciBmYWlsZWQ6XCIsIGVycm9yKVxuICAgIGZpbmlzaCgxKVxuICB9XG59XG5cbmZvciAoY29uc3Qgc2lnbmFsIG9mIFtcIlNJR1RFUk1cIiwgXCJTSUdJTlRcIl0pIHtcbiAgcHJvY2Vzcy5vbmNlKHNpZ25hbCwgKCkgPT4gcHJvY2Vzcy5leGl0KDEpKVxufVxuXG5wcm9jZXNzLm9uY2UoXCJkaXNjb25uZWN0XCIsICgpID0+IHtcbiAgaWYgKCFmaW5pc2hpbmcpIHByb2Nlc3MuZXhpdCgwKVxufSlcblxucHJvY2Vzcy5vbmNlKFwibWVzc2FnZVwiLCAobWVzc2FnZSkgPT4ge1xuICB2b2lkIGhhbmRsZUpvYk1lc3NhZ2UobWVzc2FnZSlcbn0pXG4iXX0=
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"job-runner.d.ts","sourceRoot":"","sources":["../../../src/background-jobs/job-runner.js"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"job-runner.d.ts","sourceRoot":"","sources":["../../../src/background-jobs/job-runner.js"],"names":[],"mappings":"AAgEA;;;;;;GAMG;AACH,+CALW,OAAO,YAAY,EAAE,oBAAoB,yBAEjD;IAA0B,gBAAgB;CAC1C,GAAU,OAAO,CAAC,IAAI,CAAC,CAkEzB"}
|