velocious 1.0.747 → 1.0.749
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/build/background-jobs/adapter.js +7 -0
- package/build/background-jobs/pooled-runner-child.js +41 -4
- package/build/background-jobs/pooled-runner-shutdown.js +92 -0
- package/build/background-jobs/queue-priority-order.js +32 -23
- package/build/background-jobs/store.js +443 -15
- package/build/background-jobs/types.js +31 -0
- package/build/background-jobs/worker.js +56 -3
- package/build/configuration-types.js +16 -0
- package/build/configuration.js +14 -1
- package/build/database/table-data/table-column.js +5 -1
- package/build/frontend-model-controller.js +41 -0
- package/build/frontend-models/query.js +42 -0
- package/build/jobs/prune-terminal-background-jobs.js +15 -1
- package/build/src/background-jobs/adapter.d.ts +11 -0
- package/build/src/background-jobs/adapter.d.ts.map +1 -1
- package/build/src/background-jobs/adapter.js +7 -1
- package/build/src/background-jobs/pooled-runner-child.js +39 -5
- package/build/src/background-jobs/pooled-runner-shutdown.d.ts +22 -0
- package/build/src/background-jobs/pooled-runner-shutdown.d.ts.map +1 -1
- package/build/src/background-jobs/pooled-runner-shutdown.js +87 -1
- package/build/src/background-jobs/queue-priority-order.d.ts +28 -14
- package/build/src/background-jobs/queue-priority-order.d.ts.map +1 -1
- package/build/src/background-jobs/queue-priority-order.js +33 -23
- package/build/src/background-jobs/store.d.ts +155 -3
- package/build/src/background-jobs/store.d.ts.map +1 -1
- package/build/src/background-jobs/store.js +386 -14
- package/build/src/background-jobs/types.d.ts +102 -0
- package/build/src/background-jobs/types.d.ts.map +1 -1
- package/build/src/background-jobs/types.js +32 -1
- package/build/src/background-jobs/worker.d.ts +16 -0
- package/build/src/background-jobs/worker.d.ts.map +1 -1
- package/build/src/background-jobs/worker.js +56 -4
- package/build/src/configuration-types.d.ts +48 -0
- package/build/src/configuration-types.d.ts.map +1 -1
- package/build/src/configuration-types.js +16 -1
- package/build/src/configuration.d.ts.map +1 -1
- package/build/src/configuration.js +14 -2
- package/build/src/database/table-data/table-column.d.ts.map +1 -1
- package/build/src/database/table-data/table-column.js +6 -2
- package/build/src/frontend-model-controller.d.ts +4 -0
- package/build/src/frontend-model-controller.d.ts.map +1 -1
- package/build/src/frontend-model-controller.js +34 -1
- package/build/src/frontend-models/query.d.ts +4 -0
- package/build/src/frontend-models/query.d.ts.map +1 -1
- package/build/src/frontend-models/query.js +38 -1
- package/build/src/jobs/prune-terminal-background-jobs.d.ts +4 -1
- package/build/src/jobs/prune-terminal-background-jobs.d.ts.map +1 -1
- package/build/src/jobs/prune-terminal-background-jobs.js +14 -2
- package/build/src/sync/sync-client.d.ts.map +1 -1
- package/build/src/sync/sync-client.js +2 -1
- package/build/src/sync/sync-offline-session.d.ts.map +1 -1
- package/build/src/sync/sync-offline-session.js +9 -3
- package/build/sync/sync-client.js +1 -0
- package/build/sync/sync-offline-session.js +9 -1
- package/package.json +1 -1
- package/src/background-jobs/adapter.js +7 -0
- package/src/background-jobs/pooled-runner-child.js +41 -4
- package/src/background-jobs/pooled-runner-shutdown.js +92 -0
- package/src/background-jobs/queue-priority-order.js +32 -23
- package/src/background-jobs/store.js +443 -15
- package/src/background-jobs/types.js +31 -0
- package/src/background-jobs/worker.js +56 -3
- package/src/configuration-types.js +16 -0
- package/src/configuration.js +14 -1
- package/src/database/table-data/table-column.js +5 -1
- package/src/frontend-model-controller.js +41 -0
- package/src/frontend-models/query.js +42 -0
- package/src/jobs/prune-terminal-background-jobs.js +15 -1
- package/src/sync/sync-client.js +1 -0
- package/src/sync/sync-offline-session.js +9 -1
package/README.md
CHANGED
|
@@ -2512,7 +2512,7 @@ creation across generation processes.
|
|
|
2512
2512
|
|
|
2513
2513
|
Jobs can opt into cross-worker durable concurrency limits by pairing a non-empty `concurrencyKey` with a positive-integer `maxConcurrency` in their background-job options, or by deriving the key in a hydrated job instance's non-static `concurrencyKey()` method. Explicit enqueue options win. The first cap registered for a key is stable; conflicting caps are rejected. A key's counter row is removed when its last live job settles, so keys that name a one-off unit of work do not accumulate rows forever; queue-derived lane counters are kept. See [durable concurrency limits](docs/background-jobs.md#durable-concurrency-limits).
|
|
2514
2514
|
|
|
2515
|
-
Production apps can listen for `background-job-failed` (or its `all-error` mirror) to report accepted failed attempts, including retry and terminal-state metadata. Process-level pooled-runner failures also carry one shared `context.runnerFailure` snapshot for every affected job, with active handoff identities, child instance/PID, runner/worker lifecycle, exit code/signal, an exact typed shutdown reason and request/observation timestamps, bounded in-flight job IDs,
|
|
2515
|
+
Production apps can listen for `background-job-failed` (or its `all-error` mirror) to report accepted failed attempts, including retry and terminal-state metadata. Process-level pooled-runner failures also carry one shared `context.runnerFailure` snapshot for every affected job, with active handoff identities, child instance/PID, runner/worker lifecycle, exit code/signal, an exact typed shutdown reason and request/observation timestamps, bounded in-flight job IDs, an explicit nullable OOM verdict, and the bounded `processFault` the child reported over IPC before it exited (`type` `uncaughtException`/`unhandledRejection`, `message`, `name`, `stack`, and a JSON `valuePreview` for a rejected non-`Error` value), so an unhandled rejection inside a runner names itself instead of surfacing as an unexplained runner exit. Every handed-off job also records the accepting runner's `child_instance_id`, `child_pid`, `runner_hostname` and start time; `BackgroundJobRecord#ownerIsAlive()` reports whether that recorded local process still exists (`alive`, `dead`, or `unknown` for remote/unrecorded owners), so claim ownership can be checked without polling. The deprecated `terminationReason` compatibility category remains present, while new consumers should use authoritative `shutdownReason`. Parent timeout/stop/retirement requests are recorded before signalling; normal drained retirement is permitted only with zero tracked jobs. Listen for `background-job-orphaned` to react to a specific job the main process reclaimed after its worker died mid-run — e.g. enqueue a targeted recovery for the work it left behind, instead of only polling for the aftermath. The orphan payload carries the dead owner's `childInstanceId`, `childPid`, `runnerHostname` and `workerId`, so a handler can release exactly that owner's claims on the event it already receives. Orphan handlers run before the sweep waits for reclaimed jobs to be dispatched, so a stalled dispatcher does not delay application recovery. An orphan without a delivered runner-failure report has unknown shutdown cause; do not infer OOM, deploy, timeout, signal, or normal retirement from acceptance evidence alone. See [docs/background-jobs.md](docs/background-jobs.md#failure-events) and [runner identity and owner liveness](docs/background-jobs.md#runner-identity-and-owner-liveness).
|
|
2516
2516
|
|
|
2517
2517
|
## Setup
|
|
2518
2518
|
|
|
@@ -238,4 +238,11 @@ export default class BackgroundJobsAdapter {
|
|
|
238
238
|
* @returns {Promise<number>} - Deleted rows.
|
|
239
239
|
*/
|
|
240
240
|
async pruneTerminalJobs(_args = {}) { throw new Error("BackgroundJobsAdapter#pruneTerminalJobs is not implemented") }
|
|
241
|
+
|
|
242
|
+
/**
|
|
243
|
+
* Moves terminal jobs past their retention windows into monthly archive tables.
|
|
244
|
+
* @param {{completedTtlMs?: number | null, failedTtlMs?: number | null, batchSize?: number, timeBudgetMs?: number}} [_args] - Retention options.
|
|
245
|
+
* @returns {Promise<import("./types.js").BackgroundJobsArchiveResult>} - Archived rows and per-month totals.
|
|
246
|
+
*/
|
|
247
|
+
async archiveTerminalJobs(_args = {}) { throw new Error("BackgroundJobsAdapter#archiveTerminalJobs is not implemented") }
|
|
241
248
|
}
|
|
@@ -6,7 +6,7 @@ import os from "node:os"
|
|
|
6
6
|
import v8 from "node:v8"
|
|
7
7
|
import timeout from "awaitery/build/timeout.js"
|
|
8
8
|
import runJobPayload, { BackgroundJobPerformedFailure } from "./job-runner.js"
|
|
9
|
-
import { boundedPooledRunnerInflightJobIds, isPooledChildShutdownReason, isPooledChildShutdownSignal } from "./pooled-runner-shutdown.js"
|
|
9
|
+
import { boundedPooledRunnerInflightJobIds, isPooledChildShutdownReason, isPooledChildShutdownSignal, pooledChildProcessFault } from "./pooled-runner-shutdown.js"
|
|
10
10
|
import { closeRunnerConnections, closeRunnerFrameworkConnections, currentConfigurationOrNull } from "./runner-graceful-shutdown.js"
|
|
11
11
|
import setRunnerProcessTitle from "./runner-process-title.js"
|
|
12
12
|
import PooledRunnerBrokerIdentity from "./pooled-runner-broker-identity.js"
|
|
@@ -51,6 +51,13 @@ function readPeakRssBytes() {
|
|
|
51
51
|
|
|
52
52
|
setRunnerProcessTitle()
|
|
53
53
|
|
|
54
|
+
/**
|
|
55
|
+
* Process-level fault observed by this child, carried on the shutdown observation so
|
|
56
|
+
* the worker reports the fault itself instead of only the exit it caused.
|
|
57
|
+
* @type {import("./types.js").PooledChildProcessFault | undefined}
|
|
58
|
+
*/
|
|
59
|
+
let processFault
|
|
60
|
+
|
|
54
61
|
/** @type {Promise<void> | undefined} */
|
|
55
62
|
let shutdownPromise
|
|
56
63
|
|
|
@@ -73,7 +80,16 @@ function shutdownRunner({exitCode, reason, shutdownRequestedAtMs = null, signal
|
|
|
73
80
|
|
|
74
81
|
shutdownPromise = (async () => {
|
|
75
82
|
await sendShutdownObservation({reason, shutdownRequestedAtMs, signal})
|
|
76
|
-
|
|
83
|
+
|
|
84
|
+
try {
|
|
85
|
+
await closeRunnerConnections(currentConfigurationOrNull())
|
|
86
|
+
} catch (error) {
|
|
87
|
+
// A teardown failure must not escape into the process-level handler: it would
|
|
88
|
+
// report a fault that only happened while closing as the cause of a shutdown
|
|
89
|
+
// whose reason this child already sent.
|
|
90
|
+
console.error("Pooled background job runner could not close its connections:", error)
|
|
91
|
+
}
|
|
92
|
+
|
|
77
93
|
process.exit(exitCode)
|
|
78
94
|
})()
|
|
79
95
|
|
|
@@ -147,6 +163,7 @@ async function sendShutdownObservation({reason, shutdownRequestedAtMs, signal})
|
|
|
147
163
|
/** @type {import("./types.js").PooledChildShutdownObservation & {type: "shutdown-observation"}} */
|
|
148
164
|
const message = {
|
|
149
165
|
childInstanceId,
|
|
166
|
+
fault: processFault ?? null,
|
|
150
167
|
...boundedInflight,
|
|
151
168
|
reason,
|
|
152
169
|
shutdownObservedAtMs: Date.now(),
|
|
@@ -176,6 +193,23 @@ async function sendShutdownObservation({reason, shutdownRequestedAtMs, signal})
|
|
|
176
193
|
}
|
|
177
194
|
}
|
|
178
195
|
|
|
196
|
+
/**
|
|
197
|
+
* Records one process-level fault and takes the child down through the same bounded
|
|
198
|
+
* shutdown path as a signalled or parent-requested exit, so the worker receives the
|
|
199
|
+
* fault over IPC before the process is gone. Node terminates the process on an
|
|
200
|
+
* unhandled rejection regardless, and this child's stdio is ignored by the worker
|
|
201
|
+
* fork, so the only open question is whether the worker learns why the child died:
|
|
202
|
+
* without this handler the exit is recorded as runner loss and the whole shard is
|
|
203
|
+
* retried with zero reported failures.
|
|
204
|
+
* @param {import("./types.js").PooledChildProcessFaultType} type - Process event that observed the fault.
|
|
205
|
+
* @param {ReturnType<typeof JSON.parse>} value - Thrown error or rejected value.
|
|
206
|
+
* @returns {void}
|
|
207
|
+
*/
|
|
208
|
+
function reportProcessFault(type, value) {
|
|
209
|
+
processFault = pooledChildProcessFault(type, value)
|
|
210
|
+
void shutdownRunner({exitCode: 1, reason: "process_error"})
|
|
211
|
+
}
|
|
212
|
+
|
|
179
213
|
/**
|
|
180
214
|
* Reports one acceptance observation (job received / perform started) to the
|
|
181
215
|
* worker over IPC. The message carries the job's exact handoff lease so the
|
|
@@ -359,8 +393,9 @@ function sendOutcome({jobId, acknowledged, status, error}) {
|
|
|
359
393
|
* outcome. A single job's unexpected failure reports that job for reclamation
|
|
360
394
|
* (`acknowledged: false`) but does NOT take down the child — its concurrent
|
|
361
395
|
* siblings keep running. Only a process-level fault (which escapes every
|
|
362
|
-
* per-job try/catch) ends the child,
|
|
363
|
-
*
|
|
396
|
+
* per-job try/catch) ends the child, and `reportProcessFault` names that fault
|
|
397
|
+
* on the shutdown observation so the worker reclaims the whole in-flight set
|
|
398
|
+
* with the cause instead of an unexplained exit.
|
|
364
399
|
* @param {import("./types.js").BackgroundJobPayload & {id: string}} payload - Job payload.
|
|
365
400
|
* @param {import("../testing/shared-transaction-proxy-driver.js").SharedTransactionBrokerJobConfig} sharedTransactionBroker - Per-job broker configuration.
|
|
366
401
|
* @returns {Promise<void>} - Resolves after reporting.
|
|
@@ -435,4 +470,6 @@ process.on("message", (message) => handleMessage(message))
|
|
|
435
470
|
process.once("disconnect", () => void shutdownRunner({exitCode: 0, reason: "ipc_disconnect"}))
|
|
436
471
|
process.once("SIGTERM", () => void shutdownRunner({exitCode: 1, reason: "signal_sigterm", signal: "SIGTERM"}))
|
|
437
472
|
process.once("SIGINT", () => void shutdownRunner({exitCode: 1, reason: "signal_sigint", signal: "SIGINT"}))
|
|
473
|
+
process.once("uncaughtException", (error) => reportProcessFault("uncaughtException", error))
|
|
474
|
+
process.once("unhandledRejection", (reason) => reportProcessFault("unhandledRejection", reason))
|
|
438
475
|
if (process.send) process.send({childInstanceId, childPid: process.pid, runnerHostname, type: "ready"})
|
|
@@ -4,6 +4,12 @@ import { constants } from "node:os"
|
|
|
4
4
|
|
|
5
5
|
/** Maximum job ids copied into a pooled-child shutdown observation or failure snapshot. */
|
|
6
6
|
export const POOLED_RUNNER_INFLIGHT_JOB_ID_LIMIT = 100
|
|
7
|
+
/** Maximum characters of a pooled-child fault message or error name crossing IPC. */
|
|
8
|
+
export const POOLED_CHILD_PROCESS_FAULT_MESSAGE_LIMIT = 1000
|
|
9
|
+
/** Maximum characters of a pooled-child fault stack crossing IPC. */
|
|
10
|
+
export const POOLED_CHILD_PROCESS_FAULT_STACK_LIMIT = 4000
|
|
11
|
+
/** Maximum characters of the preview of a rejected value that is not an Error. */
|
|
12
|
+
export const POOLED_CHILD_PROCESS_FAULT_VALUE_LIMIT = 500
|
|
7
13
|
|
|
8
14
|
/** @type {readonly import("./types.js").PooledChildShutdownReason[]} */
|
|
9
15
|
export const POOLED_CHILD_SHUTDOWN_REASONS = Object.freeze([
|
|
@@ -20,6 +26,12 @@ export const POOLED_CHILD_SHUTDOWN_REASONS = Object.freeze([
|
|
|
20
26
|
"unexpected_exit"
|
|
21
27
|
])
|
|
22
28
|
|
|
29
|
+
/** @type {readonly import("./types.js").PooledChildProcessFaultType[]} */
|
|
30
|
+
export const POOLED_CHILD_PROCESS_FAULT_TYPES = Object.freeze([
|
|
31
|
+
"uncaughtException",
|
|
32
|
+
"unhandledRejection"
|
|
33
|
+
])
|
|
34
|
+
|
|
23
35
|
/**
|
|
24
36
|
* Checks one dynamic IPC reason against the complete shutdown contract.
|
|
25
37
|
* @param {ReturnType<typeof JSON.parse>} value - Candidate shutdown reason.
|
|
@@ -54,3 +66,83 @@ export function boundedPooledRunnerInflightJobIds(jobIds) {
|
|
|
54
66
|
inflightJobIdsTruncatedCount: sortedJobIds.length - inflightJobIds.length
|
|
55
67
|
}
|
|
56
68
|
}
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* Bounds one dynamic fault text for the IPC/report contract: a non-text value
|
|
72
|
+
* becomes null, an over-long value is cut with a visible marker so a reader can
|
|
73
|
+
* tell a truncated record from the whole fault.
|
|
74
|
+
* @param {ReturnType<typeof JSON.parse>} value - Candidate text.
|
|
75
|
+
* @param {number} limit - Maximum characters.
|
|
76
|
+
* @returns {string | null} - Bounded text, or null when the value is not text.
|
|
77
|
+
*/
|
|
78
|
+
function boundedPooledChildFaultText(value, limit) {
|
|
79
|
+
if (typeof value !== "string") return null
|
|
80
|
+
if (value.length <= limit) return value
|
|
81
|
+
|
|
82
|
+
return `${value.slice(0, limit - 1)}…`
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* Checks one fault field against its bound.
|
|
87
|
+
* @param {ReturnType<typeof JSON.parse>} value - Candidate field.
|
|
88
|
+
* @param {number} limit - Maximum characters.
|
|
89
|
+
* @returns {boolean} - Whether the field is null or bounded text.
|
|
90
|
+
*/
|
|
91
|
+
function isBoundedPooledChildFaultText(value, limit) {
|
|
92
|
+
return value === null || (typeof value === "string" && value.length <= limit)
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* Previews a rejected value that is not an Error. `Promise.reject({})` — the shape a
|
|
97
|
+
* real incident printed — stringifies to `[object Object]` and names nothing, so a
|
|
98
|
+
* structured value is previewed as JSON instead.
|
|
99
|
+
* @param {ReturnType<typeof JSON.parse>} value - Rejected value.
|
|
100
|
+
* @returns {string | null} - Bounded preview, or null when nothing renderable was given.
|
|
101
|
+
*/
|
|
102
|
+
function previewPooledChildFaultValue(value) {
|
|
103
|
+
if (typeof value === "string" || value === null) return boundedPooledChildFaultText(value, POOLED_CHILD_PROCESS_FAULT_VALUE_LIMIT)
|
|
104
|
+
|
|
105
|
+
try {
|
|
106
|
+
const preview = JSON.stringify(value)
|
|
107
|
+
|
|
108
|
+
return boundedPooledChildFaultText(preview === undefined ? String(value) : preview, POOLED_CHILD_PROCESS_FAULT_VALUE_LIMIT)
|
|
109
|
+
} catch {
|
|
110
|
+
return boundedPooledChildFaultText(String(value), POOLED_CHILD_PROCESS_FAULT_VALUE_LIMIT)
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/**
|
|
115
|
+
* Describes one pooled-child process-level fault as the bounded record that crosses
|
|
116
|
+
* IPC, so the worker reports the fault itself instead of only the exit it caused.
|
|
117
|
+
* @param {import("./types.js").PooledChildProcessFaultType} type - Process event that observed the fault.
|
|
118
|
+
* @param {ReturnType<typeof JSON.parse>} value - Thrown error or rejected value.
|
|
119
|
+
* @returns {import("./types.js").PooledChildProcessFault} - Bounded fault record.
|
|
120
|
+
*/
|
|
121
|
+
export function pooledChildProcessFault(type, value) {
|
|
122
|
+
const faultError = value instanceof Error ? value : null
|
|
123
|
+
const valuePreview = faultError ? null : previewPooledChildFaultValue(value)
|
|
124
|
+
|
|
125
|
+
return {
|
|
126
|
+
message: boundedPooledChildFaultText(faultError ? faultError.message : valuePreview, POOLED_CHILD_PROCESS_FAULT_MESSAGE_LIMIT),
|
|
127
|
+
name: faultError ? boundedPooledChildFaultText(faultError.name, POOLED_CHILD_PROCESS_FAULT_MESSAGE_LIMIT) : null,
|
|
128
|
+
stack: faultError ? boundedPooledChildFaultText(faultError.stack, POOLED_CHILD_PROCESS_FAULT_STACK_LIMIT) : null,
|
|
129
|
+
type,
|
|
130
|
+
valuePreview
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
/**
|
|
135
|
+
* Checks one dynamic IPC value against the process-fault contract.
|
|
136
|
+
* @param {ReturnType<typeof JSON.parse>} value - Candidate fault record.
|
|
137
|
+
* @returns {value is import("./types.js").PooledChildProcessFault} - Whether the value is a bounded fault record.
|
|
138
|
+
*/
|
|
139
|
+
export function isPooledChildProcessFault(value) {
|
|
140
|
+
if (!value || typeof value !== "object") return false
|
|
141
|
+
const record = /** @type {{message?: ReturnType<typeof JSON.parse>, name?: ReturnType<typeof JSON.parse>, stack?: ReturnType<typeof JSON.parse>, type?: ReturnType<typeof JSON.parse>, valuePreview?: ReturnType<typeof JSON.parse>}} */ (value)
|
|
142
|
+
|
|
143
|
+
return POOLED_CHILD_PROCESS_FAULT_TYPES.some((type) => type === record.type)
|
|
144
|
+
&& isBoundedPooledChildFaultText(record.message, POOLED_CHILD_PROCESS_FAULT_MESSAGE_LIMIT)
|
|
145
|
+
&& isBoundedPooledChildFaultText(record.name, POOLED_CHILD_PROCESS_FAULT_MESSAGE_LIMIT)
|
|
146
|
+
&& isBoundedPooledChildFaultText(record.stack, POOLED_CHILD_PROCESS_FAULT_STACK_LIMIT)
|
|
147
|
+
&& isBoundedPooledChildFaultText(record.valuePreview, POOLED_CHILD_PROCESS_FAULT_VALUE_LIMIT)
|
|
148
|
+
}
|
|
@@ -3,11 +3,14 @@
|
|
|
3
3
|
import {DEFAULT_BACKGROUND_JOB_QUEUE, QUEUE_CONCURRENCY_KEY_PREFIX} from "./job-semantics.js"
|
|
4
4
|
|
|
5
5
|
/**
|
|
6
|
-
* How
|
|
7
|
-
* `1` priority unit buys this much queue-jumping
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
6
|
+
* How long a lane may age before its configured priority stops buying queue-jumping.
|
|
7
|
+
* `1` priority unit still buys this much queue-jumping for fresh work — a priority-`10` lane is
|
|
8
|
+
* served before a priority-`0` lane while both are younger than one unit — but the aged row's
|
|
9
|
+
* wait is multiplied by the lane's gap to the highest priority, so every lane reaches the shared
|
|
10
|
+
* ceiling after exactly one unit of eligible waiting. With the default, no lane is starved for
|
|
11
|
+
* longer than a minute of contention regardless of how deep the priority ladder is. Time is the
|
|
12
|
+
* only condition — nothing here runs on a cadence, and a lane that is dispatched normally never
|
|
13
|
+
* ages far enough for the term to matter.
|
|
11
14
|
*/
|
|
12
15
|
export const DEFAULT_QUEUE_PRIORITY_AGING_UNIT_MS = 60000
|
|
13
16
|
|
|
@@ -44,11 +47,21 @@ export function queuePriorityAgingUnitMs(configuration) {
|
|
|
44
47
|
* never skipped, and every lane below it receives zero claims for as long as it stays
|
|
45
48
|
* non-empty.
|
|
46
49
|
*
|
|
47
|
-
* Aging adds the row's own eligible wait,
|
|
48
|
-
*
|
|
49
|
-
* service age is folded into this same primary rank, so a lane that has gone without a
|
|
50
|
-
* cross the global ceiling before its own row-age cap. The queue-keyed service term
|
|
51
|
-
* constant within a lane, preserving the existing intra-lane ordering and ceiling
|
|
50
|
+
* Aging adds the row's own eligible wait, multiplied by the lane's gap to the highest
|
|
51
|
+
* configured priority, and caps that wait at one `queuePriorityAgingUnitMs` for every lane. The
|
|
52
|
+
* lane service age is folded into this same primary rank, so a lane that has gone without a
|
|
53
|
+
* claim can cross the global ceiling before its own row-age cap. The queue-keyed service term
|
|
54
|
+
* remains constant within a lane, preserving the existing intra-lane ordering and ceiling
|
|
55
|
+
* alternation.
|
|
56
|
+
*
|
|
57
|
+
* The multiplier is what makes that ceiling reachable. Capping the raw wait at the gap instead
|
|
58
|
+
* (`(highest priority − priority + 1) × unitMs`) makes a lane wait as long as the gap it has to
|
|
59
|
+
* cancel: TensorBuzz's `auditRetention` lane (priority `-10` under a highest priority of `100`)
|
|
60
|
+
* needed ~111 minutes of eligible waiting, and its prune job was measured at 284–1,388 s per
|
|
61
|
+
* claim on 2026-11-19 while the prune itself took 5–19 s. Multiplying a wait bounded by one unit
|
|
62
|
+
* by the same gap reaches the identical ceiling after one unit for every lane, so the configured
|
|
63
|
+
* priority still orders fresh work and still buys exactly one unit of queue-jumping, but no lane
|
|
64
|
+
* has to age for the width of the whole priority ladder before it becomes claimable.
|
|
52
65
|
*
|
|
53
66
|
* Without a prior lane claim, the legacy row-age behavior remains the fallback. After a lane has
|
|
54
67
|
* a durable claim stamp, its service age contributes directly to the rank, reducing the bound
|
|
@@ -80,19 +93,16 @@ export function buildQueuePriorityOrderSql({configuration, db, nowMs, laneFairne
|
|
|
80
93
|
|
|
81
94
|
const queueExpression = `COALESCE(${db.quoteColumn("queue")}, ${db.quote(DEFAULT_BACKGROUND_JOB_QUEUE)})`
|
|
82
95
|
const priorityRank = `CASE ${queueExpression} ${prioritized.map(([queue, priority]) => `WHEN ${db.quote(queue)} THEN ${priority}`).join(" ")} ELSE 0 END`
|
|
96
|
+
const priorityGap = `CASE ${queueExpression} ${prioritized.map(([queue, priority]) => `WHEN ${db.quote(queue)} THEN ${highestPriority - priority + 1}`).join(" ")} ELSE ${highestPriority + 1} END`
|
|
83
97
|
const unitMs = queuePriorityAgingUnitMs(configuration)
|
|
84
98
|
|
|
85
99
|
if (unitMs === null) return priorityRank
|
|
86
100
|
|
|
87
|
-
const highestCeilingMs = (highestPriority + 1) * unitMs
|
|
88
|
-
const agingCap = `CASE ${queueExpression} ${prioritized.map(([queue, priority]) => `WHEN ${db.quote(queue)} THEN ${(highestPriority - priority + 1) * unitMs}`).join(" ")} ELSE ${highestCeilingMs} END`
|
|
89
101
|
const scheduledAtColumn = db.quoteColumn("scheduled_at_ms")
|
|
90
|
-
|
|
102
|
+
const agingWindow = `CASE WHEN ${scheduledAtColumn} <= (${db.quote(nowMs)} - ${unitMs}) THEN ${unitMs} ELSE (${db.quote(nowMs)} - ${scheduledAtColumn}) END`
|
|
91
103
|
const laneFairness = laneFairnessRank ? ` + (${laneFairnessRank})` : ""
|
|
92
104
|
|
|
93
|
-
return `(${priorityRank} * ${unitMs} + `
|
|
94
|
-
`CASE WHEN ${scheduledAtColumn} <= (${db.quote(nowMs)} - ${agingCap}) THEN ${agingCap} ` +
|
|
95
|
-
`ELSE (${db.quote(nowMs)} - ${scheduledAtColumn}) END${laneFairness})`
|
|
105
|
+
return `(${priorityRank} * ${unitMs} + ${priorityGap} * ${agingWindow}${laneFairness})`
|
|
96
106
|
}
|
|
97
107
|
|
|
98
108
|
/**
|
|
@@ -154,10 +164,11 @@ export function buildLaneFairnessOrderSql({configuration, db, nowMs, jobsTable,
|
|
|
154
164
|
/**
|
|
155
165
|
* Builds the raw lane-fairness rank used to compose the primary queue rank. A configured capped
|
|
156
166
|
* lane without a claim stamp contributes no service-age bonus: without a claim there is no
|
|
157
|
-
* lane-service history to rank. Once its row
|
|
158
|
-
*
|
|
159
|
-
*
|
|
160
|
-
*
|
|
167
|
+
* lane-service history to rank. Once its row has aged past one aging unit — the point where its
|
|
168
|
+
* own lane reaches the shared ceiling — its configured priority is retained as a small
|
|
169
|
+
* primary-key tie signal so a deep backlog cannot win solely by row age. A long-ago claim stamp
|
|
170
|
+
* contributes its full starvation age to the primary rank. A lane without a durable queue counter
|
|
171
|
+
* contributes zero and keeps the historical row-age fallback.
|
|
161
172
|
* @param {object} args - Options.
|
|
162
173
|
* @param {import("../configuration.js").default} args.configuration - Active configuration.
|
|
163
174
|
* @param {import("../database/drivers/base.js").default} args.db - Connection.
|
|
@@ -171,7 +182,6 @@ export function buildLaneFairnessRankSql({configuration, db, nowMs, jobsTable, c
|
|
|
171
182
|
if (unitMs === null) return null
|
|
172
183
|
|
|
173
184
|
const queues = configuration.getBackgroundJobsConfig().queues || {}
|
|
174
|
-
const highestPriority = Math.max(0, ...Object.values(queues).map((queueConfig) => Number.isFinite(queueConfig?.priority) ? Number(queueConfig.priority) : 0))
|
|
175
185
|
const queueExpression = `COALESCE(${jobsTable}.${db.quoteColumn("queue")}, ${db.quote(DEFAULT_BACKGROUND_JOB_QUEUE)})`
|
|
176
186
|
const cappedQueues = Object.entries(queues).filter(([, queueConfig]) => Number.isInteger(queueConfig?.maxConcurrent) && Number(queueConfig.maxConcurrent) > 0)
|
|
177
187
|
// Only queues with a `maxConcurrent` get a durable lane counter. With none configured there is
|
|
@@ -184,8 +194,7 @@ export function buildLaneFairnessRankSql({configuration, db, nowMs, jobsTable, c
|
|
|
184
194
|
|
|
185
195
|
const neverServedTie = `CASE ${queueExpression} ${cappedQueues.map(([queue, queueConfig]) => {
|
|
186
196
|
const priority = Number.isFinite(queueConfig?.priority) ? Number(queueConfig.priority) : 0
|
|
187
|
-
|
|
188
|
-
return `WHEN ${db.quote(queue)} THEN CASE WHEN ${jobsTable}.${db.quoteColumn("scheduled_at_ms")} <= (${Number(nowMs)} - ${agingCap}) THEN ${priority} ELSE 0 END`
|
|
197
|
+
return `WHEN ${db.quote(queue)} THEN CASE WHEN ${jobsTable}.${db.quoteColumn("scheduled_at_ms")} <= (${Number(nowMs)} - ${unitMs}) THEN ${priority} ELSE 0 END`
|
|
189
198
|
}).join(" ")} ELSE 0 END`
|
|
190
199
|
|
|
191
200
|
return `(CASE WHEN ${laneKeyCase} IS NULL THEN 0 WHEN ${lastClaimedLookup} IS NULL THEN (${neverServedTie}) ELSE (${Number(nowMs)} - ${lastClaimedLookup}) END)`
|