velocious 1.0.732 → 1.0.733
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/background-jobs/local-store.js +16 -8
- package/build/background-jobs/queue-priority-order.js +26 -7
- package/build/background-jobs/store.js +15 -8
- package/build/src/background-jobs/local-store.d.ts +6 -5
- package/build/src/background-jobs/local-store.d.ts.map +1 -1
- package/build/src/background-jobs/local-store.js +15 -9
- package/build/src/background-jobs/queue-priority-order.d.ts +18 -4
- package/build/src/background-jobs/queue-priority-order.d.ts.map +1 -1
- package/build/src/background-jobs/queue-priority-order.js +26 -8
- package/build/src/background-jobs/store.d.ts +6 -5
- package/build/src/background-jobs/store.d.ts.map +1 -1
- package/build/src/background-jobs/store.js +14 -9
- package/package.json +1 -1
- package/src/background-jobs/local-store.js +16 -8
- package/src/background-jobs/queue-priority-order.js +26 -7
- package/src/background-jobs/store.js +15 -8
package/package.json
CHANGED
|
@@ -21,7 +21,7 @@ import {
|
|
|
21
21
|
rescheduledBackgroundJobAtMs,
|
|
22
22
|
retryDelayMs
|
|
23
23
|
} from "./job-semantics.js"
|
|
24
|
-
import {buildLaneFairnessOrderSql, buildQueuePriorityOrderSql} from "./queue-priority-order.js"
|
|
24
|
+
import {buildLaneFairnessOrderSql, buildQueuePriorityOrderSql, laneConcurrencyKey} from "./queue-priority-order.js"
|
|
25
25
|
|
|
26
26
|
export const LOCAL_BACKGROUND_JOBS_TABLE = "velocious_local_background_jobs"
|
|
27
27
|
export const LOCAL_BACKGROUND_JOB_CONCURRENCY_TABLE = "velocious_local_background_job_concurrency"
|
|
@@ -1040,7 +1040,7 @@ export default class LocalBackgroundJobsStore {
|
|
|
1040
1040
|
return null
|
|
1041
1041
|
}
|
|
1042
1042
|
|
|
1043
|
-
await this._recordLaneClaim(db, job.
|
|
1043
|
+
await this._recordLaneClaim(db, job.queue, handedOffAtMs)
|
|
1044
1044
|
|
|
1045
1045
|
return {handedOffAtMs, handoffId}
|
|
1046
1046
|
}))
|
|
@@ -1387,16 +1387,24 @@ export default class LocalBackgroundJobsStore {
|
|
|
1387
1387
|
|
|
1388
1388
|
/**
|
|
1389
1389
|
* Stamps the instant a lane received a claim so {@link _laneFairnessOrderSql} can break the
|
|
1390
|
-
* aging-ceiling tie by lane service rather than by backlog age.
|
|
1391
|
-
*
|
|
1392
|
-
*
|
|
1390
|
+
* aging-ceiling tie by lane service rather than by backlog age. The lane is the queue: a job
|
|
1391
|
+
* claimed under an explicit `concurrencyKey` still consumed its queue's turn, and leaving that
|
|
1392
|
+
* claim unstamped made the tie-break term equal the row's own age again — which the deepest
|
|
1393
|
+
* aged backlog won every cycle.
|
|
1393
1394
|
* @param {import("../database/drivers/base.js").default} db - Connection.
|
|
1394
|
-
* @param {string | null}
|
|
1395
|
+
* @param {string | null} queue - Queue of the job that just claimed.
|
|
1395
1396
|
* @param {number} claimedAtMs - Claim instant.
|
|
1396
1397
|
* @returns {Promise<void>} - Resolves when the lane counter is stamped.
|
|
1397
1398
|
*/
|
|
1398
|
-
async _recordLaneClaim(db,
|
|
1399
|
-
|
|
1399
|
+
async _recordLaneClaim(db, queue, claimedAtMs) {
|
|
1400
|
+
const normalizedQueue = queue || DEFAULT_BACKGROUND_JOB_QUEUE
|
|
1401
|
+
const queues = this.configuration.getBackgroundJobsConfig().queues || {}
|
|
1402
|
+
const cap = queues[normalizedQueue]?.maxConcurrent
|
|
1403
|
+
|
|
1404
|
+
if (!Number.isInteger(cap) || Number(cap) <= 0) return
|
|
1405
|
+
|
|
1406
|
+
const concurrencyKey = laneConcurrencyKey(normalizedQueue)
|
|
1407
|
+
await this._ensureConcurrency(db, {concurrencyKey, maxConcurrency: Number(cap), queueDerived: true})
|
|
1400
1408
|
|
|
1401
1409
|
const table = db.quoteTable(LOCAL_BACKGROUND_JOB_CONCURRENCY_TABLE)
|
|
1402
1410
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
// @ts-check
|
|
2
2
|
|
|
3
|
-
import {DEFAULT_BACKGROUND_JOB_QUEUE} from "./job-semantics.js"
|
|
3
|
+
import {DEFAULT_BACKGROUND_JOB_QUEUE, QUEUE_CONCURRENCY_KEY_PREFIX} from "./job-semantics.js"
|
|
4
4
|
|
|
5
5
|
/**
|
|
6
6
|
* How many milliseconds of eligible waiting cancel one unit of `queues[name].priority`.
|
|
@@ -95,6 +95,16 @@ export function buildQueuePriorityOrderSql({configuration, db, nowMs}) {
|
|
|
95
95
|
`ELSE (${db.quote(nowMs)} - ${scheduledAtColumn}) END)`
|
|
96
96
|
}
|
|
97
97
|
|
|
98
|
+
/**
|
|
99
|
+
* The counter row that represents a lane: the queue's own concurrency counter. An explicit
|
|
100
|
+
* `concurrencyKey` groups unrelated jobs, so it never names a lane.
|
|
101
|
+
* @param {string} queue - Normalized queue name.
|
|
102
|
+
* @returns {string} - Queue-derived concurrency key.
|
|
103
|
+
*/
|
|
104
|
+
export function laneConcurrencyKey(queue) {
|
|
105
|
+
return `${QUEUE_CONCURRENCY_KEY_PREFIX}${queue}`
|
|
106
|
+
}
|
|
107
|
+
|
|
98
108
|
/**
|
|
99
109
|
* Builds the lane-fairness tie-break term for the queued-job claim: how long the candidate's
|
|
100
110
|
* own lane has gone without receiving a claim, longest-starved-first.
|
|
@@ -106,10 +116,17 @@ export function buildQueuePriorityOrderSql({configuration, db, nowMs}) {
|
|
|
106
116
|
* when that cap is far above what the fleet can actually run. Ordering by time-since-last-claim
|
|
107
117
|
* makes the ceiling tie alternate between starved lanes.
|
|
108
118
|
*
|
|
109
|
-
*
|
|
110
|
-
*
|
|
111
|
-
*
|
|
112
|
-
*
|
|
119
|
+
* The lane is the queue, not the candidate's concurrency counter. Most of a busy lane's rows can
|
|
120
|
+
* sit under explicit per-group keys (TensorBuzz groups its `builds` backlog by
|
|
121
|
+
* `cleanup-build-run-output-<build group id>`), and those keys are deliberately not stamped —
|
|
122
|
+
* reading them would make the term equal the row's own age again and hand the tie back to the
|
|
123
|
+
* deepest backlog, which is the monopoly this term exists to break. Every row of a queue
|
|
124
|
+
* therefore reads its queue's counter, which keeps the term constant within a lane and leaves
|
|
125
|
+
* intra-lane order untouched.
|
|
126
|
+
*
|
|
127
|
+
* A queue with no counter row (no configured `maxConcurrent`, or a queue name the configuration
|
|
128
|
+
* does not declare) falls back to the row's own eligible wait, which reproduces the previous
|
|
129
|
+
* `scheduled_at_ms ASC` ordering exactly.
|
|
113
130
|
*
|
|
114
131
|
* Returns null when aging is opted out (`queuePriorityAgingUnitMs: null`): opting out asks for
|
|
115
132
|
* strict queue-priority ordering, so no time-based reordering is applied.
|
|
@@ -124,10 +141,12 @@ export function buildQueuePriorityOrderSql({configuration, db, nowMs}) {
|
|
|
124
141
|
export function buildLaneFairnessOrderSql({configuration, db, nowMs, jobsTable, concurrencyTable}) {
|
|
125
142
|
if (queuePriorityAgingUnitMs(configuration) === null) return null
|
|
126
143
|
|
|
127
|
-
const
|
|
144
|
+
const queues = configuration.getBackgroundJobsConfig().queues || {}
|
|
145
|
+
const queueExpression = `COALESCE(${jobsTable}.${db.quoteColumn("queue")}, ${db.quote(DEFAULT_BACKGROUND_JOB_QUEUE)})`
|
|
146
|
+
const laneKeyCase = `CASE ${queueExpression} ${Object.keys(queues).map((queue) => `WHEN ${db.quote(queue)} THEN ${db.quote(laneConcurrencyKey(queue))}`).join(" ")} ELSE NULL END`
|
|
128
147
|
const lastClaimedLookup =
|
|
129
148
|
`(SELECT ${concurrencyTable}.${db.quoteColumn("last_claimed_at_ms")} FROM ${concurrencyTable} ` +
|
|
130
|
-
`WHERE ${concurrencyTable}.${
|
|
149
|
+
`WHERE ${concurrencyTable}.${db.quoteColumn("concurrency_key")} = ${laneKeyCase})`
|
|
131
150
|
|
|
132
151
|
return `(${Number(nowMs)} - COALESCE(${lastClaimedLookup}, ${jobsTable}.${db.quoteColumn("scheduled_at_ms")})) DESC`
|
|
133
152
|
}
|
|
@@ -27,7 +27,7 @@ import {
|
|
|
27
27
|
rescheduledBackgroundJobAtMs,
|
|
28
28
|
retryDelayMs
|
|
29
29
|
} from "./job-semantics.js"
|
|
30
|
-
import {buildLaneFairnessOrderSql, buildQueuePriorityOrderSql} from "./queue-priority-order.js"
|
|
30
|
+
import {buildLaneFairnessOrderSql, buildQueuePriorityOrderSql, laneConcurrencyKey} from "./queue-priority-order.js"
|
|
31
31
|
import {
|
|
32
32
|
MAIL_DELIVERY_OPERATIONS_TABLE,
|
|
33
33
|
mailDeliveryOperationForJob,
|
|
@@ -1528,7 +1528,7 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
|
|
|
1528
1528
|
return null
|
|
1529
1529
|
}
|
|
1530
1530
|
|
|
1531
|
-
await this._recordLaneClaim(db, queuedJob.
|
|
1531
|
+
await this._recordLaneClaim(db, queuedJob.queue, handedOffAtMs)
|
|
1532
1532
|
await this._recordStatusTransition(db, "queued", "handed_off")
|
|
1533
1533
|
/** @type {import("./types.js").BackgroundJobRow} */
|
|
1534
1534
|
const handedOffJob = {
|
|
@@ -4033,16 +4033,23 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
|
|
|
4033
4033
|
|
|
4034
4034
|
/**
|
|
4035
4035
|
* Stamps the instant a lane received a claim so {@link _laneFairnessOrderSql} can break the
|
|
4036
|
-
* aging-ceiling tie by lane service rather than by backlog age.
|
|
4037
|
-
*
|
|
4038
|
-
*
|
|
4036
|
+
* aging-ceiling tie by lane service rather than by backlog age. The lane is the queue: a job
|
|
4037
|
+
* claimed under an explicit `concurrencyKey` still consumed its queue's turn, and leaving that
|
|
4038
|
+
* claim unstamped made the tie-break term equal the row's own age again — which the deepest
|
|
4039
|
+
* aged backlog won every cycle.
|
|
4039
4040
|
* @param {import("../database/drivers/base.js").default} db - Database connection.
|
|
4040
|
-
* @param {string | null}
|
|
4041
|
+
* @param {string | null} queue - Queue of the job that just claimed.
|
|
4041
4042
|
* @param {number} claimedAtMs - Claim instant.
|
|
4042
4043
|
* @returns {Promise<void>} - Resolves when the lane counter is stamped.
|
|
4043
4044
|
*/
|
|
4044
|
-
async _recordLaneClaim(db,
|
|
4045
|
-
|
|
4045
|
+
async _recordLaneClaim(db, queue, claimedAtMs) {
|
|
4046
|
+
const normalizedQueue = queue || DEFAULT_BACKGROUND_JOB_QUEUE
|
|
4047
|
+
const cap = this._queueMaxConcurrency(normalizedQueue)
|
|
4048
|
+
|
|
4049
|
+
if (cap === null) return
|
|
4050
|
+
|
|
4051
|
+
const concurrencyKey = laneConcurrencyKey(normalizedQueue)
|
|
4052
|
+
await this._ensureQueueConcurrencyKey(db, {concurrencyKey, maxConcurrency: cap})
|
|
4046
4053
|
|
|
4047
4054
|
const table = db.quoteTable(CONCURRENCY_TABLE)
|
|
4048
4055
|
|