velocious 1.0.732 → 1.0.733

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -3,7 +3,7 @@
3
3
  "velocious": "build/bin/velocious.js"
4
4
  },
5
5
  "name": "velocious",
6
- "version": "1.0.732",
6
+ "version": "1.0.733",
7
7
  "main": "build/index.js",
8
8
  "types": "build/index.d.ts",
9
9
  "files": [
@@ -21,7 +21,7 @@ import {
21
21
  rescheduledBackgroundJobAtMs,
22
22
  retryDelayMs
23
23
  } from "./job-semantics.js"
24
- import {buildLaneFairnessOrderSql, buildQueuePriorityOrderSql} from "./queue-priority-order.js"
24
+ import {buildLaneFairnessOrderSql, buildQueuePriorityOrderSql, laneConcurrencyKey} from "./queue-priority-order.js"
25
25
 
26
26
  export const LOCAL_BACKGROUND_JOBS_TABLE = "velocious_local_background_jobs"
27
27
  export const LOCAL_BACKGROUND_JOB_CONCURRENCY_TABLE = "velocious_local_background_job_concurrency"
@@ -1040,7 +1040,7 @@ export default class LocalBackgroundJobsStore {
1040
1040
  return null
1041
1041
  }
1042
1042
 
1043
- await this._recordLaneClaim(db, job.concurrencyKey, handedOffAtMs)
1043
+ await this._recordLaneClaim(db, job.queue, handedOffAtMs)
1044
1044
 
1045
1045
  return {handedOffAtMs, handoffId}
1046
1046
  }))
@@ -1387,16 +1387,24 @@ export default class LocalBackgroundJobsStore {
1387
1387
 
1388
1388
  /**
1389
1389
  * Stamps the instant a lane received a claim so {@link _laneFairnessOrderSql} can break the
1390
- * aging-ceiling tie by lane service rather than by backlog age. Only queue-derived counters
1391
- * name a lane: an explicit `concurrencyKey` groups unrelated jobs, so stamping it would
1392
- * couple lanes the queue configuration never joined.
1390
+ * aging-ceiling tie by lane service rather than by backlog age. The lane is the queue: a job
1391
+ * claimed under an explicit `concurrencyKey` still consumed its queue's turn, and leaving that
1392
+ * claim unstamped made the tie-break term equal the row's own age again — which the deepest
1393
+ * aged backlog won every cycle.
1393
1394
  * @param {import("../database/drivers/base.js").default} db - Connection.
1394
- * @param {string | null} concurrencyKey - Concurrency key of the job that just claimed.
1395
+ * @param {string | null} queue - Queue of the job that just claimed.
1395
1396
  * @param {number} claimedAtMs - Claim instant.
1396
1397
  * @returns {Promise<void>} - Resolves when the lane counter is stamped.
1397
1398
  */
1398
- async _recordLaneClaim(db, concurrencyKey, claimedAtMs) {
1399
- if (!concurrencyKey || !concurrencyKey.startsWith(QUEUE_CONCURRENCY_KEY_PREFIX)) return
1399
+ async _recordLaneClaim(db, queue, claimedAtMs) {
1400
+ const normalizedQueue = queue || DEFAULT_BACKGROUND_JOB_QUEUE
1401
+ const queues = this.configuration.getBackgroundJobsConfig().queues || {}
1402
+ const cap = queues[normalizedQueue]?.maxConcurrent
1403
+
1404
+ if (!Number.isInteger(cap) || Number(cap) <= 0) return
1405
+
1406
+ const concurrencyKey = laneConcurrencyKey(normalizedQueue)
1407
+ await this._ensureConcurrency(db, {concurrencyKey, maxConcurrency: Number(cap), queueDerived: true})
1400
1408
 
1401
1409
  const table = db.quoteTable(LOCAL_BACKGROUND_JOB_CONCURRENCY_TABLE)
1402
1410
 
@@ -1,6 +1,6 @@
1
1
  // @ts-check
2
2
 
3
- import {DEFAULT_BACKGROUND_JOB_QUEUE} from "./job-semantics.js"
3
+ import {DEFAULT_BACKGROUND_JOB_QUEUE, QUEUE_CONCURRENCY_KEY_PREFIX} from "./job-semantics.js"
4
4
 
5
5
  /**
6
6
  * How many milliseconds of eligible waiting cancel one unit of `queues[name].priority`.
@@ -95,6 +95,16 @@ export function buildQueuePriorityOrderSql({configuration, db, nowMs}) {
95
95
  `ELSE (${db.quote(nowMs)} - ${scheduledAtColumn}) END)`
96
96
  }
97
97
 
98
+ /**
99
+ * The counter row that represents a lane: the queue's own concurrency counter. An explicit
100
+ * `concurrencyKey` groups unrelated jobs, so it never names a lane.
101
+ * @param {string} queue - Normalized queue name.
102
+ * @returns {string} - Queue-derived concurrency key.
103
+ */
104
+ export function laneConcurrencyKey(queue) {
105
+ return `${QUEUE_CONCURRENCY_KEY_PREFIX}${queue}`
106
+ }
107
+
98
108
  /**
99
109
  * Builds the lane-fairness tie-break term for the queued-job claim: how long the candidate's
100
110
  * own lane has gone without receiving a claim, longest-starved-first.
@@ -106,10 +116,17 @@ export function buildQueuePriorityOrderSql({configuration, db, nowMs}) {
106
116
  * when that cap is far above what the fleet can actually run. Ordering by time-since-last-claim
107
117
  * makes the ceiling tie alternate between starved lanes.
108
118
  *
109
- * Intra-lane order is untouched, because every row of one lane shares the same counter row and
110
- * the term is therefore constant within a lane. Lanes with no counter row (an uncapped queue)
111
- * fall back to the row's own eligible wait, which reproduces the previous `scheduled_at_ms ASC`
112
- * ordering exactly.
119
+ * The lane is the queue, not the candidate's concurrency counter. Most of a busy lane's rows can
120
+ * sit under explicit per-group keys (TensorBuzz groups its `builds` backlog by
121
+ * `cleanup-build-run-output-<build group id>`), and those keys are deliberately not stamped —
122
+ * reading them would make the term equal the row's own age again and hand the tie back to the
123
+ * deepest backlog, which is the monopoly this term exists to break. Every row of a queue
124
+ * therefore reads its queue's counter, which keeps the term constant within a lane and leaves
125
+ * intra-lane order untouched.
126
+ *
127
+ * A queue with no counter row (no configured `maxConcurrent`, or a queue name the configuration
128
+ * does not declare) falls back to the row's own eligible wait, which reproduces the previous
129
+ * `scheduled_at_ms ASC` ordering exactly.
113
130
  *
114
131
  * Returns null when aging is opted out (`queuePriorityAgingUnitMs: null`): opting out asks for
115
132
  * strict queue-priority ordering, so no time-based reordering is applied.
@@ -124,10 +141,12 @@ export function buildQueuePriorityOrderSql({configuration, db, nowMs}) {
124
141
  export function buildLaneFairnessOrderSql({configuration, db, nowMs, jobsTable, concurrencyTable}) {
125
142
  if (queuePriorityAgingUnitMs(configuration) === null) return null
126
143
 
127
- const keyColumn = db.quoteColumn("concurrency_key")
144
+ const queues = configuration.getBackgroundJobsConfig().queues || {}
145
+ const queueExpression = `COALESCE(${jobsTable}.${db.quoteColumn("queue")}, ${db.quote(DEFAULT_BACKGROUND_JOB_QUEUE)})`
146
+ const laneKeyCase = `CASE ${queueExpression} ${Object.keys(queues).map((queue) => `WHEN ${db.quote(queue)} THEN ${db.quote(laneConcurrencyKey(queue))}`).join(" ")} ELSE NULL END`
128
147
  const lastClaimedLookup =
129
148
  `(SELECT ${concurrencyTable}.${db.quoteColumn("last_claimed_at_ms")} FROM ${concurrencyTable} ` +
130
- `WHERE ${concurrencyTable}.${keyColumn} = ${jobsTable}.${keyColumn})`
149
+ `WHERE ${concurrencyTable}.${db.quoteColumn("concurrency_key")} = ${laneKeyCase})`
131
150
 
132
151
  return `(${Number(nowMs)} - COALESCE(${lastClaimedLookup}, ${jobsTable}.${db.quoteColumn("scheduled_at_ms")})) DESC`
133
152
  }
@@ -27,7 +27,7 @@ import {
27
27
  rescheduledBackgroundJobAtMs,
28
28
  retryDelayMs
29
29
  } from "./job-semantics.js"
30
- import {buildLaneFairnessOrderSql, buildQueuePriorityOrderSql} from "./queue-priority-order.js"
30
+ import {buildLaneFairnessOrderSql, buildQueuePriorityOrderSql, laneConcurrencyKey} from "./queue-priority-order.js"
31
31
  import {
32
32
  MAIL_DELIVERY_OPERATIONS_TABLE,
33
33
  mailDeliveryOperationForJob,
@@ -1528,7 +1528,7 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
1528
1528
  return null
1529
1529
  }
1530
1530
 
1531
- await this._recordLaneClaim(db, queuedJob.concurrencyKey, handedOffAtMs)
1531
+ await this._recordLaneClaim(db, queuedJob.queue, handedOffAtMs)
1532
1532
  await this._recordStatusTransition(db, "queued", "handed_off")
1533
1533
  /** @type {import("./types.js").BackgroundJobRow} */
1534
1534
  const handedOffJob = {
@@ -4033,16 +4033,23 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
4033
4033
 
4034
4034
  /**
4035
4035
  * Stamps the instant a lane received a claim so {@link _laneFairnessOrderSql} can break the
4036
- * aging-ceiling tie by lane service rather than by backlog age. Only queue-derived counters
4037
- * name a lane: an explicit `concurrencyKey` groups unrelated jobs, so stamping it would
4038
- * couple lanes the queue configuration never joined.
4036
+ * aging-ceiling tie by lane service rather than by backlog age. The lane is the queue: a job
4037
+ * claimed under an explicit `concurrencyKey` still consumed its queue's turn, and leaving that
4038
+ * claim unstamped made the tie-break term equal the row's own age again — which the deepest
4039
+ * aged backlog won every cycle.
4039
4040
  * @param {import("../database/drivers/base.js").default} db - Database connection.
4040
- * @param {string | null} concurrencyKey - Concurrency key of the job that just claimed.
4041
+ * @param {string | null} queue - Queue of the job that just claimed.
4041
4042
  * @param {number} claimedAtMs - Claim instant.
4042
4043
  * @returns {Promise<void>} - Resolves when the lane counter is stamped.
4043
4044
  */
4044
- async _recordLaneClaim(db, concurrencyKey, claimedAtMs) {
4045
- if (!concurrencyKey || !concurrencyKey.startsWith(QUEUE_CONCURRENCY_KEY_PREFIX)) return
4045
+ async _recordLaneClaim(db, queue, claimedAtMs) {
4046
+ const normalizedQueue = queue || DEFAULT_BACKGROUND_JOB_QUEUE
4047
+ const cap = this._queueMaxConcurrency(normalizedQueue)
4048
+
4049
+ if (cap === null) return
4050
+
4051
+ const concurrencyKey = laneConcurrencyKey(normalizedQueue)
4052
+ await this._ensureQueueConcurrencyKey(db, {concurrencyKey, maxConcurrency: cap})
4046
4053
 
4047
4054
  const table = db.quoteTable(CONCURRENCY_TABLE)
4048
4055