velocious 1.0.738 → 1.0.740

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -2510,7 +2510,7 @@ also repairs secondary indexes missed by older background-job add-column
2510
2510
  upgrades through a one-time internal migration, with conflict-safe SQLite index
2511
2511
  creation across generation processes.
2512
2512
 
2513
- Jobs can opt into cross-worker durable concurrency limits by pairing a non-empty `concurrencyKey` with a positive-integer `maxConcurrency` in their background-job options, or by deriving the key in a hydrated job instance's non-static `concurrencyKey()` method. Explicit enqueue options win. The first cap registered for a key is stable; conflicting caps are rejected. See [durable concurrency limits](docs/background-jobs.md#durable-concurrency-limits).
2513
+ Jobs can opt into cross-worker durable concurrency limits by pairing a non-empty `concurrencyKey` with a positive-integer `maxConcurrency` in their background-job options, or by deriving the key in a hydrated job instance's non-static `concurrencyKey()` method. Explicit enqueue options win. The first cap registered for a key is stable; conflicting caps are rejected. A key's counter row is removed when its last live job settles, so keys that name a one-off unit of work do not accumulate rows forever; queue-derived lane counters are kept. See [durable concurrency limits](docs/background-jobs.md#durable-concurrency-limits).
2514
2514
 
2515
2515
  Production apps can listen for `background-job-failed` (or its `all-error` mirror) to report accepted failed attempts, including retry and terminal-state metadata. Process-level pooled-runner failures also carry one shared `context.runnerFailure` snapshot for every affected job, with active handoff identities, child instance/PID, runner/worker lifecycle, exit code/signal, an exact typed shutdown reason and request/observation timestamps, bounded in-flight job IDs, and an explicit nullable OOM verdict. Every handed-off job also records the accepting runner's `child_instance_id`, `child_pid`, `runner_hostname` and start time; `BackgroundJobRecord#ownerIsAlive()` reports whether that recorded local process still exists (`alive`, `dead`, or `unknown` for remote/unrecorded owners), so claim ownership can be checked without polling. The deprecated `terminationReason` compatibility category remains present, while new consumers should use authoritative `shutdownReason`. Parent timeout/stop/retirement requests are recorded before signalling; normal drained retirement is permitted only with zero tracked jobs. Listen for `background-job-orphaned` to react to a specific job the main process reclaimed after its worker died mid-run — e.g. enqueue a targeted recovery for the work it left behind, instead of only polling for the aftermath. The orphan payload carries the dead owner's `childInstanceId`, `childPid`, `runnerHostname` and `workerId`, so a handler can release exactly that owner's claims on the event it already receives. Orphan handlers run before the sweep waits for reclaimed jobs to be dispatched, so a stalled dispatcher does not delay application recovery. An orphan without a delivered runner-failure report has unknown shutdown cause; do not infer OOM, deploy, timeout, signal, or normal retirement from acceptance evidence alone. See [docs/background-jobs.md](docs/background-jobs.md#failure-events) and [runner identity and owner liveness](docs/background-jobs.md#runner-identity-and-owner-liveness).
2516
2516
 
@@ -153,9 +153,11 @@ export function buildLaneFairnessOrderSql({configuration, db, nowMs, jobsTable,
153
153
 
154
154
  /**
155
155
  * Builds the raw lane-fairness rank used to compose the primary queue rank. A configured capped
156
- * lane without a claim stamp receives the full global ceiling boost as never-served work; a lane
157
- * without a durable queue counter contributes zero and keeps the historical row-age fallback.
158
- * A long-ago claim stamp contributes its full starvation age to the primary rank.
156
+ * lane without a claim stamp contributes no service-age bonus: without a claim there is no
157
+ * lane-service history to rank. Once its row reaches its own cap, its configured priority is
158
+ * retained as a small primary-key tie signal so a deep backlog cannot win solely by row age. A
159
+ * long-ago claim stamp contributes its full starvation age to the primary rank. A lane without a
160
+ * durable queue counter contributes zero and keeps the historical row-age fallback.
159
161
  * @param {object} args - Options.
160
162
  * @param {import("../configuration.js").default} args.configuration - Active configuration.
161
163
  * @param {import("../database/drivers/base.js").default} args.db - Connection.
@@ -170,7 +172,6 @@ export function buildLaneFairnessRankSql({configuration, db, nowMs, jobsTable, c
170
172
 
171
173
  const queues = configuration.getBackgroundJobsConfig().queues || {}
172
174
  const highestPriority = Math.max(0, ...Object.values(queues).map((queueConfig) => Number.isFinite(queueConfig?.priority) ? Number(queueConfig.priority) : 0))
173
- const neverServedBoost = (highestPriority + 1) * unitMs
174
175
  const queueExpression = `COALESCE(${jobsTable}.${db.quoteColumn("queue")}, ${db.quote(DEFAULT_BACKGROUND_JOB_QUEUE)})`
175
176
  const cappedQueues = Object.entries(queues).filter(([, queueConfig]) => Number.isInteger(queueConfig?.maxConcurrent) && Number(queueConfig.maxConcurrent) > 0)
176
177
  // Only queues with a `maxConcurrent` get a durable lane counter. With none configured there is
@@ -181,5 +182,11 @@ export function buildLaneFairnessRankSql({configuration, db, nowMs, jobsTable, c
181
182
  `(SELECT ${concurrencyTable}.${db.quoteColumn("last_claimed_at_ms")} FROM ${concurrencyTable} ` +
182
183
  `WHERE ${concurrencyTable}.${db.quoteColumn("concurrency_key")} = ${laneKeyCase})`
183
184
 
184
- return `(CASE WHEN ${laneKeyCase} IS NULL THEN 0 WHEN ${lastClaimedLookup} IS NULL THEN ${neverServedBoost} ELSE (${Number(nowMs)} - ${lastClaimedLookup}) END)`
185
+ const neverServedTie = `CASE ${queueExpression} ${cappedQueues.map(([queue, queueConfig]) => {
186
+ const priority = Number.isFinite(queueConfig?.priority) ? Number(queueConfig.priority) : 0
187
+ const agingCap = (highestPriority - priority + 1) * unitMs
188
+ return `WHEN ${db.quote(queue)} THEN CASE WHEN ${jobsTable}.${db.quoteColumn("scheduled_at_ms")} <= (${Number(nowMs)} - ${agingCap}) THEN ${priority} ELSE 0 END`
189
+ }).join(" ")} ELSE 0 END`
190
+
191
+ return `(CASE WHEN ${laneKeyCase} IS NULL THEN 0 WHEN ${lastClaimedLookup} IS NULL THEN (${neverServedTie}) ELSE (${Number(nowMs)} - ${lastClaimedLookup}) END)`
185
192
  }
@@ -12,6 +12,7 @@ import {evaluateBackgroundJobOwnerLiveness, isProcessAlive} from "./owner-livene
12
12
  import { coordinateSharedTransactionConnection } from "../testing/shared-transaction-connection-coordinator.js"
13
13
  import stableJsonStringify from "../utils/stable-json.js"
14
14
  import {
15
+ BACKGROUND_JOB_ACTIVE_STATUSES,
15
16
  BACKGROUND_JOB_TERMINAL_STATUSES,
16
17
  BACKGROUND_JOB_EXECUTION_MODES,
17
18
  DEFAULT_BACKGROUND_JOB_EXECUTION_MODE,
@@ -81,6 +82,13 @@ import {
81
82
  * @property {string} concurrency_key - Durable cap identity.
82
83
  */
83
84
 
85
+ /**
86
+ * Handed-off row count for one lane, aggregated off the jobs table by queue.
87
+ * @typedef {object} BackgroundJobQueueActiveCountRow
88
+ * @property {number | string} active_count - Handed-off rows in the lane.
89
+ * @property {string} queue - Lane name.
90
+ */
91
+
84
92
  /**
85
93
  * BackgroundJobQueuedConcurrency type.
86
94
  * @typedef {object} BackgroundJobQueuedConcurrency
@@ -1211,6 +1219,10 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
1211
1219
  `${concurrencyTable}.${db.quoteColumn("concurrency_key")} = ${jobsTable}.${db.quoteColumn("concurrency_key")} AND ` +
1212
1220
  `${concurrencyTable}.${db.quoteColumn("active_count")} < ${concurrencyTable}.${db.quoteColumn("max_concurrency")}))`
1213
1221
  )
1222
+
1223
+ const laneCapacity = this._laneCapacityFilterSql({concurrencyTable, db, jobsTable})
1224
+
1225
+ if (laneCapacity) query = query.where(laneCapacity)
1214
1226
  }
1215
1227
 
1216
1228
  if (executionMode) query = this._whereExecutionMode({db, executionMode, query})
@@ -1322,10 +1334,13 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
1322
1334
  const jobsTable = db.quoteTable(JOBS_TABLE)
1323
1335
  const concurrencyTable = db.quoteTable(CONCURRENCY_TABLE)
1324
1336
  const queueExpression = `COALESCE(${jobsTable}.${db.quoteColumn("queue")}, ${db.quote(DEFAULT_BACKGROUND_JOB_QUEUE)})`
1325
- const eligible = `(${jobsTable}.${db.quoteColumn("concurrency_key")} IS NULL OR EXISTS (` +
1337
+ const ownCapacity = `(${jobsTable}.${db.quoteColumn("concurrency_key")} IS NULL OR EXISTS (` +
1326
1338
  `SELECT 1 FROM ${concurrencyTable} WHERE ` +
1327
1339
  `${concurrencyTable}.${db.quoteColumn("concurrency_key")} = ${jobsTable}.${db.quoteColumn("concurrency_key")} AND ` +
1328
1340
  `${concurrencyTable}.${db.quoteColumn("active_count")} < ${concurrencyTable}.${db.quoteColumn("max_concurrency")}))`
1341
+ const eligible = [ownCapacity, this._laneCapacityFilterSql({concurrencyTable, db, jobsTable})]
1342
+ .filter((term) => term !== "")
1343
+ .join(" AND ")
1329
1344
  const backlogRows = await db.query(`
1330
1345
  SELECT ${queueExpression} AS queue, COUNT(*) AS backlog_depth, MIN(${jobsTable}.${db.quoteColumn("scheduled_at_ms")}) AS oldest_scheduled_at_ms
1331
1346
  FROM ${jobsTable}
@@ -1524,7 +1539,10 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
1524
1539
  const queuedJob = await this._reconcileQueuedJobConcurrency(db, selectedJob)
1525
1540
 
1526
1541
  if (!queuedJob) return null
1527
- if (queuedJob.concurrencyKey && !(await this._reserveConcurrency(db, queuedJob.concurrencyKey))) return null
1542
+
1543
+ const reservedKeys = await this._reserveJobConcurrency(db, queuedJob)
1544
+
1545
+ if (!reservedKeys) return null
1528
1546
  const affectedRows = await this._updateAffectedRows(db, {
1529
1547
  tableName: JOBS_TABLE,
1530
1548
  data: {
@@ -1538,7 +1556,7 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
1538
1556
  })
1539
1557
 
1540
1558
  if (affectedRows !== 1) {
1541
- await this._releaseConcurrency(db, queuedJob.concurrencyKey)
1559
+ await this._releaseConcurrencyKeys(db, reservedKeys)
1542
1560
  return null
1543
1561
  }
1544
1562
 
@@ -1576,7 +1594,7 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
1576
1594
  if (!job) return false
1577
1595
  if (!this._shouldAcceptReport({job, handoffId, workerId, handedOffAtMs})) return false
1578
1596
 
1579
- await this._lockConcurrencyRow(db, job.concurrencyKey)
1597
+ await this._lockConcurrencyRows(db, this._jobConcurrencyKeys(job))
1580
1598
  const affectedRows = await this._updateAffectedRows(db, {
1581
1599
  tableName: JOBS_TABLE,
1582
1600
  data: {
@@ -1588,7 +1606,8 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
1588
1606
 
1589
1607
  if (affectedRows !== 1) return false
1590
1608
  await this._releaseScheduleOwnershipForJob(db, job)
1591
- await this._releaseConcurrency(db, job.concurrencyKey)
1609
+ await this._releaseJobConcurrency(db, job)
1610
+ await this._pruneSettledConcurrencyKey(db, job.concurrencyKey)
1592
1611
  await this._recordStatusTransition(db, "handed_off", "completed")
1593
1612
  return true
1594
1613
  })
@@ -1678,7 +1697,7 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
1678
1697
  if (!job) return false
1679
1698
  if (!this._shouldAcceptReport({job, handoffId, workerId, handedOffAtMs})) return false
1680
1699
 
1681
- await this._lockConcurrencyRow(db, job.concurrencyKey)
1700
+ await this._lockConcurrencyRows(db, this._jobConcurrencyKeys(job))
1682
1701
  const scheduledAtMs = this._rescheduledAtMs(delayMs)
1683
1702
  const affectedRows = await this._updateAffectedRows(db, {
1684
1703
  tableName: JOBS_TABLE,
@@ -1694,7 +1713,7 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
1694
1713
  })
1695
1714
 
1696
1715
  if (affectedRows !== 1) return false
1697
- await this._releaseConcurrency(db, job.concurrencyKey)
1716
+ await this._releaseJobConcurrency(db, job)
1698
1717
  await this._recordStatusTransition(db, "handed_off", "queued")
1699
1718
  return true
1700
1719
  })
@@ -1713,7 +1732,7 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
1713
1732
  await this._serializedCountMutation(async (db) => {
1714
1733
  const job = await this._getJobRowById(db, jobId)
1715
1734
  if (!job || job.handoffId !== handoffId || job.status !== "handed_off") return
1716
- await this._lockConcurrencyRow(db, job.concurrencyKey)
1735
+ await this._lockConcurrencyRows(db, this._jobConcurrencyKeys(job))
1717
1736
  const affectedRows = await this._updateAffectedRows(db, {
1718
1737
  tableName: JOBS_TABLE,
1719
1738
  data: {
@@ -1727,7 +1746,7 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
1727
1746
  conditions: {handoff_id: handoffId, id: jobId, status: "handed_off"}
1728
1747
  })
1729
1748
  if (affectedRows === 1) {
1730
- await this._releaseConcurrency(db, job.concurrencyKey)
1749
+ await this._releaseJobConcurrency(db, job)
1731
1750
  await this._recordStatusTransition(db, "handed_off", "queued")
1732
1751
  }
1733
1752
  })
@@ -2208,11 +2227,12 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
2208
2227
  if (!job || (job.status !== "queued" && job.status !== "handed_off")) return false
2209
2228
  // Only a handed_off job holds a concurrency reservation, so only that case touches the
2210
2229
  // shared counter row and needs the concurrency-then-job lock ordering.
2211
- if (job.status === "handed_off") await this._lockConcurrencyRow(db, job.concurrencyKey)
2230
+ if (job.status === "handed_off") await this._lockConcurrencyRows(db, this._jobConcurrencyKeys(job))
2212
2231
  const affectedRows = await this._updateAffectedRows(db, {tableName: JOBS_TABLE, data: {status: "cancelled"}, conditions: {id: job.id, status: job.status}})
2213
2232
  if (affectedRows !== 1) return false
2214
2233
  await this._releaseScheduleOwnershipForJob(db, job)
2215
- if (job.status === "handed_off") await this._releaseConcurrency(db, job.concurrencyKey)
2234
+ if (job.status === "handed_off") await this._releaseJobConcurrency(db, job)
2235
+ await this._pruneSettledConcurrencyKey(db, job.concurrencyKey)
2216
2236
  await this._recordStatusTransition(db, job.status, "cancelled")
2217
2237
  return true
2218
2238
  })
@@ -3407,7 +3427,7 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
3407
3427
  shouldRetry
3408
3428
  })
3409
3429
 
3410
- await this._lockConcurrencyRow(db, job.concurrencyKey)
3430
+ await this._lockConcurrencyRows(db, this._jobConcurrencyKeys(job))
3411
3431
  const affectedRows = await this._updateAffectedRows(db, {
3412
3432
  tableName: JOBS_TABLE,
3413
3433
  data: update,
@@ -3416,7 +3436,8 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
3416
3436
 
3417
3437
  if (affectedRows !== 1) return null
3418
3438
  if (!shouldRetry) await this._releaseScheduleOwnershipForJob(db, job)
3419
- await this._releaseConcurrency(db, job.concurrencyKey)
3439
+ await this._releaseJobConcurrency(db, job)
3440
+ await this._pruneSettledConcurrencyKey(db, job.concurrencyKey)
3420
3441
 
3421
3442
  // Return a snapshot of the transition this update just applied rather than re-reading the row.
3422
3443
  // We won the conditional update (affectedRows === 1), so this state is authoritative; re-reading
@@ -3644,6 +3665,172 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
3644
3665
  return null
3645
3666
  }
3646
3667
 
3668
+ /**
3669
+ * The lane counter a queue enforces, or null when the queue has no configured cap.
3670
+ *
3671
+ * A job's own `concurrencyKey` isolates one record (`cleanup-build-run-output-<id>`), so it
3672
+ * bounds that record and never the lane. While only the job's own key was reserved, a queue's
3673
+ * `maxConcurrent` ranked its lane without limiting admission for any job that carried a key of
3674
+ * its own, which is how 85 per-record cleanup jobs each holding `maxConcurrency: 1` filled a
3675
+ * 100-slot pooled runner inside a lane configured `maxConcurrent: 200`.
3676
+ * @param {string | null} queue - Queue name as persisted on the job row.
3677
+ * @returns {{concurrencyKey: string, maxConcurrency: number} | null} - Lane cap, or null when the lane is uncapped.
3678
+ */
3679
+ _laneConcurrencyForQueue(queue) {
3680
+ const normalizedQueue = queue || DEFAULT_BACKGROUND_JOB_QUEUE
3681
+ const maxConcurrency = this._queueMaxConcurrency(normalizedQueue)
3682
+
3683
+ if (maxConcurrency === null) return null
3684
+
3685
+ return {concurrencyKey: laneConcurrencyKey(normalizedQueue), maxConcurrency}
3686
+ }
3687
+
3688
+ /**
3689
+ * The lane counter keys the current configuration owns, mapped to their queue. A leftover
3690
+ * `queue:`-prefixed counter for a queue the configuration no longer caps is absent, so it keeps
3691
+ * the per-key accounting {@link _reconcileQueueConcurrency} releases it into.
3692
+ * @returns {Map<string, string>} - Lane counter key to queue name.
3693
+ */
3694
+ _laneQueueByConcurrencyKey() {
3695
+ const queues = this.configuration.getBackgroundJobsConfig().queues || {}
3696
+ /** @type {Map<string, string>} */
3697
+ const laneQueues = new Map()
3698
+
3699
+ for (const queue of Object.keys(queues)) {
3700
+ if (this._queueMaxConcurrency(queue) === null) continue
3701
+
3702
+ laneQueues.set(laneConcurrencyKey(queue), queue)
3703
+ }
3704
+
3705
+ return laneQueues
3706
+ }
3707
+
3708
+ /**
3709
+ * The queue a counter key caps, or null when it is a per-record key or a lane row the
3710
+ * configuration no longer bounds.
3711
+ * @param {string} concurrencyKey - Counter key.
3712
+ * @returns {string | null} - Queue name, or null.
3713
+ */
3714
+ _laneQueueForConcurrencyKey(concurrencyKey) {
3715
+ return this._laneQueueByConcurrencyKey().get(concurrencyKey) ?? null
3716
+ }
3717
+
3718
+ /**
3719
+ * Every counter a job holds a reservation on while it is `handed_off`: its own key and its
3720
+ * lane key. Returned sorted so every transition locks the pair in the same order — the AB-BA
3721
+ * cycle {@link _lockConcurrencyRow} exists to prevent reappears as soon as a second counter
3722
+ * joins a transaction. Deduplicated because a queue-derived job's key already is its lane key.
3723
+ * @param {import("./types.js").BackgroundJobRow} job - Job row.
3724
+ * @returns {string[]} - Counter keys in lock order.
3725
+ */
3726
+ _jobConcurrencyKeys(job) {
3727
+ /** @type {string[]} */
3728
+ const keys = []
3729
+
3730
+ if (job.concurrencyKey) keys.push(job.concurrencyKey)
3731
+
3732
+ const lane = this._laneConcurrencyForQueue(job.queue)
3733
+
3734
+ if (lane && !keys.includes(lane.concurrencyKey)) keys.push(lane.concurrencyKey)
3735
+
3736
+ return keys.sort()
3737
+ }
3738
+
3739
+ /**
3740
+ * Locks several counters in the caller's key order, which {@link _jobConcurrencyKeys} keeps
3741
+ * sorted across every handoff, completion, failure, reschedule and cancel path.
3742
+ * @param {import("../database/drivers/base.js").default} db - Database connection.
3743
+ * @param {string[]} concurrencyKeys - Counter keys in lock order.
3744
+ * @returns {Promise<void>} - Resolves when every counter row is locked.
3745
+ */
3746
+ async _lockConcurrencyRows(db, concurrencyKeys) {
3747
+ for (const concurrencyKey of concurrencyKeys) await this._lockConcurrencyRow(db, concurrencyKey)
3748
+ }
3749
+
3750
+ /**
3751
+ * Reserves every counter a job needs for its handoff, in lock order, and hands the whole
3752
+ * partial reservation back when any one of them is already at its cap. Returning the reserved
3753
+ * keys rather than a boolean lets the caller release exactly what it took when the fenced job
3754
+ * update loses the race.
3755
+ * @param {import("../database/drivers/base.js").default} db - Database connection.
3756
+ * @param {import("./types.js").BackgroundJobRow} job - Queued job snapshot.
3757
+ * @returns {Promise<string[] | null>} - Reserved keys in lock order, or null when the job may not claim.
3758
+ */
3759
+ async _reserveJobConcurrency(db, job) {
3760
+ const lane = this._laneConcurrencyForQueue(job.queue)
3761
+
3762
+ if (lane) await this._ensureQueueConcurrencyKey(db, lane)
3763
+
3764
+ /** @type {string[]} */
3765
+ const reserved = []
3766
+
3767
+ for (const concurrencyKey of this._jobConcurrencyKeys(job)) {
3768
+ if (await this._reserveConcurrency(db, concurrencyKey)) {
3769
+ reserved.push(concurrencyKey)
3770
+ continue
3771
+ }
3772
+
3773
+ await this._releaseConcurrencyKeys(db, reserved)
3774
+
3775
+ return null
3776
+ }
3777
+
3778
+ return reserved
3779
+ }
3780
+
3781
+ /**
3782
+ * Releases a fixed set of counters.
3783
+ * @param {import("../database/drivers/base.js").default} db - Database connection.
3784
+ * @param {string[]} concurrencyKeys - Counter keys.
3785
+ * @returns {Promise<void>} - Resolves when released.
3786
+ */
3787
+ async _releaseConcurrencyKeys(db, concurrencyKeys) {
3788
+ for (const concurrencyKey of concurrencyKeys) await this._releaseConcurrency(db, concurrencyKey)
3789
+ }
3790
+
3791
+ /**
3792
+ * Releases every counter a handed-off job holds, matching its reservation exactly.
3793
+ * @param {import("../database/drivers/base.js").default} db - Database connection.
3794
+ * @param {import("./types.js").BackgroundJobRow} job - Handed-off job row.
3795
+ * @returns {Promise<void>} - Resolves when released.
3796
+ */
3797
+ async _releaseJobConcurrency(db, job) {
3798
+ await this._releaseConcurrencyKeys(db, this._jobConcurrencyKeys(job))
3799
+ }
3800
+
3801
+ /**
3802
+ * Builds the claim-gate term that excludes every lane already at its `maxConcurrent` ceiling.
3803
+ *
3804
+ * The job-key gate below cannot express a lane bound: a lane whose rows each carry their own
3805
+ * `concurrencyKey` never consults the lane counter, so ranking alone cannot stop that lane
3806
+ * consuming the whole pool. This term makes the lane counter part of eligibility, so a capped
3807
+ * lane at its ceiling is skipped and dispatch falls through to the next lane — the same
3808
+ * fall-through the ranking already relies on for a lane at its own key cap.
3809
+ *
3810
+ * A lane with no counter row yet is not treated as full: {@link _reserveJobConcurrency} and
3811
+ * {@link _recordLaneClaim} create the row before the lane's first claim, and
3812
+ * {@link reconcileQueueConcurrency} syncs it to the configured cap at startup.
3813
+ * @param {object} args - Options.
3814
+ * @param {import("../database/drivers/base.js").default} args.db - Database connection.
3815
+ * @param {string} args.concurrencyTable - Quoted concurrency counter table.
3816
+ * @param {string} args.jobsTable - Quoted jobs table.
3817
+ * @returns {string} - Raw SQL `WHERE` term, or an empty string when no queue is capped.
3818
+ */
3819
+ _laneCapacityFilterSql({concurrencyTable, db, jobsTable}) {
3820
+ const queues = this.configuration.getBackgroundJobsConfig().queues || {}
3821
+ const cappedQueues = Object.keys(queues).filter((queue) => this._queueMaxConcurrency(queue) !== null)
3822
+
3823
+ if (cappedQueues.length === 0) return ""
3824
+
3825
+ const queueExpression = `COALESCE(${jobsTable}.${db.quoteColumn("queue")}, ${db.quote(DEFAULT_BACKGROUND_JOB_QUEUE)})`
3826
+ const laneKeyCase = `CASE ${queueExpression} ${cappedQueues.map((queue) => `WHEN ${db.quote(queue)} THEN ${db.quote(laneConcurrencyKey(queue))}`).join(" ")} ELSE NULL END`
3827
+ const laneLookup = (/** @type {string} */ capacityCondition) =>
3828
+ `SELECT 1 FROM ${concurrencyTable} WHERE ${concurrencyTable}.${db.quoteColumn("concurrency_key")} = ${laneKeyCase}${capacityCondition}`
3829
+ const laneHasCapacity = laneLookup(` AND ${concurrencyTable}.${db.quoteColumn("active_count")} < ${concurrencyTable}.${db.quoteColumn("max_concurrency")}`)
3830
+
3831
+ return `(NOT EXISTS (${laneLookup("")}) OR EXISTS (${laneHasCapacity}))`
3832
+ }
3833
+
3647
3834
  /**
3648
3835
  * Like {@link _ensureConcurrencyKey}, but for queue-derived keys the configured
3649
3836
  * queue cap is the source of truth: if it changed, update the stored cap
@@ -4093,6 +4280,40 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
4093
4280
  await db.query(`UPDATE ${table} SET ${count} = ${count} - 1 WHERE ${db.quoteColumn("concurrency_key")} = ${db.quote(concurrencyKey)} AND ${count} > 0`)
4094
4281
  }
4095
4282
 
4283
+ /**
4284
+ * Removes a counter row once its last live job has settled.
4285
+ *
4286
+ * Keys derived from a unit of work that never comes back — one key per build-group completion
4287
+ * generation, for example — otherwise keep a row for every generation that ever ran, because
4288
+ * nothing after the terminal transition touches them again. The removal rides the transition
4289
+ * that made the key useless instead of a scheduled sweep: it runs in the same transaction that
4290
+ * published the terminal status, and the guard is evaluated in the single statement, so a key
4291
+ * that still has queued or handed-off work, or a reservation still held, keeps its row. Every
4292
+ * enqueue and every terminal transition takes the serialized count-revision lock, so this check
4293
+ * and delete cannot interleave with an enqueue that would have needed the row.
4294
+ *
4295
+ * Queue-derived keys are excluded: `queue:<name>` is a config-owned lane counter that startup
4296
+ * reconciliation and the lane-fairness tie-break read directly, so dropping one would change
4297
+ * queue behavior rather than just remove a dead counter.
4298
+ * @param {import("../database/drivers/base.js").default} db - Transaction connection.
4299
+ * @param {string | null} concurrencyKey - Concurrency key.
4300
+ * @returns {Promise<boolean>} - Whether the counter row was removed.
4301
+ */
4302
+ async _pruneSettledConcurrencyKey(db, concurrencyKey) {
4303
+ if (!concurrencyKey || concurrencyKey.startsWith(QUEUE_CONCURRENCY_KEY_PREFIX)) return false
4304
+
4305
+ const keyColumn = db.quoteColumn("concurrency_key")
4306
+ const liveStatuses = BACKGROUND_JOB_ACTIVE_STATUSES.map((status) => db.quote(status)).join(", ")
4307
+
4308
+ return (await db.affectedRows(
4309
+ `DELETE FROM ${db.quoteTable(CONCURRENCY_TABLE)} ` +
4310
+ `WHERE ${keyColumn} = ${db.quote(concurrencyKey)} ` +
4311
+ `AND ${db.quoteColumn("active_count")} = 0 ` +
4312
+ `AND NOT EXISTS (SELECT 1 FROM ${db.quoteTable(JOBS_TABLE)} ` +
4313
+ `WHERE ${keyColumn} = ${db.quote(concurrencyKey)} AND ${db.quoteColumn("status")} IN (${liveStatuses}))`
4314
+ )) === 1
4315
+ }
4316
+
4096
4317
  /**
4097
4318
  * Rebuilds durable counts from active handoffs.
4098
4319
  * @param {import("../database/drivers/base.js").default} db - Database connection.
@@ -4130,6 +4351,29 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
4130
4351
  activeCounts.set(row.concurrency_key, this._validatedConcurrencyCount(row.active_count, row.concurrency_key))
4131
4352
  }
4132
4353
 
4354
+ // A lane counter covers every handed-off row of its queue, including the rows that isolate
4355
+ // themselves with their own concurrency key, because that is what its reservation counts.
4356
+ // Rebuilding it from the key grouping alone would report 0 for a lane whose work all carries
4357
+ // per-record keys and free the ceiling the moment the reconciliation ran.
4358
+ const laneQueues = this._laneQueueByConcurrencyKey()
4359
+
4360
+ if (laneQueues.size > 0) {
4361
+ const queueExpression = `COALESCE(${db.quoteColumn("queue")}, ${db.quote(DEFAULT_BACKGROUND_JOB_QUEUE)})`
4362
+ const laneRows = await db.query(
4363
+ `SELECT ${queueExpression} AS queue, COUNT(*) AS active_count FROM ${db.quoteTable(JOBS_TABLE)} ` +
4364
+ `WHERE ${db.quoteColumn("status")} = ${db.quote("handed_off")} GROUP BY ${queueExpression}`
4365
+ )
4366
+
4367
+ for (const rawRow of laneRows) {
4368
+ const row = /** @type {BackgroundJobQueueActiveCountRow} */ (rawRow)
4369
+ const laneKey = laneConcurrencyKey(String(row.queue))
4370
+
4371
+ if (!laneQueues.has(laneKey)) continue
4372
+
4373
+ activeCounts.set(laneKey, this._validatedConcurrencyCount(row.active_count, laneKey))
4374
+ }
4375
+ }
4376
+
4133
4377
  for (const rawRow of staleRows) {
4134
4378
  const row = /** @type {BackgroundJobConcurrencyCountRow} */ (rawRow)
4135
4379
  persistedCounts.set(row.concurrency_key, this._validatedConcurrencyCount(row.active_count, row.concurrency_key))
@@ -4185,12 +4429,25 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
4185
4429
 
4186
4430
  const persistedRow = /** @type {BackgroundJobConcurrencyCountRow} */ (persistedRows[0])
4187
4431
  const previousActiveCount = this._validatedConcurrencyCount(persistedRow.active_count, concurrencyKey)
4188
- const rows = await db
4189
- .newQuery()
4190
- .from(JOBS_TABLE)
4191
- .select("COUNT(*) AS active_count")
4192
- .where({concurrency_key: concurrencyKey, status: "handed_off"})
4193
- .results()
4432
+ const laneQueue = this._laneQueueForConcurrencyKey(concurrencyKey)
4433
+ /** @type {Array<ReturnType<typeof JSON.parse>>} */
4434
+ let rows
4435
+
4436
+ if (laneQueue === null) {
4437
+ rows = await db
4438
+ .newQuery()
4439
+ .from(JOBS_TABLE)
4440
+ .select("COUNT(*) AS active_count")
4441
+ .where({concurrency_key: concurrencyKey, status: "handed_off"})
4442
+ .results()
4443
+ } else {
4444
+ rows = await db.query(
4445
+ `SELECT COUNT(*) AS active_count FROM ${db.quoteTable(JOBS_TABLE)} ` +
4446
+ `WHERE ${db.quoteColumn("status")} = ${db.quote("handed_off")} ` +
4447
+ `AND COALESCE(${db.quoteColumn("queue")}, ${db.quote(DEFAULT_BACKGROUND_JOB_QUEUE)}) = ${db.quote(laneQueue)}`
4448
+ )
4449
+ }
4450
+
4194
4451
  const countRow = /** @type {{active_count: number | string}} */ (rows[0])
4195
4452
  const activeCount = this._validatedConcurrencyCount(countRow.active_count, concurrencyKey)
4196
4453
 
@@ -100,9 +100,11 @@ export declare function buildLaneFairnessOrderSql({ configuration, db, nowMs, jo
100
100
  }): string | null;
101
101
  /**
102
102
  * Builds the raw lane-fairness rank used to compose the primary queue rank. A configured capped
103
- * lane without a claim stamp receives the full global ceiling boost as never-served work; a lane
104
- * without a durable queue counter contributes zero and keeps the historical row-age fallback.
105
- * A long-ago claim stamp contributes its full starvation age to the primary rank.
103
+ * lane without a claim stamp contributes no service-age bonus: without a claim there is no
104
+ * lane-service history to rank. Once its row reaches its own cap, its configured priority is
105
+ * retained as a small primary-key tie signal so a deep backlog cannot win solely by row age. A
106
+ * long-ago claim stamp contributes its full starvation age to the primary rank. A lane without a
107
+ * durable queue counter contributes zero and keeps the historical row-age fallback.
106
108
  * @param {object} args - Options.
107
109
  * @param {import("../configuration.js").default} args.configuration - Active configuration.
108
110
  * @param {import("../database/drivers/base.js").default} args.db - Connection.
@@ -1 +1 @@
1
- {"version":3,"file":"queue-priority-order.d.ts","sourceRoot":"","sources":["../../../src/background-jobs/queue-priority-order.js"],"names":[],"mappings":"AAIA;;;;;;GAMG;AACH,eAAO,MAAM,oCAAoC,QAAQ,CAAA;AAEzD;;;;;GAKG;AACH,wBAAgB,iCAAiC,CAAC,KAAK,EAH5C,OAG4C,GAF1C,MAAM,GAAG,IAAI,CAQzB;AAED;;;;GAIG;AACH,wBAAgB,wBAAwB,CAAC,aAAa,EAH3C,OAAO,qBAAqB,EAAE,OAGa,GAFzC,MAAM,GAAG,IAAI,CAIzB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,wBAAgB,0BAA0B,CAAC,EAAC,aAAa,EAAE,EAAE,EAAE,KAAK,EAAE,gBAAgB,EAAC,EANpF;IAAoD,aAAa,EAAzD,OAAO,qBAAqB,EAAE,OAAO,CAC7C;IAA4D,EAAE,EAAtD,OAAO,6BAA6B,EAAE,OAAO,CACrD;IAAqB,KAAK,EAAlB,MAAM,CACd;IAA6B,gBAAgB,AAA7C,CACA,EADQ,MAAM,GAAG,IAAI,CACrB;CAEoF,GAF1E,MAAM,GAAG,IAAI,CAmCzB;AAED;;;;;GAKG;AACH,wBAAgB,kBAAkB,CAAC,KAAK,EAH7B,MAG6B,GAF3B,MAAM,CAIlB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACH,wBAAgB,yBAAyB,CAAC,EAAC,aAAa,EAAE,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,gBAAgB,EAAC,EAP9F;IAAoD,aAAa,EAAzD,OAAO,qBAAqB,EAAE,OAAO,CAC7C;IAA4D,EAAE,EAAtD,OAAO,6BAA6B,EAAE,OAAO,CACrD;IAAqB,KAAK,EAAlB,MAAM,CACd;IAAqB,SAAS,EAAtB,MAAM,CACd;IAAqB,gBAAgB,EAA7B,MAAM,CACd;CAE8F,GAFpF,MAAM,GAAG,IAAI,CAazB;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,wBAAwB,CAAC,EAAC,aAAa,EAAE,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,gBAAgB,EAAC,EAP7F;IAAoD,aAAa,EAAzD,OAAO,qBAAqB,EAAE,OAAO,CAC7C;IAA4D,EAAE,EAAtD,OAAO,6BAA6B,EAAE,OAAO,CACrD;IAAqB,KAAK,EAAlB,MAAM,CACd;IAAqB,SAAS,EAAtB,MAAM,CACd;IAAqB,gBAAgB,EAA7B,MAAM,CACd;CAE6F,GAFnF,MAAM,GAAG,IAAI,CAoBzB"}
1
+ {"version":3,"file":"queue-priority-order.d.ts","sourceRoot":"","sources":["../../../src/background-jobs/queue-priority-order.js"],"names":[],"mappings":"AAIA;;;;;;GAMG;AACH,eAAO,MAAM,oCAAoC,QAAQ,CAAA;AAEzD;;;;;GAKG;AACH,wBAAgB,iCAAiC,CAAC,KAAK,EAH5C,OAG4C,GAF1C,MAAM,GAAG,IAAI,CAQzB;AAED;;;;GAIG;AACH,wBAAgB,wBAAwB,CAAC,aAAa,EAH3C,OAAO,qBAAqB,EAAE,OAGa,GAFzC,MAAM,GAAG,IAAI,CAIzB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,wBAAgB,0BAA0B,CAAC,EAAC,aAAa,EAAE,EAAE,EAAE,KAAK,EAAE,gBAAgB,EAAC,EANpF;IAAoD,aAAa,EAAzD,OAAO,qBAAqB,EAAE,OAAO,CAC7C;IAA4D,EAAE,EAAtD,OAAO,6BAA6B,EAAE,OAAO,CACrD;IAAqB,KAAK,EAAlB,MAAM,CACd;IAA6B,gBAAgB,AAA7C,CACA,EADQ,MAAM,GAAG,IAAI,CACrB;CAEoF,GAF1E,MAAM,GAAG,IAAI,CAmCzB;AAED;;;;;GAKG;AACH,wBAAgB,kBAAkB,CAAC,KAAK,EAH7B,MAG6B,GAF3B,MAAM,CAIlB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACH,wBAAgB,yBAAyB,CAAC,EAAC,aAAa,EAAE,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,gBAAgB,EAAC,EAP9F;IAAoD,aAAa,EAAzD,OAAO,qBAAqB,EAAE,OAAO,CAC7C;IAA4D,EAAE,EAAtD,OAAO,6BAA6B,EAAE,OAAO,CACrD;IAAqB,KAAK,EAAlB,MAAM,CACd;IAAqB,SAAS,EAAtB,MAAM,CACd;IAAqB,gBAAgB,EAA7B,MAAM,CACd;CAE8F,GAFpF,MAAM,GAAG,IAAI,CAazB;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,wBAAwB,CAAC,EAAC,aAAa,EAAE,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,gBAAgB,EAAC,EAP7F;IAAoD,aAAa,EAAzD,OAAO,qBAAqB,EAAE,OAAO,CAC7C;IAA4D,EAAE,EAAtD,OAAO,6BAA6B,EAAE,OAAO,CACrD;IAAqB,KAAK,EAAlB,MAAM,CACd;IAAqB,SAAS,EAAtB,MAAM,CACd;IAAqB,gBAAgB,EAA7B,MAAM,CACd;CAE6F,GAFnF,MAAM,GAAG,IAAI,CAyBzB"}