velocious 1.0.728 → 1.0.730

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. package/README.md +1 -1
  2. package/build/background-jobs/adapter.js +13 -1
  3. package/build/background-jobs/local-store.js +10 -16
  4. package/build/background-jobs/main.js +65 -2
  5. package/build/background-jobs/queue-priority-order.js +96 -0
  6. package/build/background-jobs/store.js +138 -29
  7. package/build/background-jobs/types.js +10 -0
  8. package/build/configuration-types.js +9 -0
  9. package/build/configuration.js +7 -1
  10. package/build/database/database-name.js +25 -0
  11. package/build/database/drivers/base.js +26 -0
  12. package/build/database/drivers/mssql/index.js +10 -0
  13. package/build/database/drivers/mysql/index.js +10 -0
  14. package/build/database/drivers/pgsql/index.js +10 -0
  15. package/build/database/drivers/sqlite/base.js +13 -0
  16. package/build/database/query/model-class-query.js +85 -0
  17. package/build/database/record/index.js +33 -0
  18. package/build/src/background-jobs/adapter.d.ts +17 -1
  19. package/build/src/background-jobs/adapter.d.ts.map +1 -1
  20. package/build/src/background-jobs/adapter.js +13 -2
  21. package/build/src/background-jobs/local-store.d.ts +5 -2
  22. package/build/src/background-jobs/local-store.d.ts.map +1 -1
  23. package/build/src/background-jobs/local-store.js +11 -15
  24. package/build/src/background-jobs/main.d.ts +16 -0
  25. package/build/src/background-jobs/main.d.ts.map +1 -1
  26. package/build/src/background-jobs/main.js +58 -3
  27. package/build/src/background-jobs/queue-priority-order.d.ts +55 -0
  28. package/build/src/background-jobs/queue-priority-order.d.ts.map +1 -0
  29. package/build/src/background-jobs/queue-priority-order.js +89 -0
  30. package/build/src/background-jobs/store.d.ts +71 -14
  31. package/build/src/background-jobs/store.d.ts.map +1 -1
  32. package/build/src/background-jobs/store.js +129 -34
  33. package/build/src/background-jobs/types.d.ts +32 -0
  34. package/build/src/background-jobs/types.d.ts.map +1 -1
  35. package/build/src/background-jobs/types.js +11 -1
  36. package/build/src/configuration-types.d.ts +21 -0
  37. package/build/src/configuration-types.d.ts.map +1 -1
  38. package/build/src/configuration-types.js +10 -1
  39. package/build/src/configuration.d.ts.map +1 -1
  40. package/build/src/configuration.js +8 -2
  41. package/build/src/database/database-name.d.ts +9 -0
  42. package/build/src/database/database-name.d.ts.map +1 -0
  43. package/build/src/database/database-name.js +23 -0
  44. package/build/src/database/drivers/base.d.ts +16 -0
  45. package/build/src/database/drivers/base.d.ts.map +1 -1
  46. package/build/src/database/drivers/base.js +24 -1
  47. package/build/src/database/drivers/mssql/index.d.ts +5 -0
  48. package/build/src/database/drivers/mssql/index.d.ts.map +1 -1
  49. package/build/src/database/drivers/mssql/index.js +9 -1
  50. package/build/src/database/drivers/mysql/index.d.ts +5 -0
  51. package/build/src/database/drivers/mysql/index.d.ts.map +1 -1
  52. package/build/src/database/drivers/mysql/index.js +9 -1
  53. package/build/src/database/drivers/pgsql/index.d.ts +5 -0
  54. package/build/src/database/drivers/pgsql/index.d.ts.map +1 -1
  55. package/build/src/database/drivers/pgsql/index.js +9 -1
  56. package/build/src/database/drivers/sqlite/base.d.ts +8 -0
  57. package/build/src/database/drivers/sqlite/base.d.ts.map +1 -1
  58. package/build/src/database/drivers/sqlite/base.js +12 -1
  59. package/build/src/database/query/model-class-query.d.ts +20 -0
  60. package/build/src/database/query/model-class-query.d.ts.map +1 -1
  61. package/build/src/database/query/model-class-query.js +75 -1
  62. package/build/src/database/record/index.d.ts +46 -0
  63. package/build/src/database/record/index.d.ts.map +1 -1
  64. package/build/src/database/record/index.js +30 -1
  65. package/build/src/tenants/default-tenant-database-provisioning.d.ts +14 -0
  66. package/build/src/tenants/default-tenant-database-provisioning.d.ts.map +1 -1
  67. package/build/src/tenants/default-tenant-database-provisioning.js +25 -17
  68. package/build/tenants/default-tenant-database-provisioning.js +26 -20
  69. package/package.json +3 -2
  70. package/scripts/verify-dependency-refs.js +14 -0
  71. package/src/background-jobs/adapter.js +13 -1
  72. package/src/background-jobs/local-store.js +10 -16
  73. package/src/background-jobs/main.js +65 -2
  74. package/src/background-jobs/queue-priority-order.js +96 -0
  75. package/src/background-jobs/store.js +138 -29
  76. package/src/background-jobs/types.js +10 -0
  77. package/src/configuration-types.js +9 -0
  78. package/src/configuration.js +7 -1
  79. package/src/database/database-name.js +25 -0
  80. package/src/database/drivers/base.js +26 -0
  81. package/src/database/drivers/mssql/index.js +10 -0
  82. package/src/database/drivers/mysql/index.js +10 -0
  83. package/src/database/drivers/pgsql/index.js +10 -0
  84. package/src/database/drivers/sqlite/base.js +13 -0
  85. package/src/database/query/model-class-query.js +85 -0
  86. package/src/database/record/index.js +33 -0
  87. package/src/tenants/default-tenant-database-provisioning.js +26 -20
package/README.md CHANGED
@@ -2895,7 +2895,7 @@ backgroundJobs: {
2895
2895
 
2896
2896
  A job with no queue runs on `"default"`; a queue with no cap is unlimited. Caps are enforced through the durable per-key concurrency mechanism (the reserved `queue:<name>` key) and hold regardless of how many worker processes run. Queue-policy changes are adopted by queued backlog rows when `background-jobs-main` starts; already handed-off jobs drain under their original policy. If they return, reschedule, or retry, the reporting generation releases the original reservation without changing shared policy, and the active generation applies its current queue policy immediately before the next handoff and sends that committed policy to the worker; explicit concurrency remains unchanged. Handoff persistence is fenced against concurrent policy changes. Startup also rebuilds durable active counts under a database advisory lock. The active main then checks those counts every minute, performs no counter writes while they are exact, repairs only locked mismatches, logs a bounded structured repair summary, and immediately retries dispatch so stale capacity cannot require a restart. `db:migrate`, `db:tenants:*`, and routine store/application initialization with an intact jobs table never adopt queued jobs or rebuild global concurrency counts. If schema repair must recreate a missing `background_jobs` table while the migration marker and concurrency table survive, it resets the now-orphaned active counts against that newly empty table. Scheduled jobs honor a job's `static queue` too.
2897
2897
 
2898
- Set `priority` (default `0`) to dispatch a queue ahead of lower-priority ones regardless of enqueue order, so a small time-critical queue is never starved by a flood of low-priority work sharing a worker pool. Unlike Sidekiq's strict queue ordering, priority composes with the caps: a higher-priority queue already at its `maxConcurrent` is skipped and dispatch falls through to the next eligible job. See [docs/background-jobs.md](docs/background-jobs.md#queues-per-queue-concurrency-caps).
2898
+ Set `priority` (default `0`) to dispatch a queue ahead of lower-priority ones regardless of enqueue order, so a small time-critical queue is never starved by a flood of low-priority work sharing a worker pool. Unlike Sidekiq's strict queue ordering, priority composes with the caps: a higher-priority queue already at its `maxConcurrent` is skipped and dispatch falls through to the next eligible job. That fallthrough only happens at the cap, so a queue whose cap the fleet never actually reaches keeps every lower queue out of the claim set for as long as it holds any eligible row. `queuePriorityAgingUnitMs` (default `60000` milliseconds per priority unit, `null` to opt out) adds each row's own eligible wait to its rank, capped per queue at the gap to the highest configured priority plus one unit, so a queue can cancel exactly its own priority gap by waiting and no queue can rank above the shared ceiling — which is what makes the skip finite. A queue with eligible backlog that receives no claims is named on the framework-error channel as `background-jobs-lane-starvation` with its backlog depth, oldest eligible row and zero claims, once per entry into that state on the maintenance sweep the main already runs. See [docs/background-jobs.md](docs/background-jobs.md#queues-per-queue-concurrency-caps), [priority aging](docs/background-jobs.md#priority-aging), and [starved queues are named, not just deep](docs/background-jobs.md#starved-queues-are-named-not-just-deep).
2899
2899
 
2900
2900
  ## Retention
2901
2901
 
@@ -65,6 +65,18 @@ export default class BackgroundJobsAdapter {
65
65
  return {candidateCount: 0, checkedCount: 0, repairedCount: 0, repairs: [], repairsTruncatedCount: 0}
66
66
  }
67
67
 
68
+ /**
69
+ * Reports dispatch state per queue: eligible backlog depth, the oldest eligible
70
+ * schedule, and claims observed in the window. Adapters that cannot observe the
71
+ * claim loop keep this empty result.
72
+ * @param {object} [_args] - Options.
73
+ * @param {number} [_args.nowMs] - Observation instant.
74
+ * @param {number} [_args.starvationThresholdMs] - Age past which an unclaimed backlog is starved.
75
+ * @param {number} [_args.windowMs] - Claim observation window.
76
+ * @returns {Promise<import("./types.js").BackgroundJobLaneDiagnostics[]>} - One row per observed queue.
77
+ */
78
+ async readQueueDispatchDiagnostics(_args = {}) { return [] }
79
+
68
80
  /**
69
81
  * Enqueues a job.
70
82
  * @param {{jobName: string, args: Array<ReturnType<typeof JSON.parse>>, options?: import("./types.js").BackgroundJobOptions}} _args - Job request.
@@ -208,7 +220,7 @@ export default class BackgroundJobsAdapter {
208
220
 
209
221
  /**
210
222
  * Marks a handed-off job failed or retryable.
211
- * @param {{jobId: string, error: ReturnType<typeof JSON.parse>, handoffId?: string, workerId?: string, handedOffAtMs?: number}} _args - Failure report.
223
+ * @param {{jobId: string, error: ReturnType<typeof JSON.parse>, handoffId?: string, workerId?: string, handedOffAtMs?: number, runnerFailure?: import("./types.js").PooledRunnerFailure}} _args - Failure report.
212
224
  * @returns {Promise<import("./types.js").BackgroundJobRow | null>} - Updated job when accepted.
213
225
  */
214
226
  async markFailed(_args) { throw new Error("BackgroundJobsAdapter#markFailed is not implemented") }
@@ -21,6 +21,7 @@ import {
21
21
  rescheduledBackgroundJobAtMs,
22
22
  retryDelayMs
23
23
  } from "./job-semantics.js"
24
+ import {buildQueuePriorityOrderSql} from "./queue-priority-order.js"
24
25
 
25
26
  export const LOCAL_BACKGROUND_JOBS_TABLE = "velocious_local_background_jobs"
26
27
  export const LOCAL_BACKGROUND_JOB_CONCURRENCY_TABLE = "velocious_local_background_job_concurrency"
@@ -896,14 +897,15 @@ export default class LocalBackgroundJobsStore {
896
897
  await this.ensureReady()
897
898
 
898
899
  return await this._withDb(async (db) => {
900
+ const nowMs = this.clock.now()
899
901
  const jobsTable = db.quoteTable(LOCAL_BACKGROUND_JOBS_TABLE)
900
902
  const concurrencyTable = db.quoteTable(LOCAL_BACKGROUND_JOB_CONCURRENCY_TABLE)
901
- const priorityOrder = this._queuePriorityOrderSql(db)
903
+ const priorityOrder = this._queuePriorityOrderSql(db, nowMs)
902
904
  let query = db
903
905
  .newQuery()
904
906
  .from(LOCAL_BACKGROUND_JOBS_TABLE)
905
907
  .where({status: "queued"})
906
- .where(`scheduled_at_ms <= ${db.quote(this.clock.now())}`)
908
+ .where(`scheduled_at_ms <= ${db.quote(nowMs)}`)
907
909
  .where(
908
910
  `(${jobsTable}.${db.quoteColumn("concurrency_key")} IS NULL OR EXISTS (` +
909
911
  `SELECT 1 FROM ${concurrencyTable} WHERE ` +
@@ -1399,23 +1401,15 @@ export default class LocalBackgroundJobsStore {
1399
1401
  }
1400
1402
 
1401
1403
  /**
1402
- * Builds the configured queue-priority ordering expression.
1404
+ * Builds the configured queue-priority ordering expression, including the eligible-wait
1405
+ * aging term that keeps a low-priority lane reachable. See
1406
+ * {@link buildQueuePriorityOrderSql} for the ranking contract.
1403
1407
  * @param {import("../database/drivers/base.js").default} db - Connection.
1408
+ * @param {number} nowMs - Dispatch instant.
1404
1409
  * @returns {string | null} - Queue priority expression.
1405
1410
  */
1406
- _queuePriorityOrderSql(db) {
1407
- const queues = this.configuration.getBackgroundJobsConfig().queues
1408
- const prioritized = Object.entries(queues)
1409
- .filter(([, queue]) => Number.isFinite(queue?.priority) && Number(queue.priority) !== 0)
1410
- .map(([queueName, queue]) => [queueName, Number(queue.priority)])
1411
-
1412
- if (prioritized.length === 0) return null
1413
-
1414
- const whens = prioritized
1415
- .map(([queue, priority]) => `WHEN ${db.quote(queue)} THEN ${priority}`)
1416
- .join(" ")
1417
-
1418
- return `CASE COALESCE(${db.quoteColumn("queue")}, ${db.quote(DEFAULT_BACKGROUND_JOB_QUEUE)}) ${whens} ELSE 0 END`
1411
+ _queuePriorityOrderSql(db, nowMs) {
1412
+ return buildQueuePriorityOrderSql({configuration: this.configuration, db, nowMs})
1419
1413
  }
1420
1414
 
1421
1415
  /**
@@ -173,6 +173,11 @@ export default class BackgroundJobsMain {
173
173
  * Active durable handoffs keyed by the exact worker socket that received them.
174
174
  * @type {Map<JsonSocket, Map<string, string>>} */
175
175
  this.workerHandoffs = new Map()
176
+ /**
177
+ * Lane names already reported as queued-but-unclaimed, so an ongoing starvation is
178
+ * named once per lane rather than on every maintenance pass.
179
+ * @type {Set<string>} */
180
+ this._reportedStarvedLanes = new Set()
176
181
  /**
177
182
  * Exact caller-generated leases whose claim outcome was ambiguous or whose
178
183
  * pre-dispatch release has not yet been acknowledged. Retained until a
@@ -2006,7 +2011,8 @@ export default class BackgroundJobsMain {
2006
2011
  error: message.error,
2007
2012
  handoffId: message.handoffId,
2008
2013
  workerId: message.workerId,
2009
- handedOffAtMs: message.handedOffAtMs
2014
+ handedOffAtMs: message.handedOffAtMs,
2015
+ runnerFailure: message.runnerFailure
2010
2016
  })
2011
2017
 
2012
2018
  if (failedJob) {
@@ -2748,7 +2754,64 @@ export default class BackgroundJobsMain {
2748
2754
  errorEvents.emit("all-error", {...payload, errorType: "framework-error"})
2749
2755
  }
2750
2756
 
2751
- if (this.lifecycleState === "active") await this._reconcileActiveConcurrency()
2757
+ if (this.lifecycleState === "active") {
2758
+ await this._reconcileActiveConcurrency()
2759
+ await this._reportStarvedLanes()
2760
+ }
2761
+ }
2762
+
2763
+ /**
2764
+ * Names a lane that has eligible queued work but is not being claimed for it, on the
2765
+ * active main's existing maintenance cadence. A lane whose oldest eligible row waits past
2766
+ * the starvation threshold while claiming nothing in the observation window is not merely
2767
+ * deep — it is not being reached, and until now the only evidence was a backlog that kept
2768
+ * growing. Reports are edge-triggered per lane: one report when a lane enters the state,
2769
+ * none while it stays in it, and the lane is eligible to report again once it recovers.
2770
+ * This reads and reports only; it never dispatches, re-enqueues, or drains anything.
2771
+ * @returns {Promise<void>} - Resolves after the observation and any new report.
2772
+ */
2773
+ async _reportStarvedLanes() {
2774
+ try {
2775
+ const lanes = await this.store.readQueueDispatchDiagnostics()
2776
+ const starvedLanes = lanes.filter((lane) => lane.starved)
2777
+ const reportedLanes = this._reportedStarvedLanes
2778
+ const newlyStarvedLanes = starvedLanes.filter((lane) => !reportedLanes.has(lane.queue))
2779
+
2780
+ for (const lane of starvedLanes) reportedLanes.add(lane.queue)
2781
+
2782
+ const currentLaneNames = new Set(starvedLanes.map((lane) => lane.queue))
2783
+
2784
+ for (const laneName of [...reportedLanes]) {
2785
+ if (!currentLaneNames.has(laneName)) reportedLanes.delete(laneName)
2786
+ }
2787
+
2788
+ if (newlyStarvedLanes.length === 0) return
2789
+
2790
+ const summary = newlyStarvedLanes.map((lane) => {
2791
+ const oldestIso = lane.oldestScheduledAtMs === null ? "unknown" : new Date(lane.oldestScheduledAtMs).toISOString()
2792
+
2793
+ return `${lane.queue}: ${lane.backlogDepth} queued, oldest eligible ${oldestIso}, 0 claims`
2794
+ })
2795
+
2796
+ this.logger.warn(() => ["Background-jobs lanes queued but unclaimed:", summary.join("; ")])
2797
+
2798
+ const payload = {
2799
+ context: {generationId: this.generationId, stage: "background-jobs-lane-starvation", starvedLanes: newlyStarvedLanes},
2800
+ error: new Error(`Background-jobs lane starvation: ${summary.join("; ")}`)
2801
+ }
2802
+ const errorEvents = this.configuration.getErrorEvents()
2803
+
2804
+ errorEvents.emit("framework-error", payload)
2805
+ errorEvents.emit("all-error", {...payload, errorType: "framework-error"})
2806
+ } catch (error) {
2807
+ const normalizedError = error instanceof Error ? error : new Error(String(error))
2808
+ const payload = {context: {generationId: this.generationId, stage: "background-jobs-lane-starvation-read"}, error: normalizedError}
2809
+ const errorEvents = this.configuration.getErrorEvents()
2810
+
2811
+ this.logger.error(() => ["Failed to read background job lane diagnostics:", normalizedError])
2812
+ errorEvents.emit("framework-error", payload)
2813
+ errorEvents.emit("all-error", {...payload, errorType: "framework-error"})
2814
+ }
2752
2815
  }
2753
2816
 
2754
2817
  /**
@@ -0,0 +1,96 @@
1
+ // @ts-check
2
+
3
+ import {DEFAULT_BACKGROUND_JOB_QUEUE} from "./job-semantics.js"
4
+
5
+ /**
6
+ * How many milliseconds of eligible waiting cancel one unit of `queues[name].priority`.
7
+ * `1` priority unit buys this much queue-jumping: with the default, a lane ranked 10 points
8
+ * below another is overtaken once the lower lane has an eligible job that has been waiting
9
+ * 10 minutes. Time is the only condition — nothing here runs on a cadence, and a lane that
10
+ * is dispatched normally never ages far enough for the term to matter.
11
+ */
12
+ export const DEFAULT_QUEUE_PRIORITY_AGING_UNIT_MS = 60000
13
+
14
+ /**
15
+ * Normalizes `backgroundJobs.queuePriorityAgingUnitMs`. Invalid values fall back to the
16
+ * default; `null` opts out and restores the strict queue-priority ordering.
17
+ * @param {unknown} value - Configured value.
18
+ * @returns {number | null} - Milliseconds per priority unit, or null when aging is disabled.
19
+ */
20
+ export function normalizeQueuePriorityAgingUnitMs(value) {
21
+ if (value === null) return null
22
+ if (value === undefined) return DEFAULT_QUEUE_PRIORITY_AGING_UNIT_MS
23
+ if (!Number.isFinite(value) || Number(value) <= 0) return DEFAULT_QUEUE_PRIORITY_AGING_UNIT_MS
24
+
25
+ return Number(value)
26
+ }
27
+
28
+ /**
29
+ * Reads the configured aging unit.
30
+ * @param {import("../configuration.js").default} configuration - Active configuration.
31
+ * @returns {number | null} - Milliseconds per priority unit, or null when aging is disabled.
32
+ */
33
+ export function queuePriorityAgingUnitMs(configuration) {
34
+ return normalizeQueuePriorityAgingUnitMs(configuration.getBackgroundJobsConfig().queuePriorityAgingUnitMs)
35
+ }
36
+
37
+ /**
38
+ * Builds the dispatch ranking expression for the queued-job claim.
39
+ *
40
+ * `queues[name].priority` alone is a strict global order: the claim takes one job per cycle,
41
+ * so a lane only gets dispatched when every higher-priority lane is empty or already at its
42
+ * own `maxConcurrent` cap. A lane whose cap is far above what the fleet can actually run —
43
+ * a `builds` lane capped at 200 that never holds more than a dozen runners — is therefore
44
+ * never skipped, and every lane below it receives zero claims for as long as it stays
45
+ * non-empty.
46
+ *
47
+ * Aging adds the row's own eligible wait, scaled by the priority unit, and caps that wait per
48
+ * lane at the gap between the lane and the highest configured priority plus one unit. The cap
49
+ * is per lane so every lane converges on the same ceiling — `(highest priority + 1) × unitMs`
50
+ * — which is strictly above every row that has not aged to the ceiling. Below the ceiling the
51
+ * configured priority order is unchanged: a lane cancels exactly its own priority gap by
52
+ * waiting, and a lane can never outrank a lane that is itself still below the ceiling. At the
53
+ * ceiling all lanes are equal, so the claim falls back to `scheduled_at_ms` and the oldest
54
+ * overdue row is taken first.
55
+ *
56
+ * Starvation bound for a lane: it reaches the ceiling after
57
+ * `(highest configured priority - its own priority + 1) × unitMs` of eligible waiting, and at
58
+ * the ceiling it competes for the claim position on age alone.
59
+ * @param {object} args - Options.
60
+ * @param {import("../configuration.js").default} args.configuration - Active configuration.
61
+ * @param {import("../database/drivers/base.js").default} args.db - Connection.
62
+ * @param {number} args.nowMs - Dispatch instant.
63
+ * @returns {string | null} - Raw SQL rank expression, or null when no queue is prioritized.
64
+ */
65
+ export function buildQueuePriorityOrderSql({configuration, db, nowMs}) {
66
+ const queues = configuration.getBackgroundJobsConfig().queues || {}
67
+ /** @type {Array<[string, number]>} */
68
+ const prioritized = []
69
+ let highestPriority = 0
70
+
71
+ for (const [queue, queueConfig] of Object.entries(queues)) {
72
+ const priority = queueConfig?.priority
73
+
74
+ if (!Number.isFinite(priority)) continue
75
+
76
+ if (Number(priority) !== 0) prioritized.push([queue, Number(priority)])
77
+
78
+ highestPriority = Math.max(highestPriority, Number(priority))
79
+ }
80
+
81
+ if (prioritized.length === 0) return null
82
+
83
+ const queueExpression = `COALESCE(${db.quoteColumn("queue")}, ${db.quote(DEFAULT_BACKGROUND_JOB_QUEUE)})`
84
+ const priorityRank = `CASE ${queueExpression} ${prioritized.map(([queue, priority]) => `WHEN ${db.quote(queue)} THEN ${priority}`).join(" ")} ELSE 0 END`
85
+ const unitMs = queuePriorityAgingUnitMs(configuration)
86
+
87
+ if (unitMs === null) return priorityRank
88
+
89
+ const highestCeilingMs = (highestPriority + 1) * unitMs
90
+ const agingCap = `CASE ${queueExpression} ${prioritized.map(([queue, priority]) => `WHEN ${db.quote(queue)} THEN ${(highestPriority - priority + 1) * unitMs}`).join(" ")} ELSE ${highestCeilingMs} END`
91
+ const scheduledAtColumn = db.quoteColumn("scheduled_at_ms")
92
+
93
+ return `(${priorityRank} * ${unitMs} + ` +
94
+ `CASE WHEN ${scheduledAtColumn} <= (${db.quote(nowMs)} - ${agingCap}) THEN ${agingCap} ` +
95
+ `ELSE (${db.quote(nowMs)} - ${scheduledAtColumn}) END)`
96
+ }
@@ -27,6 +27,7 @@ import {
27
27
  rescheduledBackgroundJobAtMs,
28
28
  retryDelayMs
29
29
  } from "./job-semantics.js"
30
+ import {buildQueuePriorityOrderSql} from "./queue-priority-order.js"
30
31
  import {
31
32
  MAIL_DELIVERY_OPERATIONS_TABLE,
32
33
  mailDeliveryOperationForJob,
@@ -87,6 +88,21 @@ import {
87
88
  * @property {number | null} maxConcurrency - Current concurrency cap for queued work.
88
89
  */
89
90
 
91
+ /**
92
+ * Aggregated queued backlog for one lane, read straight off the jobs table.
93
+ * @typedef {object} BackgroundJobQueueBacklogRow
94
+ * @property {number | string} backlog_depth - Eligible queued rows in the lane.
95
+ * @property {number | string | null} oldest_scheduled_at_ms - Oldest eligible queued row.
96
+ * @property {string} queue - Lane name.
97
+ */
98
+
99
+ /**
100
+ * Aggregated claims for one lane inside the observation window.
101
+ * @typedef {object} BackgroundJobQueueClaimRow
102
+ * @property {number | string} claim_count - Rows the dispatcher handed off in the window.
103
+ * @property {string} queue - Lane name.
104
+ */
105
+
90
106
  const MIGRATIONS_TABLE = "velocious_internal_migrations"
91
107
  const MIGRATION_SCOPE = "background_jobs"
92
108
  const MIGRATION_VERSION = "20250215000000"
@@ -104,6 +120,22 @@ const JOBS_INDEX_REPAIR_MIGRATION_VERSION = "20260903120000"
104
120
  const LEGACY_POOLED_HANDOFF_ID_PREFIX = "velocious-pooled:"
105
121
  const LEGACY_POOLED_QUEUED_HANDOFF_ID = `${LEGACY_POOLED_HANDOFF_ID_PREFIX}queued`
106
122
  const JOBS_TABLE = "background_jobs"
123
+
124
+ /**
125
+ * Adds parent-side pooled-runner termination provenance to the durable error.
126
+ * @param {ReturnType<typeof JSON.parse>} error - Reported failure.
127
+ * @param {import("./types.js").PooledRunnerFailure} [runnerFailure] - Pooled-child failure provenance.
128
+ * @returns {ReturnType<typeof JSON.parse>} - Error with durable provenance.
129
+ */
130
+ function errorWithPooledRunnerProvenance(error, runnerFailure) {
131
+ if (!runnerFailure?.shutdownReason) return error
132
+
133
+ const message = normalizeBackgroundJobError(error)
134
+ const shutdownSignal = runnerFailure.shutdownSignal || "none"
135
+
136
+ return `${message}; pooledRunnerShutdownReason=${runnerFailure.shutdownReason} pooledRunnerShutdownSignal=${shutdownSignal}`
137
+ }
138
+
107
139
  const JOBS_INDEX_COLUMN_NAMES = [
108
140
  "job_name",
109
141
  "queue",
@@ -144,6 +176,16 @@ const JOB_TIMEOUT_VALIDATION_MESSAGE = `background job timeoutMs must be a finit
144
176
  const BACKGROUND_JOB_PRIORITY_VALIDATION_MESSAGE = "background job priority must be a non-negative integer"
145
177
  const ORPHANED_AFTER_MS = 2 * 60 * 60 * 1000
146
178
 
179
+ /**
180
+ * How long a lane's oldest eligible queued row may wait before the lane counts as starved.
181
+ * Normal dispatch claims an eligible job within one cycle, so a five-minute wait with no
182
+ * claim in the observation window is not ordinary queue depth — it is the lane not being
183
+ * reached, which is what {@link BackgroundJobsStore#readQueueDispatchDiagnostics} names.
184
+ */
185
+ export const QUEUE_LANE_STARVATION_THRESHOLD_MS = 5 * 60 * 1000
186
+ /** Window over which a lane's claims are counted for the starvation diagnostic. */
187
+ export const QUEUE_CLAIM_OBSERVATION_WINDOW_MS = 10 * 60 * 1000
188
+
147
189
  /**
148
190
  * Priority a job is demoted to when it is re-queued after a failed attempt.
149
191
  * A retry has already consumed capacity and time, so it must never overtake
@@ -1174,7 +1216,7 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
1174
1216
  if (executionMode) query = this._whereExecutionMode({db, executionMode, query})
1175
1217
 
1176
1218
  if (scheduledAtOperator === "<=") {
1177
- const priorityOrder = this._queuePriorityOrderSql(db)
1219
+ const priorityOrder = this._queuePriorityOrderSql(db, now)
1178
1220
 
1179
1221
  if (priorityOrder) query = query.order(`${priorityOrder} DESC`)
1180
1222
 
@@ -1200,36 +1242,102 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
1200
1242
 
1201
1243
  /**
1202
1244
  * Builds a raw SQL ORDER BY expression ranking queued jobs by their queue's
1203
- * configured priority (`backgroundJobs.queues[queue].priority`, default `0`),
1204
- * so the dispatcher picks higher-priority queues first regardless of enqueue
1205
- * order. Only applied to the dispatch path (`scheduledAtOperator === "<="`);
1206
- * the future-scheduled lookup must stay strictly time-ordered. Composes with
1207
- * the concurrency EXISTS filter: a higher-priority queue already at its cap is
1208
- * filtered out, so dispatch falls through to the next eligible lower-priority
1209
- * job. Returns null when no queue configures a non-zero priority so the plain
1210
- * FIFO ordering is left untouched (and no needless filesort is introduced).
1245
+ * configured priority (`backgroundJobs.queues[queue].priority`, default `0`) plus the
1246
+ * row's own eligible wait, so a lane cannot be starved by a higher-priority lane that
1247
+ * stays non-empty without ever reaching its own cap. See
1248
+ * {@link buildQueuePriorityOrderSql} for the ranking contract. Only applied to the
1249
+ * dispatch path (`scheduledAtOperator === "<="`); the future-scheduled lookup must stay
1250
+ * strictly time-ordered. Composes with the concurrency EXISTS filter: a higher-priority
1251
+ * queue already at its cap is filtered out, so dispatch falls through to the next eligible
1252
+ * lower-priority job. Returns null when no queue configures a non-zero priority so the
1253
+ * plain FIFO ordering is left untouched (and no needless filesort is introduced).
1211
1254
  * @param {import("../database/drivers/base.js").default} db - Database connection.
1212
- * @returns {string | null} - Raw SQL CASE expression, or null when no queue is prioritized.
1255
+ * @param {number} nowMs - Dispatch instant.
1256
+ * @returns {string | null} - Raw SQL rank expression, or null when no queue is prioritized.
1213
1257
  */
1214
- _queuePriorityOrderSql(db) {
1215
- const queues = this.configuration.getBackgroundJobsConfig().queues || {}
1216
- /** @type {Array<[string, number]>} */
1217
- const prioritized = []
1258
+ _queuePriorityOrderSql(db, nowMs) {
1259
+ return buildQueuePriorityOrderSql({configuration: this.configuration, db, nowMs})
1260
+ }
1218
1261
 
1219
- for (const [queue, queueConfig] of Object.entries(queues)) {
1220
- const priority = queueConfig?.priority
1262
+ /**
1263
+ * Reads the per-lane dispatch boundary for every queue holding eligible queued work:
1264
+ * backlog depth, the oldest eligible row, and how many jobs that lane actually claimed
1265
+ * inside the observation window. A lane whose oldest eligible row has waited past
1266
+ * `starvationThresholdMs` while claiming nothing in the window is starved, and this is the
1267
+ * named diagnostic for that state instead of a backlog that only grows.
1268
+ *
1269
+ * The backlog uses the dispatcher's own eligibility gate, so a lane that is legitimately
1270
+ * sitting at its `maxConcurrent` cap is not reported as starved.
1271
+ * @param {object} [args] - Options.
1272
+ * @param {number} [args.nowMs] - Observation instant. Defaults to the store clock.
1273
+ * @param {number} [args.starvationThresholdMs] - How long the oldest eligible row may wait before the lane counts as starved.
1274
+ * @param {number} [args.windowMs] - Window in which lane claims are counted. Default 10 minutes.
1275
+ * @returns {Promise<import("./types.js").BackgroundJobLaneDiagnostics[]>} - Per-lane diagnostics, oldest backlog first.
1276
+ */
1277
+ async readQueueDispatchDiagnostics({nowMs = this.clock.now(), starvationThresholdMs = QUEUE_LANE_STARVATION_THRESHOLD_MS, windowMs = QUEUE_CLAIM_OBSERVATION_WINDOW_MS} = {}) {
1278
+ await this.ensureReady()
1279
+
1280
+ const rows = await this._withDb(async (db) => {
1281
+ const jobsTable = db.quoteTable(JOBS_TABLE)
1282
+ const concurrencyTable = db.quoteTable(CONCURRENCY_TABLE)
1283
+ const queueExpression = `COALESCE(${jobsTable}.${db.quoteColumn("queue")}, ${db.quote(DEFAULT_BACKGROUND_JOB_QUEUE)})`
1284
+ const eligible = `(${jobsTable}.${db.quoteColumn("concurrency_key")} IS NULL OR EXISTS (` +
1285
+ `SELECT 1 FROM ${concurrencyTable} WHERE ` +
1286
+ `${concurrencyTable}.${db.quoteColumn("concurrency_key")} = ${jobsTable}.${db.quoteColumn("concurrency_key")} AND ` +
1287
+ `${concurrencyTable}.${db.quoteColumn("active_count")} < ${concurrencyTable}.${db.quoteColumn("max_concurrency")}))`
1288
+ const backlogRows = await db.query(`
1289
+ SELECT ${queueExpression} AS queue, COUNT(*) AS backlog_depth, MIN(${jobsTable}.${db.quoteColumn("scheduled_at_ms")}) AS oldest_scheduled_at_ms
1290
+ FROM ${jobsTable}
1291
+ WHERE ${jobsTable}.${db.quoteColumn("status")} = ${db.quote("queued")}
1292
+ AND ${jobsTable}.${db.quoteColumn("scheduled_at_ms")} <= ${db.quote(nowMs)}
1293
+ AND ${eligible}
1294
+ GROUP BY ${queueExpression}
1295
+ `)
1296
+ const claimRows = await db.query(`
1297
+ SELECT ${queueExpression} AS queue, COUNT(*) AS claim_count
1298
+ FROM ${jobsTable}
1299
+ WHERE ${jobsTable}.${db.quoteColumn("handed_off_at_ms")} >= ${db.quote(nowMs - windowMs)}
1300
+ GROUP BY ${queueExpression}
1301
+ `)
1302
+
1303
+ return {backlogRows, claimRows}
1304
+ })
1305
+
1306
+ const lanesByQueue = new Map()
1307
+
1308
+ for (const rawRow of rows.backlogRows) {
1309
+ const row = /** @type {BackgroundJobQueueBacklogRow} */ (rawRow)
1310
+ const queue = String(row.queue)
1221
1311
 
1222
- if (Number.isFinite(priority) && Number(priority) !== 0) prioritized.push([queue, Number(priority)])
1312
+ lanesByQueue.set(queue, {
1313
+ backlogDepth: this._validatedConcurrencyCount(row.backlog_depth, queue),
1314
+ claimsCount: 0,
1315
+ oldestScheduledAtMs: row.oldest_scheduled_at_ms === null || row.oldest_scheduled_at_ms === undefined
1316
+ ? null
1317
+ : Number(row.oldest_scheduled_at_ms),
1318
+ queue,
1319
+ starved: false
1320
+ })
1223
1321
  }
1224
1322
 
1225
- if (prioritized.length === 0) return null
1323
+ for (const rawRow of rows.claimRows) {
1324
+ const row = /** @type {BackgroundJobQueueClaimRow} */ (rawRow)
1325
+ const queue = String(row.queue)
1326
+ const lane = lanesByQueue.get(queue) || {backlogDepth: 0, claimsCount: 0, oldestScheduledAtMs: null, queue, starved: false}
1226
1327
 
1227
- const queueColumn = db.quoteColumn("queue")
1228
- const whens = prioritized
1229
- .map(([queue, priority]) => `WHEN ${db.quote(queue)} THEN ${priority}`)
1230
- .join(" ")
1328
+ lane.claimsCount = this._validatedConcurrencyCount(row.claim_count, queue)
1329
+ lanesByQueue.set(queue, lane)
1330
+ }
1331
+
1332
+ /** @type {import("./types.js").BackgroundJobLaneDiagnostics[]} */
1333
+ const diagnostics = []
1334
+
1335
+ for (const lane of lanesByQueue.values()) {
1336
+ lane.starved = lane.claimsCount === 0 && lane.oldestScheduledAtMs !== null && nowMs - lane.oldestScheduledAtMs >= starvationThresholdMs
1337
+ diagnostics.push(lane)
1338
+ }
1231
1339
 
1232
- return `CASE COALESCE(${queueColumn}, ${db.quote(DEFAULT_BACKGROUND_JOB_QUEUE)}) ${whens} ELSE 0 END`
1340
+ return diagnostics.sort((left, right) => (left.oldestScheduledAtMs ?? Number.MAX_SAFE_INTEGER) - (right.oldestScheduledAtMs ?? Number.MAX_SAFE_INTEGER))
1233
1341
  }
1234
1342
 
1235
1343
  /**
@@ -1773,9 +1881,10 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
1773
1881
  * @param {string} [args.handoffId] - Handoff lease id.
1774
1882
  * @param {string} [args.workerId] - Worker id.
1775
1883
  * @param {number} [args.handedOffAtMs] - Handed off timestamp.
1884
+ * @param {import("./types.js").PooledRunnerFailure} [args.runnerFailure] - Pooled-child process failure provenance.
1776
1885
  * @returns {Promise<import("./types.js").BackgroundJobRow | null>} - Updated job row when the report was accepted.
1777
1886
  */
1778
- async markFailed({jobId, error, handoffId, workerId, handedOffAtMs}) {
1887
+ async markFailed({jobId, error, handoffId, workerId, handedOffAtMs, runnerFailure}) {
1779
1888
  await this.ensureReady()
1780
1889
 
1781
1890
  return await this._serializedCountMutation(async (db) => {
@@ -1784,7 +1893,7 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
1784
1893
  if (!job) return null
1785
1894
  if (!this._shouldAcceptReport({job, handoffId, workerId, handedOffAtMs})) return null
1786
1895
 
1787
- const updatedJob = await this._applyFailure({db, job, error, markOrphaned: false})
1896
+ const updatedJob = await this._applyFailure({db, job, error: errorWithPooledRunnerProvenance(error, runnerFailure), markOrphaned: false})
1788
1897
 
1789
1898
  if (updatedJob) await this._recordStatusTransition(db, job.status, updatedJob.status)
1790
1899
  return updatedJob
@@ -3997,16 +4106,16 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
3997
4106
  }
3998
4107
 
3999
4108
  /**
4000
- * Validates a database count before it participates in reconciliation.
4109
+ * Validates a database count before it participates in a diagnostic or reconciliation.
4001
4110
  * @param {number | string} value - Raw count.
4002
- * @param {string} concurrencyKey - Counter key for diagnostics.
4111
+ * @param {string} counterName - Counter identity used in the failure message.
4003
4112
  * @returns {number} - Safe non-negative count.
4004
4113
  */
4005
- _validatedConcurrencyCount(value, concurrencyKey) {
4114
+ _validatedConcurrencyCount(value, counterName) {
4006
4115
  const count = this._normalizeNumber(value)
4007
4116
 
4008
4117
  if (count === null || !Number.isSafeInteger(count) || count < 0) {
4009
- throw new Error(`Invalid reconciled background job concurrency count for ${concurrencyKey}: ${count}`)
4118
+ throw new Error(`Invalid reconciled background job concurrency count for ${counterName}: ${count}`)
4010
4119
  }
4011
4120
 
4012
4121
  return count
@@ -121,6 +121,16 @@
121
121
  * @property {BackgroundJobConcurrencyRepair[]} repairs - Bounded deterministic sample of applied repairs.
122
122
  * @property {number} repairsTruncatedCount - Applied repairs omitted from the sample.
123
123
  */
124
+ /**
125
+ * One queue lane's dispatch boundary: how deep its eligible backlog is, how long its oldest
126
+ * eligible row has waited, and how many jobs the dispatcher actually claimed from it.
127
+ * @typedef {object} BackgroundJobLaneDiagnostics
128
+ * @property {number} backlogDepth - Eligible queued rows in the lane.
129
+ * @property {number} claimsCount - Rows handed off by the dispatcher inside the observation window.
130
+ * @property {number | null} oldestScheduledAtMs - Oldest eligible queued row, null when the lane is empty.
131
+ * @property {string} queue - Lane name.
132
+ * @property {boolean} starved - Backlog past the starvation threshold with zero claims in the window.
133
+ */
124
134
  /**
125
135
  * @typedef {object} PreparedLocalBackgroundJob
126
136
  * @property {string} argsDigest - Fixed-width digest of the serialized arguments.
@@ -241,6 +241,15 @@
241
241
  * Priorities may be any number, including negative to sink a queue below the
242
242
  * default. Jobs within the same priority keep FIFO (`scheduled_at`, then
243
243
  * `created_at`) order. Default: `{}` (no queue caps, all queues priority `0`).
244
+ * @property {number | null} [queuePriorityAgingUnitMs] - How many milliseconds of eligible
245
+ * waiting cancel one unit of `queues[name].priority` in the dispatch claim. Strict queue
246
+ * priority alone starves a lane that sits below a queue which stays non-empty without
247
+ * ever reaching its own `maxConcurrent` cap: the claim takes one job per cycle, so the
248
+ * lower lane is never selected. Aging adds each row's own eligible wait to its queue rank,
249
+ * capped at the full configured priority span, so the wait eventually cancels every gap
250
+ * and no lane can be starved for longer than `(highest priority - lowest priority) × unitMs`.
251
+ * Fresh work keeps its configured order exactly. Default: `60000` (one priority unit per
252
+ * minute of eligible waiting). `null` opts out and restores strict queue-priority ordering.
244
253
  * @property {number | null} [scheduledEnqueueTimeoutMs] - Bounds one scheduled enqueue attempt in the main process. A durable enqueue that never settles (wedged DB connection, contended advisory lock) would otherwise keep its schedule key permanently in flight, so every later tick skipped itself and shutdown could never drain. On timeout the attempt is reported as `background-jobs-scheduled-enqueue-stall` and the key clears for the next tick. Default: 20 minutes — deliberately 5 minutes above `advisoryLockAcquireTimeoutMs` so the precise lock error surfaces first. `null` opts out.
245
254
  * @property {BackgroundJobsDispatchStrategy} [dispatchStrategy] - How the main process
246
255
  * detects new work. Defaults to `"beacon"` (event-driven). Set to `"polling"`
@@ -26,6 +26,7 @@ import UUID from "pure-uuid"
26
26
  import translate from "gettext-universal/build/src/translate.js"
27
27
  import Ability from "./authorization/ability.js"
28
28
  import BackgroundJobsAdapter from "./background-jobs/adapter.js"
29
+ import {normalizeQueuePriorityAgingUnitMs} from "./background-jobs/queue-priority-order.js"
29
30
  import {DEFAULT_SCHEDULED_ENQUEUE_TIMEOUT_MS} from "./background-jobs/scheduled-enqueue-bound.js"
30
31
  import DatabaseOperation from "./database/operation.js"
31
32
  import { initializeAuditedModelRelationships } from "./database/record/auditing.js"
@@ -1953,6 +1954,11 @@ export default class VelociousConfiguration {
1953
1954
  )
1954
1955
  : DEFAULT_SCHEDULED_ENQUEUE_TIMEOUT_MS
1955
1956
  const queues = configured.queues && typeof configured.queues === "object" ? configured.queues : {}
1957
+ // `null` opts out of queue-priority aging and restores the strict configured priority
1958
+ // order. A present but non-positive or non-finite value is a mistake, not an opt-out, so
1959
+ // it falls back to the framework default instead of silently re-opening the starvation
1960
+ // path the aging term exists to close.
1961
+ const queuePriorityAgingUnitMs = normalizeQueuePriorityAgingUnitMs(configured.queuePriorityAgingUnitMs)
1956
1962
  // An explicit config value wins over the env var — including `null`/`0`,
1957
1963
  // which disable the backstop even when the environment sets a default.
1958
1964
  // Only fall through to the env var when config omits `jobTimeoutMs` entirely.
@@ -1977,7 +1983,7 @@ export default class VelociousConfiguration {
1977
1983
 
1978
1984
  const jobClasses = this.getBackgroundJobClasses()
1979
1985
 
1980
- return {host, port, databaseIdentifier, maxConcurrentForkedJobs, maxConcurrentInlineJobs, mode, pooledRunnerCount, pooledRunnerConcurrency, pooledRunnerMaxJobs, pooledRunnerMaxRssBytes, pooledRunnerMaxLifetimeMs, pooledRunnerSilenceTimeoutMs, runnerMaxOldSpaceSizeMb, runnerMallocArenaMax, dispatchStrategy, pollIntervalMs, drainStoreOperationTimeoutMs, scheduledEnqueueTimeoutMs, queues, jobClasses, jobTimeoutMs, retention, generationId, initialGenerationState, lifecycleSocketPath}
1986
+ return {host, port, databaseIdentifier, maxConcurrentForkedJobs, maxConcurrentInlineJobs, mode, pooledRunnerCount, pooledRunnerConcurrency, pooledRunnerMaxJobs, pooledRunnerMaxRssBytes, pooledRunnerMaxLifetimeMs, pooledRunnerSilenceTimeoutMs, runnerMaxOldSpaceSizeMb, runnerMallocArenaMax, dispatchStrategy, pollIntervalMs, drainStoreOperationTimeoutMs, scheduledEnqueueTimeoutMs, queues, queuePriorityAgingUnitMs, jobClasses, jobTimeoutMs, retention, generationId, initialGenerationState, lifecycleSocketPath}
1981
1987
  }
1982
1988
 
1983
1989
  /**
@@ -0,0 +1,25 @@
1
+ // @ts-check
2
+
3
+ /**
4
+ * Database names that may be interpolated into CREATE/DROP DATABASE DDL:
5
+ * plain lowercase identifiers only. Anything else (quotes, semicolons,
6
+ * wildcards, dots) is rejected before a statement is built, so every
7
+ * database-level helper operates on one explicitly named database and a
8
+ * name can never smuggle a pattern sweep or statement injection.
9
+ */
10
+ const SAFE_DATABASE_NAME = /^[a-z0-9_]+$/
11
+
12
+ /**
13
+ * Validates a database name against the strict allowlist before it is
14
+ * interpolated into DDL.
15
+ * @param {string | null | undefined} databaseName - Candidate database name.
16
+ * @param {string} operation - Operation name used in the rejection message.
17
+ * @returns {string} - The validated database name.
18
+ */
19
+ export function safeDatabaseName(databaseName, operation) {
20
+ if (!databaseName || !SAFE_DATABASE_NAME.test(databaseName)) {
21
+ throw new Error(`Unsafe database name for ${operation}: ${databaseName}`)
22
+ }
23
+
24
+ return databaseName
25
+ }