velocious 1.0.517 → 1.0.519

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. package/README.md +9 -1
  2. package/build/background-jobs/job.js +27 -2
  3. package/build/background-jobs/main.js +31 -0
  4. package/build/background-jobs/store.js +226 -5
  5. package/build/background-jobs/types.js +3 -1
  6. package/build/configuration-types.js +30 -1
  7. package/build/configuration.js +18 -2
  8. package/build/database/record/attachments/handle.js +18 -0
  9. package/build/database/record/attachments/store.js +45 -0
  10. package/build/http-server/client/index.js +1 -1
  11. package/build/http-server/client/request-buffer/index.js +151 -89
  12. package/build/src/background-jobs/job.d.ts +15 -0
  13. package/build/src/background-jobs/job.d.ts.map +1 -1
  14. package/build/src/background-jobs/job.js +24 -3
  15. package/build/src/background-jobs/main.d.ts +12 -0
  16. package/build/src/background-jobs/main.d.ts.map +1 -1
  17. package/build/src/background-jobs/main.js +32 -1
  18. package/build/src/background-jobs/store.d.ts +84 -1
  19. package/build/src/background-jobs/store.d.ts.map +1 -1
  20. package/build/src/background-jobs/store.js +198 -7
  21. package/build/src/background-jobs/types.d.ts +12 -2
  22. package/build/src/background-jobs/types.d.ts.map +1 -1
  23. package/build/src/background-jobs/types.js +4 -2
  24. package/build/src/configuration-types.d.ts +77 -2
  25. package/build/src/configuration-types.d.ts.map +1 -1
  26. package/build/src/configuration-types.js +30 -2
  27. package/build/src/configuration.d.ts.map +1 -1
  28. package/build/src/configuration.js +18 -2
  29. package/build/src/database/record/attachments/handle.d.ts +11 -0
  30. package/build/src/database/record/attachments/handle.d.ts.map +1 -1
  31. package/build/src/database/record/attachments/handle.js +17 -1
  32. package/build/src/database/record/attachments/store.d.ts +13 -0
  33. package/build/src/database/record/attachments/store.d.ts.map +1 -1
  34. package/build/src/database/record/attachments/store.js +40 -1
  35. package/build/src/http-server/client/index.js +2 -2
  36. package/build/src/http-server/client/request-buffer/index.d.ts +23 -2
  37. package/build/src/http-server/client/request-buffer/index.d.ts.map +1 -1
  38. package/build/src/http-server/client/request-buffer/index.js +141 -86
  39. package/package.json +2 -2
  40. package/src/background-jobs/job.js +27 -2
  41. package/src/background-jobs/main.js +31 -0
  42. package/src/background-jobs/store.js +226 -5
  43. package/src/background-jobs/types.js +3 -1
  44. package/src/configuration-types.js +30 -1
  45. package/src/configuration.js +18 -2
  46. package/src/database/record/attachments/handle.js +18 -0
  47. package/src/database/record/attachments/store.js +45 -0
  48. package/src/http-server/client/index.js +1 -1
  49. package/src/http-server/client/request-buffer/index.js +151 -89
package/README.md CHANGED
@@ -440,7 +440,7 @@ This creates `src/frontend-models/user.js` (and one file per configured resource
440
440
  - Attribute methods like `user.name()` and `user.setName(...)`
441
441
  - Relationship helpers (when `relationships` are configured), for example `task.project()`, `await task.projectOrLoad()`, `await project.tasks().toArray()`, `await project.tasks().load()`, and `project.tasks().build({...})`
442
442
  - Preload relationships onto records you already have with `await record.preload(Model.preload({...}).select({...}))` (or `Preloader.preload(records, ...)` for arrays), including `selectsExtra(...)` and a `{force: true}` reload option — see [docs/frontend-models.md](docs/frontend-models.md#preloading-onto-loaded-records)
443
- - Attachment helpers (when `attachments` are configured), for example `await task.descriptionFile().attach(file)`, `await task.descriptionFile().download()`, and `await task.update({descriptionFile: file})`
443
+ - Attachment helpers (when `attachments` are configured), for example `await task.descriptionFile().attach(file)`, `await task.descriptionFile().download()`, `await task.files().purgeAll()`, and `await task.update({descriptionFile: file})`
444
444
 
445
445
  React components can subscribe to lifecycle broadcasts without manual cleanup code:
446
446
 
@@ -501,6 +501,14 @@ await task.update({
501
501
  })
502
502
  ```
503
503
 
504
+ Purge a record's attachments — both the stored files and their rows — for example before destroying the owner record:
505
+
506
+ ```js
507
+ const purgedCount = await task.files().purgeAll()
508
+ ```
509
+
510
+ `purgeAll()` deletes each attachment's backing storage and then its row, and removes only the attachments that existed when the purge started (a concurrent `attach()` for the same record/name is left intact). It throws without deleting anything if a storage driver has no `delete` operation, so a driver configured without deletion can never silently leak storage. It is a no-op for unpersisted records and returns the number of attachments purged.
511
+
504
512
  Configure attachment storage drivers in `Configuration`:
505
513
 
506
514
  ```js
@@ -13,6 +13,15 @@ import BackgroundJobsClient from "./client.js"
13
13
  * @template {Array<?>} [TArgs=[]]
14
14
  */
15
15
  export default class VelociousJob {
16
+ /**
17
+ * Queue this job class runs on. Subclasses set e.g. `static queue = "builds"`
18
+ * to route onto a queue with its own cluster-wide concurrency cap (configured
19
+ * via `backgroundJobs.queues`). The `{queue}` enqueue option overrides it.
20
+ * Left undefined, jobs run on the `"default"` queue.
21
+ * @type {string | undefined}
22
+ */
23
+ static queue = undefined
24
+
16
25
  /**
17
26
  * Runs job name.
18
27
  * @returns {string} - Job name.
@@ -21,6 +30,22 @@ export default class VelociousJob {
21
30
  return this.name
22
31
  }
23
32
 
33
+ /**
34
+ * Folds this job class's static `queue` into the enqueue options unless the
35
+ * caller already specified one.
36
+ * @param {import("./types.js").BackgroundJobOptions | undefined} options - Job options.
37
+ * @returns {import("./types.js").BackgroundJobOptions} - Options including the resolved queue.
38
+ */
39
+ static _withQueue(options) {
40
+ const merged = options ? {...options} : {}
41
+
42
+ if (merged.queue === undefined && typeof this.queue === "string" && this.queue.length > 0) {
43
+ merged.queue = this.queue
44
+ }
45
+
46
+ return merged
47
+ }
48
+
24
49
  /**
25
50
  * Runs perform later.
26
51
  * @param {...?} args - Job args.
@@ -33,7 +58,7 @@ export default class VelociousJob {
33
58
  return await client.enqueue({
34
59
  jobName: this.jobName(),
35
60
  args: jobArgs,
36
- options: jobOptions
61
+ options: this._withQueue(jobOptions)
37
62
  })
38
63
  }
39
64
 
@@ -50,7 +75,7 @@ export default class VelociousJob {
50
75
  return await client.enqueue({
51
76
  jobName: this.jobName(),
52
77
  args,
53
- options
78
+ options: this._withQueue(options)
54
79
  })
55
80
  }
56
81
 
@@ -54,6 +54,7 @@ export default class BackgroundJobsMain {
54
54
  this.port = typeof port === "number" ? port : config.port
55
55
  this.dispatchStrategy = config.dispatchStrategy
56
56
  this.pollIntervalMs = config.pollIntervalMs
57
+ this.retention = config.retention
57
58
  this.store = new BackgroundJobsStore({configuration, databaseIdentifier: config.databaseIdentifier})
58
59
  this.logger = new Logger(this)
59
60
  /**
@@ -88,6 +89,10 @@ export default class BackgroundJobsMain {
88
89
  * Narrows the runtime value to the documented type.
89
90
  * @type {ReturnType<typeof setTimeout> | undefined} */
90
91
  this._orphanTimer = undefined
92
+ /**
93
+ * Narrows the runtime value to the documented type.
94
+ * @type {ReturnType<typeof setTimeout> | undefined} */
95
+ this._retentionTimer = undefined
91
96
  /**
92
97
  * Narrows the runtime value to the documented type.
93
98
  * @type {BackgroundJobsScheduler | undefined} */
@@ -139,6 +144,10 @@ export default class BackgroundJobsMain {
139
144
  void this._sweepOrphans()
140
145
  }, 60000)
141
146
 
147
+ this._retentionTimer = setInterval(() => {
148
+ void this._sweepRetention()
149
+ }, this.retention.sweepIntervalMs)
150
+
142
151
  this.scheduler = new BackgroundJobsScheduler({
143
152
  configuration: this.configuration,
144
153
  enqueueJob: async ({args, jobClass, options}) => {
@@ -197,10 +206,12 @@ export default class BackgroundJobsMain {
197
206
  if (this._scheduledTimer) clearTimeout(this._scheduledTimer)
198
207
  if (this._errorRetryTimer) clearTimeout(this._errorRetryTimer)
199
208
  if (this._orphanTimer) clearInterval(this._orphanTimer)
209
+ if (this._retentionTimer) clearInterval(this._retentionTimer)
200
210
  this._pollTimer = undefined
201
211
  this._scheduledTimer = undefined
202
212
  this._errorRetryTimer = undefined
203
213
  this._orphanTimer = undefined
214
+ this._retentionTimer = undefined
204
215
  }
205
216
 
206
217
  /**
@@ -1085,4 +1096,24 @@ export default class BackgroundJobsMain {
1085
1096
  this.logger.error(() => ["Failed to mark orphaned jobs:", error])
1086
1097
  }
1087
1098
  }
1099
+
1100
+ /**
1101
+ * Deletes terminal job rows past their retention window so the jobs table
1102
+ * does not grow unbounded. Runs on its own interval; failures are logged and
1103
+ * retried on the next tick.
1104
+ * @returns {Promise<void>} - Resolves after one retention sweep.
1105
+ */
1106
+ async _sweepRetention() {
1107
+ try {
1108
+ const deleted = await this.store.pruneTerminalJobs({
1109
+ completedTtlMs: this.retention.completedTtlMs,
1110
+ failedTtlMs: this.retention.failedTtlMs,
1111
+ batchSize: this.retention.batchSize
1112
+ })
1113
+
1114
+ if (deleted > 0) this.logger.warn(() => ["Pruned terminal background jobs", deleted])
1115
+ } catch (error) {
1116
+ this.logger.error(() => ["Failed to prune terminal jobs:", error])
1117
+ }
1118
+ }
1088
1119
  }
@@ -19,6 +19,10 @@ const ORPHANED_AFTER_MS = 2 * 60 * 60 * 1000
19
19
  * @type {import("./types.js").BackgroundJobExecutionMode[]} */
20
20
  const EXECUTION_MODES = ["inline", "forked", "spawned"]
21
21
  const DEFAULT_EXECUTION_MODE = "forked"
22
+ const DEFAULT_QUEUE = "default"
23
+ // Queue-derived durable concurrency keys are namespaced so they can't collide
24
+ // with explicit caller-supplied concurrencyKeys.
25
+ const QUEUE_CONCURRENCY_KEY_PREFIX = "queue:"
22
26
 
23
27
  /**
24
28
  * Columns the dashboard is allowed to sort job listings by, mapped to their
@@ -95,10 +99,17 @@ export default class BackgroundJobsStore {
95
99
  const executionMode = this._normalizeExecutionMode(options)
96
100
  const maxRetries = this._normalizeMaxRetries(options?.maxRetries)
97
101
  const argsJson = JSON.stringify(args || [])
98
- const concurrency = this._normalizeConcurrencyOptions(options)
102
+ const queue = this._normalizeQueue(options)
103
+ const concurrency = this._resolveConcurrency(options, queue)
99
104
 
100
105
  await this._withDb(async (db) => {
101
- if (concurrency) await this._ensureConcurrencyKey(db, concurrency)
106
+ if (concurrency) {
107
+ if (concurrency.queueDerived) {
108
+ await this._ensureQueueConcurrencyKey(db, concurrency)
109
+ } else {
110
+ await this._ensureConcurrencyKey(db, concurrency)
111
+ }
112
+ }
102
113
  await db.insert({
103
114
  tableName: JOBS_TABLE,
104
115
  data: {
@@ -107,6 +118,7 @@ export default class BackgroundJobsStore {
107
118
  args_json: argsJson,
108
119
  forked: executionMode !== "inline",
109
120
  execution_mode: executionMode,
121
+ queue,
110
122
  max_retries: maxRetries,
111
123
  attempts: 0,
112
124
  status: "queued",
@@ -464,11 +476,25 @@ export default class BackgroundJobsStore {
464
476
  for (const row of rows) {
465
477
  const job = this._normalizeJobRow(row)
466
478
 
479
+ // Fence the reclaim on the exact handoff this sweep selected, using its
480
+ // `handed_off_at_ms` rather than its `handoff_id`. Two reasons:
481
+ // 1. Null-safe. Some rows have a null `handoff_id` (handed off by an
482
+ // older velocious before handoff-id fencing). `{handoff_id: null}`
483
+ // renders as `handoff_id = NULL`, which matches nothing, so those
484
+ // rows would be stranded in `handed_off` forever.
485
+ // 2. Race-safe. If the row is returned to the queue and re-handed-off
486
+ // between the SELECT above and this update, it gets a fresh
487
+ // `handed_off_at_ms` (always "now"), so this stale cutoff-era
488
+ // timestamp no longer matches and we won't fail/orphan — or
489
+ // wrongly release the concurrency reservation of — that new lease.
490
+ // `handed_off_at_ms` is always set on a handed-off row (and the SELECT
491
+ // required it `<= cutoff`), so it is a reliable null-safe lease pin.
467
492
  const orphanedJob = await this._applyFailure({
468
493
  db,
469
494
  job,
470
495
  error: "Job orphaned after timeout",
471
- markOrphaned: true
496
+ markOrphaned: true,
497
+ conditions: {id: job.id, status: "handed_off", handed_off_at_ms: job.handedOffAtMs}
472
498
  })
473
499
 
474
500
  if (orphanedJob) orphanedCount += 1
@@ -478,6 +504,78 @@ export default class BackgroundJobsStore {
478
504
  })
479
505
  }
480
506
 
507
+ /**
508
+ * Deletes terminal job rows past their retention window so the jobs table
509
+ * does not grow unbounded (completed rows in particular accumulate forever
510
+ * otherwise). Batched by id — SELECT a page of ids, then
511
+ * `DELETE ... WHERE id IN (...)` — rather than `DELETE ... LIMIT`, which not
512
+ * every driver supports; each batch runs on its own connection so the sweep
513
+ * yields between batches instead of holding one long transaction.
514
+ * @param {object} [args] - Options.
515
+ * @param {number | null} [args.completedTtlMs] - Delete `completed` jobs whose `completed_at_ms` is older than this many ms. Falsy or `<= 0` disables completed pruning.
516
+ * @param {number | null} [args.failedTtlMs] - Delete terminal `failed`/`orphaned` jobs older than this many ms (by `failed_at_ms`/`orphaned_at_ms`). Falsy or `<= 0` disables.
517
+ * @param {number} [args.batchSize] - Max rows deleted per batch. Default `1000`.
518
+ * @returns {Promise<number>} - Total rows deleted.
519
+ */
520
+ async pruneTerminalJobs({completedTtlMs = null, failedTtlMs = null, batchSize = 1000} = {}) {
521
+ await this.ensureReady()
522
+
523
+ const now = Date.now()
524
+ const size = batchSize > 0 ? batchSize : 1000
525
+ let deleted = 0
526
+
527
+ if (completedTtlMs && completedTtlMs > 0) {
528
+ deleted += await this._pruneStatusBatches({status: "completed", column: "completed_at_ms", cutoff: now - completedTtlMs, batchSize: size})
529
+ }
530
+
531
+ if (failedTtlMs && failedTtlMs > 0) {
532
+ deleted += await this._pruneStatusBatches({status: "failed", column: "failed_at_ms", cutoff: now - failedTtlMs, batchSize: size})
533
+ deleted += await this._pruneStatusBatches({status: "orphaned", column: "orphaned_at_ms", cutoff: now - failedTtlMs, batchSize: size})
534
+ }
535
+
536
+ return deleted
537
+ }
538
+
539
+ /**
540
+ * Deletes rows of one terminal status older than a cutoff, batch by batch,
541
+ * until a page returns fewer than `batchSize` rows.
542
+ * @param {object} args - Options.
543
+ * @param {string} args.status - Terminal status to prune.
544
+ * @param {string} args.column - Timestamp column compared against the cutoff.
545
+ * @param {number} args.cutoff - Delete rows whose column value is `<= cutoff`.
546
+ * @param {number} args.batchSize - Max rows per batch.
547
+ * @returns {Promise<number>} - Rows deleted for this status.
548
+ */
549
+ async _pruneStatusBatches({status, column, cutoff, batchSize}) {
550
+ let deleted = 0
551
+
552
+ for (;;) {
553
+ const removed = await this._withDb(async (db) => {
554
+ const rows = await db
555
+ .newQuery()
556
+ .from(JOBS_TABLE)
557
+ .select("id")
558
+ .where({status})
559
+ .where(`${db.quoteColumn(column)} <= ${db.quote(cutoff)}`)
560
+ .limit(batchSize)
561
+ .results()
562
+
563
+ if (rows.length === 0) return 0
564
+
565
+ const ids = rows.map((/** @type {Record<string, ?>} */ row) => db.quote(String(row.id))).join(", ")
566
+
567
+ await db.query(`DELETE FROM ${db.quoteTable(JOBS_TABLE)} WHERE ${db.quoteColumn("id")} IN (${ids})`)
568
+
569
+ return rows.length
570
+ })
571
+
572
+ deleted += removed
573
+ if (removed < batchSize) break
574
+ }
575
+
576
+ return deleted
577
+ }
578
+
481
579
  /**
482
580
  * Runs clear all.
483
581
  * @returns {Promise<void>} - Resolves when cleared.
@@ -621,6 +719,7 @@ export default class BackgroundJobsStore {
621
719
  table.text("args_json", {null: false})
622
720
  table.boolean("forked", {null: false})
623
721
  table.string("execution_mode", {null: false})
722
+ table.string("queue", {null: true, index: true})
624
723
  table.integer("max_retries", {null: false})
625
724
  table.integer("attempts", {null: false})
626
725
  table.string("status", {null: false, index: true})
@@ -723,6 +822,43 @@ export default class BackgroundJobsStore {
723
822
  } finally {
724
823
  await db.releaseAdvisoryLock(lockName)
725
824
  }
825
+
826
+ await this._ensureQueueColumn(db)
827
+ }
828
+
829
+ /**
830
+ * Idempotently adds the `queue` column to an existing jobs table. Existing
831
+ * rows read back as the default queue (see {@link _normalizeJobRow}), so no
832
+ * data backfill is required.
833
+ * @param {import("../database/drivers/base.js").default} db - Database connection.
834
+ * @returns {Promise<void>} - Resolves when ensured.
835
+ */
836
+ async _ensureQueueColumn(db) {
837
+ const table = await db.getTableByNameOrFail(JOBS_TABLE)
838
+
839
+ if (await table.getColumnByName("queue")) return
840
+
841
+ const lockName = `${MIGRATION_SCOPE}:queue_column`
842
+ const acquired = await db.acquireAdvisoryLock(lockName)
843
+
844
+ if (!acquired) throw new Error("Failed to acquire background jobs queue schema lock")
845
+
846
+ try {
847
+ db.clearSchemaCache()
848
+ const lockedTable = await db.getTableByNameOrFail(JOBS_TABLE)
849
+
850
+ if (!(await lockedTable.getColumnByName("queue"))) {
851
+ const tableData = new TableData(JOBS_TABLE)
852
+
853
+ tableData.string("queue", {null: true, index: true})
854
+
855
+ for (const sql of await db.alterTableSQLs(tableData)) await db.query(sql)
856
+
857
+ db.clearSchemaCache()
858
+ }
859
+ } finally {
860
+ await db.releaseAdvisoryLock(lockName)
861
+ }
726
862
  }
727
863
 
728
864
  /**
@@ -819,9 +955,10 @@ export default class BackgroundJobsStore {
819
955
  * @param {import("./types.js").BackgroundJobRow} args.job - Job row.
820
956
  * @param {?} args.error - Error.
821
957
  * @param {boolean} args.markOrphaned - Whether marking orphaned.
958
+ * @param {Record<string, ?>} [args.conditions] - Update fencing conditions. Defaults to the active-handoff lease match; the time-based orphan sweep overrides this with an id/status match so it can reclaim rows whose `handoff_id` is null (e.g. handed off by an older velocious before handoff-id fencing existed).
822
959
  * @returns {Promise<import("./types.js").BackgroundJobRow | null>} - Updated job row when the lease transition won.
823
960
  */
824
- async _applyFailure({db, job, error, markOrphaned}) {
961
+ async _applyFailure({db, job, error, markOrphaned, conditions}) {
825
962
  const now = Date.now()
826
963
  const nextAttempt = (job.attempts || 0) + 1
827
964
  const maxRetries = this._normalizeMaxRetries(job.maxRetries)
@@ -840,7 +977,7 @@ export default class BackgroundJobsStore {
840
977
  const affectedRows = await this._updateAffectedRows(db, {
841
978
  tableName: JOBS_TABLE,
842
979
  data: update,
843
- conditions: this._activeHandoffConditions(job)
980
+ conditions: conditions ?? this._activeHandoffConditions(job)
844
981
  })
845
982
 
846
983
  if (affectedRows !== 1) return null
@@ -934,6 +1071,7 @@ export default class BackgroundJobsStore {
934
1071
  args: this._parseArgs(row.args_json),
935
1072
  executionMode,
936
1073
  forked: executionMode !== "inline",
1074
+ queue: row.queue ? String(row.queue) : DEFAULT_QUEUE,
937
1075
  status: row.status ? String(row.status) : "queued",
938
1076
  attempts: this._normalizeNumber(row.attempts),
939
1077
  maxRetries: this._normalizeNumber(row.max_retries),
@@ -966,6 +1104,89 @@ export default class BackgroundJobsStore {
966
1104
  return {concurrencyKey: key, maxConcurrency: Number(cap)}
967
1105
  }
968
1106
 
1107
+ /**
1108
+ * Normalizes a job's queue name, defaulting to "default".
1109
+ * @param {import("./types.js").BackgroundJobOptions | undefined} options - Job options.
1110
+ * @returns {string} - Queue name.
1111
+ */
1112
+ _normalizeQueue(options) {
1113
+ const queue = options?.queue
1114
+
1115
+ if (typeof queue === "string" && queue.trim().length > 0) return queue.trim()
1116
+
1117
+ return DEFAULT_QUEUE
1118
+ }
1119
+
1120
+ /**
1121
+ * Resolves a job's durable concurrency. An explicit concurrencyKey/maxConcurrency
1122
+ * pair always wins. Otherwise, when the job's queue has a configured cap
1123
+ * (`backgroundJobs.queues[queue].maxConcurrent`), derive a queue-scoped
1124
+ * concurrency key so the queue cap is enforced cluster-wide through the
1125
+ * existing durable concurrency mechanism.
1126
+ * @param {import("./types.js").BackgroundJobOptions | undefined} options - Job options.
1127
+ * @param {string} queue - Normalized queue name.
1128
+ * @returns {{concurrencyKey: string, maxConcurrency: number, queueDerived: boolean} | null} - Resolved concurrency.
1129
+ */
1130
+ _resolveConcurrency(options, queue) {
1131
+ const explicit = this._normalizeConcurrencyOptions(options)
1132
+
1133
+ if (explicit) return {...explicit, queueDerived: false}
1134
+
1135
+ const cap = this._queueMaxConcurrency(queue)
1136
+
1137
+ if (cap === null) return null
1138
+
1139
+ return {concurrencyKey: `${QUEUE_CONCURRENCY_KEY_PREFIX}${queue}`, maxConcurrency: cap, queueDerived: true}
1140
+ }
1141
+
1142
+ /**
1143
+ * Reads the configured max concurrency for a queue from the background-jobs config.
1144
+ * @param {string} queue - Queue name.
1145
+ * @returns {number | null} - Positive integer cap, or null when the queue has no configured cap.
1146
+ */
1147
+ _queueMaxConcurrency(queue) {
1148
+ const queues = this.configuration.getBackgroundJobsConfig().queues
1149
+ const cap = queues?.[queue]?.maxConcurrent
1150
+
1151
+ if (Number.isInteger(cap) && Number(cap) > 0) return Number(cap)
1152
+
1153
+ return null
1154
+ }
1155
+
1156
+ /**
1157
+ * Like {@link _ensureConcurrencyKey}, but for queue-derived keys the configured
1158
+ * queue cap is the source of truth: if it changed, update the stored cap
1159
+ * instead of throwing on conflict (config-driven caps must be tunable).
1160
+ * @param {import("../database/drivers/base.js").default} db - Database connection.
1161
+ * @param {{concurrencyKey: string, maxConcurrency: number}} concurrency - Concurrency configuration.
1162
+ * @returns {Promise<void>} - Resolves when ensured.
1163
+ */
1164
+ async _ensureQueueConcurrencyKey(db, {concurrencyKey, maxConcurrency}) {
1165
+ const rows = await db.newQuery().from(CONCURRENCY_TABLE).where({concurrency_key: concurrencyKey}).limit(1).results()
1166
+
1167
+ if (!rows[0]) {
1168
+ try {
1169
+ await db.insert({tableName: CONCURRENCY_TABLE, data: {active_count: 0, concurrency_key: concurrencyKey, max_concurrency: maxConcurrency}})
1170
+
1171
+ return
1172
+ } catch (error) {
1173
+ const racedRows = await db.newQuery().from(CONCURRENCY_TABLE).where({concurrency_key: concurrencyKey}).limit(1).results()
1174
+
1175
+ if (!racedRows[0]) throw error
1176
+
1177
+ rows[0] = racedRows[0]
1178
+ }
1179
+ }
1180
+
1181
+ const configured = /** @type {{max_concurrency?: number | string}} */ (rows[0])
1182
+
1183
+ if (this._normalizeNumber(configured.max_concurrency) !== maxConcurrency) {
1184
+ const table = db.quoteTable(CONCURRENCY_TABLE)
1185
+
1186
+ await db.query(`UPDATE ${table} SET ${db.quoteColumn("max_concurrency")} = ${Number(maxConcurrency)} WHERE ${db.quoteColumn("concurrency_key")} = ${db.quote(concurrencyKey)}`)
1187
+ }
1188
+ }
1189
+
969
1190
  /**
970
1191
  * Ensures the concurrency state table exists.
971
1192
  * @param {import("../database/drivers/base.js").default} db - Database connection.
@@ -13,7 +13,8 @@
13
13
  * @property {BackgroundJobExecutionMode} [executionMode] - How the job should run. Defaults to `"forked"`.
14
14
  * @property {boolean} [forked] - Compatibility alias: `false` maps to `"inline"` and `true` maps to `"forked"`.
15
15
  * @property {number} [maxRetries] - Max retries for a failed job before it is marked failed.
16
- * @property {string} [concurrencyKey] - Opaque non-empty key used to share a concurrency cap.
16
+ * @property {string} [queue] - Queue name. Defaults to `"default"`. When the queue has a configured cap in `backgroundJobs.queues`, that cap is enforced cluster-wide.
17
+ * @property {string} [concurrencyKey] - Opaque non-empty key used to share a concurrency cap. Overrides any queue-derived cap.
17
18
  * @property {number} [maxConcurrency] - Positive integer cap; must be paired with `concurrencyKey`.
18
19
  */
19
20
  /**
@@ -33,6 +34,7 @@
33
34
  * @property {Array<?>} args - Serialized job arguments.
34
35
  * @property {BackgroundJobExecutionMode} executionMode - How the job should run.
35
36
  * @property {boolean} forked - Compatibility flag; true for non-inline execution.
37
+ * @property {string} queue - Queue name (defaults to `"default"`).
36
38
  * @property {string} status - Current job status.
37
39
  * @property {number | null} attempts - Failure attempts count.
38
40
  * @property {number | null} maxRetries - Max retry attempts.
@@ -161,12 +161,41 @@
161
161
  * long-running jobs and for using more cores. Default: `4`.
162
162
  * @property {number} [maxConcurrentForkedJobs] - How many out-of-process
163
163
  * `"forked"` or `"spawned"` jobs a single `background-jobs-worker` is
164
- * allowed to keep in flight. Default: `4`.
164
+ * allowed to keep in flight. Default: `4`. This is a per-worker safety cap;
165
+ * for workload-shaped limits use per-queue caps (`queues`) instead, which are
166
+ * enforced cluster-wide and are immune to duplicate worker processes.
167
+ * @property {Record<string, {maxConcurrent?: number}>} [queues] - Per-queue
168
+ * concurrency caps, Sidekiq-style. A job declares its queue (static `queue` on
169
+ * the job class, or the `queue` enqueue option; defaults to `"default"`), and
170
+ * `queues[name].maxConcurrent` bounds how many jobs from that queue may be
171
+ * in flight across the whole cluster (enforced via durable per-key
172
+ * concurrency, so it holds regardless of how many worker processes run).
173
+ * Size each queue to its workload: I/O-bound queues (e.g. build runners
174
+ * waiting on remote Docker servers) can run far above the core count, while
175
+ * CPU-bound queues should stay near it. Default: `{}` (no queue caps).
165
176
  * @property {BackgroundJobsDispatchStrategy} [dispatchStrategy] - How the main process
166
177
  * detects new work. Defaults to `"beacon"` (event-driven). Set to `"polling"`
167
178
  * to restore the legacy fixed-interval poll.
168
179
  * @property {number} [pollIntervalMs] - Poll interval in milliseconds. Only used
169
180
  * when `dispatchStrategy === "polling"`. Default: `1000`.
181
+ * @property {BackgroundJobsRetentionConfiguration} [retention] - Retention/pruning
182
+ * of terminal job rows. Without pruning the jobs table grows unbounded
183
+ * (completed rows accumulate forever), which bloats storage and indexes and
184
+ * eventually slows dispatch. The main process sweeps terminal rows past their
185
+ * window on an interval.
186
+ */
187
+
188
+ /**
189
+ * @typedef {object} BackgroundJobsRetentionConfiguration
190
+ * @property {number | null} [completedTtlMs] - Delete `completed` jobs whose
191
+ * `completed_at_ms` is older than this many ms. `null` or `<= 0` disables
192
+ * completed pruning. Default: `604800000` (7 days).
193
+ * @property {number | null} [failedTtlMs] - Delete terminal `failed`/`orphaned`
194
+ * jobs older than this many ms. `null` or `<= 0` disables (keeps them for
195
+ * debugging). Default: `2592000000` (30 days).
196
+ * @property {number} [batchSize] - Rows deleted per batch. Default: `1000`.
197
+ * @property {number} [sweepIntervalMs] - How often the main process runs the
198
+ * retention sweep. Default: `3600000` (1 hour).
170
199
  */
171
200
 
172
201
  /**
@@ -1291,8 +1291,24 @@ export default class VelociousConfiguration {
1291
1291
  const pollIntervalMs = typeof configured.pollIntervalMs === "number" && configured.pollIntervalMs >= 1
1292
1292
  ? configured.pollIntervalMs
1293
1293
  : (typeof envPollInterval === "number" && Number.isFinite(envPollInterval) && envPollInterval >= 1 ? envPollInterval : 1000)
1294
-
1295
- return {host, port, databaseIdentifier, maxConcurrentForkedJobs, maxConcurrentInlineJobs, dispatchStrategy, pollIntervalMs}
1294
+ const queues = configured.queues && typeof configured.queues === "object" ? configured.queues : {}
1295
+ const configuredRetention = configured.retention && typeof configured.retention === "object" ? configured.retention : {}
1296
+ const retention = {
1297
+ completedTtlMs: typeof configuredRetention.completedTtlMs === "number" || configuredRetention.completedTtlMs === null
1298
+ ? configuredRetention.completedTtlMs
1299
+ : 7 * 24 * 60 * 60 * 1000,
1300
+ failedTtlMs: typeof configuredRetention.failedTtlMs === "number" || configuredRetention.failedTtlMs === null
1301
+ ? configuredRetention.failedTtlMs
1302
+ : 30 * 24 * 60 * 60 * 1000,
1303
+ batchSize: typeof configuredRetention.batchSize === "number" && configuredRetention.batchSize > 0
1304
+ ? configuredRetention.batchSize
1305
+ : 1000,
1306
+ sweepIntervalMs: typeof configuredRetention.sweepIntervalMs === "number" && configuredRetention.sweepIntervalMs > 0
1307
+ ? configuredRetention.sweepIntervalMs
1308
+ : 60 * 60 * 1000
1309
+ }
1310
+
1311
+ return {host, port, databaseIdentifier, maxConcurrentForkedJobs, maxConcurrentInlineJobs, dispatchStrategy, pollIntervalMs, queues, retention}
1296
1312
  }
1297
1313
 
1298
1314
  /**
@@ -217,4 +217,22 @@ export default class RecordAttachmentHandle {
217
217
 
218
218
  return await store.attachmentRowUrl({model: this.model, name: this.name, row})
219
219
  }
220
+
221
+ /**
222
+ * Purges every attachment under this (record, name): deletes the backing
223
+ * storage for each and removes the attachment rows. A no-op for unpersisted
224
+ * records. Only the attachments present when the purge starts are removed, so a
225
+ * concurrent attach for the same (record, name) is left intact. Throws (without
226
+ * deleting any rows) if a storage driver cannot delete its object, so a driver
227
+ * configured without a `delete` operation can never leak storage. Callers use
228
+ * this to clean up attachments before destroying the owner record.
229
+ * @returns {Promise<number>} - Number of attachments purged.
230
+ */
231
+ async purgeAll() {
232
+ if (!this.model.isPersisted()) return 0
233
+
234
+ const store = recordAttachmentsStoreForModel(this.model)
235
+
236
+ return await store.purgeAll({model: this.model, name: this.name})
237
+ }
220
238
  }
@@ -391,6 +391,51 @@ export default class RecordAttachmentsStore {
391
391
  })
392
392
  }
393
393
 
394
+ /**
395
+ * Purges every attachment stored under (model, name): deletes each row's
396
+ * backing storage and then removes the attachment rows. Used to clean up an
397
+ * owner record's attachments before/when the owner is destroyed.
398
+ * @param {object} args - Options.
399
+ * @param {import("../index.js").default} args.model - Model instance.
400
+ * @param {string} args.name - Attachment name.
401
+ * @returns {Promise<number>} - Number of attachments purged.
402
+ */
403
+ async purgeAll({model, name}) {
404
+ await this.ensureReady()
405
+
406
+ return await this._withDb(async (db) => {
407
+ const recordType = model.getModelClass().getModelName()
408
+ const recordId = String(model.id())
409
+ /** @type {Array<Record<string, ?>>} */
410
+ const rows = await db
411
+ .newQuery()
412
+ .from(ATTACHMENTS_TABLE)
413
+ .where({name, record_id: recordId, record_type: recordType})
414
+ .results()
415
+
416
+ // Refuse to purge when any row's driver cannot delete its backing storage:
417
+ // removing the row while the object stays behind would leak storage and
418
+ // discard the metadata needed to retry cleanup. Fail loudly instead.
419
+ for (const row of rows) {
420
+ const attachmentDriver = await this.resolveAttachmentDriver({model, name, row})
421
+
422
+ if (typeof attachmentDriver.delete !== "function") {
423
+ throw new Error(`Cannot purge attachment ${row.id} for ${recordType}#${recordId} (${name}): its storage driver does not support deletion.`)
424
+ }
425
+ }
426
+
427
+ for (const row of rows) {
428
+ await this.deleteAttachmentRowStorage({model, name, row})
429
+ // Delete only the snapshotted row by id, so an attachment inserted for the
430
+ // same (record, name) after the snapshot is not removed with its storage
431
+ // still present (which would leave it as unreachable storage).
432
+ await db.delete({conditions: {id: row.id}, tableName: ATTACHMENTS_TABLE})
433
+ }
434
+
435
+ return rows.length
436
+ })
437
+ }
438
+
394
439
  /**
395
440
  * Runs attachment driver by name.
396
441
  * @param {string} driverName - Driver name.
@@ -400,7 +400,7 @@ export default class VeoliciousHttpServerClient {
400
400
  const stats = await fs.stat(filePath)
401
401
  contentLength = stats.size
402
402
  } else {
403
- contentLength = bodyIsString ? new TextEncoder().encode(body).length : body.byteLength
403
+ contentLength = bodyIsString ? Buffer.byteLength(body, "utf8") : body.byteLength
404
404
  }
405
405
 
406
406
  response.setHeader("Content-Length", contentLength)