velocious 1.0.517 → 1.0.519
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -1
- package/build/background-jobs/job.js +27 -2
- package/build/background-jobs/main.js +31 -0
- package/build/background-jobs/store.js +226 -5
- package/build/background-jobs/types.js +3 -1
- package/build/configuration-types.js +30 -1
- package/build/configuration.js +18 -2
- package/build/database/record/attachments/handle.js +18 -0
- package/build/database/record/attachments/store.js +45 -0
- package/build/http-server/client/index.js +1 -1
- package/build/http-server/client/request-buffer/index.js +151 -89
- package/build/src/background-jobs/job.d.ts +15 -0
- package/build/src/background-jobs/job.d.ts.map +1 -1
- package/build/src/background-jobs/job.js +24 -3
- package/build/src/background-jobs/main.d.ts +12 -0
- package/build/src/background-jobs/main.d.ts.map +1 -1
- package/build/src/background-jobs/main.js +32 -1
- package/build/src/background-jobs/store.d.ts +84 -1
- package/build/src/background-jobs/store.d.ts.map +1 -1
- package/build/src/background-jobs/store.js +198 -7
- package/build/src/background-jobs/types.d.ts +12 -2
- package/build/src/background-jobs/types.d.ts.map +1 -1
- package/build/src/background-jobs/types.js +4 -2
- package/build/src/configuration-types.d.ts +77 -2
- package/build/src/configuration-types.d.ts.map +1 -1
- package/build/src/configuration-types.js +30 -2
- package/build/src/configuration.d.ts.map +1 -1
- package/build/src/configuration.js +18 -2
- package/build/src/database/record/attachments/handle.d.ts +11 -0
- package/build/src/database/record/attachments/handle.d.ts.map +1 -1
- package/build/src/database/record/attachments/handle.js +17 -1
- package/build/src/database/record/attachments/store.d.ts +13 -0
- package/build/src/database/record/attachments/store.d.ts.map +1 -1
- package/build/src/database/record/attachments/store.js +40 -1
- package/build/src/http-server/client/index.js +2 -2
- package/build/src/http-server/client/request-buffer/index.d.ts +23 -2
- package/build/src/http-server/client/request-buffer/index.d.ts.map +1 -1
- package/build/src/http-server/client/request-buffer/index.js +141 -86
- package/package.json +2 -2
- package/src/background-jobs/job.js +27 -2
- package/src/background-jobs/main.js +31 -0
- package/src/background-jobs/store.js +226 -5
- package/src/background-jobs/types.js +3 -1
- package/src/configuration-types.js +30 -1
- package/src/configuration.js +18 -2
- package/src/database/record/attachments/handle.js +18 -0
- package/src/database/record/attachments/store.js +45 -0
- package/src/http-server/client/index.js +1 -1
- package/src/http-server/client/request-buffer/index.js +151 -89
package/README.md
CHANGED
|
@@ -440,7 +440,7 @@ This creates `src/frontend-models/user.js` (and one file per configured resource
|
|
|
440
440
|
- Attribute methods like `user.name()` and `user.setName(...)`
|
|
441
441
|
- Relationship helpers (when `relationships` are configured), for example `task.project()`, `await task.projectOrLoad()`, `await project.tasks().toArray()`, `await project.tasks().load()`, and `project.tasks().build({...})`
|
|
442
442
|
- Preload relationships onto records you already have with `await record.preload(Model.preload({...}).select({...}))` (or `Preloader.preload(records, ...)` for arrays), including `selectsExtra(...)` and a `{force: true}` reload option — see [docs/frontend-models.md](docs/frontend-models.md#preloading-onto-loaded-records)
|
|
443
|
-
- Attachment helpers (when `attachments` are configured), for example `await task.descriptionFile().attach(file)`, `await task.descriptionFile().download()`, and `await task.update({descriptionFile: file})`
|
|
443
|
+
- Attachment helpers (when `attachments` are configured), for example `await task.descriptionFile().attach(file)`, `await task.descriptionFile().download()`, `await task.files().purgeAll()`, and `await task.update({descriptionFile: file})`
|
|
444
444
|
|
|
445
445
|
React components can subscribe to lifecycle broadcasts without manual cleanup code:
|
|
446
446
|
|
|
@@ -501,6 +501,14 @@ await task.update({
|
|
|
501
501
|
})
|
|
502
502
|
```
|
|
503
503
|
|
|
504
|
+
Purge a record's attachments — both the stored files and their rows — for example before destroying the owner record:
|
|
505
|
+
|
|
506
|
+
```js
|
|
507
|
+
const purgedCount = await task.files().purgeAll()
|
|
508
|
+
```
|
|
509
|
+
|
|
510
|
+
`purgeAll()` deletes each attachment's backing storage and then its row, and removes only the attachments that existed when the purge started (a concurrent `attach()` for the same record/name is left intact). It throws without deleting anything if a storage driver has no `delete` operation, so a driver configured without deletion can never silently leak storage. It is a no-op for unpersisted records and returns the number of attachments purged.
|
|
511
|
+
|
|
504
512
|
Configure attachment storage drivers in `Configuration`:
|
|
505
513
|
|
|
506
514
|
```js
|
|
@@ -13,6 +13,15 @@ import BackgroundJobsClient from "./client.js"
|
|
|
13
13
|
* @template {Array<?>} [TArgs=[]]
|
|
14
14
|
*/
|
|
15
15
|
export default class VelociousJob {
|
|
16
|
+
/**
|
|
17
|
+
* Queue this job class runs on. Subclasses set e.g. `static queue = "builds"`
|
|
18
|
+
* to route onto a queue with its own cluster-wide concurrency cap (configured
|
|
19
|
+
* via `backgroundJobs.queues`). The `{queue}` enqueue option overrides it.
|
|
20
|
+
* Left undefined, jobs run on the `"default"` queue.
|
|
21
|
+
* @type {string | undefined}
|
|
22
|
+
*/
|
|
23
|
+
static queue = undefined
|
|
24
|
+
|
|
16
25
|
/**
|
|
17
26
|
* Runs job name.
|
|
18
27
|
* @returns {string} - Job name.
|
|
@@ -21,6 +30,22 @@ export default class VelociousJob {
|
|
|
21
30
|
return this.name
|
|
22
31
|
}
|
|
23
32
|
|
|
33
|
+
/**
|
|
34
|
+
* Folds this job class's static `queue` into the enqueue options unless the
|
|
35
|
+
* caller already specified one.
|
|
36
|
+
* @param {import("./types.js").BackgroundJobOptions | undefined} options - Job options.
|
|
37
|
+
* @returns {import("./types.js").BackgroundJobOptions} - Options including the resolved queue.
|
|
38
|
+
*/
|
|
39
|
+
static _withQueue(options) {
|
|
40
|
+
const merged = options ? {...options} : {}
|
|
41
|
+
|
|
42
|
+
if (merged.queue === undefined && typeof this.queue === "string" && this.queue.length > 0) {
|
|
43
|
+
merged.queue = this.queue
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
return merged
|
|
47
|
+
}
|
|
48
|
+
|
|
24
49
|
/**
|
|
25
50
|
* Runs perform later.
|
|
26
51
|
* @param {...?} args - Job args.
|
|
@@ -33,7 +58,7 @@ export default class VelociousJob {
|
|
|
33
58
|
return await client.enqueue({
|
|
34
59
|
jobName: this.jobName(),
|
|
35
60
|
args: jobArgs,
|
|
36
|
-
options: jobOptions
|
|
61
|
+
options: this._withQueue(jobOptions)
|
|
37
62
|
})
|
|
38
63
|
}
|
|
39
64
|
|
|
@@ -50,7 +75,7 @@ export default class VelociousJob {
|
|
|
50
75
|
return await client.enqueue({
|
|
51
76
|
jobName: this.jobName(),
|
|
52
77
|
args,
|
|
53
|
-
options
|
|
78
|
+
options: this._withQueue(options)
|
|
54
79
|
})
|
|
55
80
|
}
|
|
56
81
|
|
|
@@ -54,6 +54,7 @@ export default class BackgroundJobsMain {
|
|
|
54
54
|
this.port = typeof port === "number" ? port : config.port
|
|
55
55
|
this.dispatchStrategy = config.dispatchStrategy
|
|
56
56
|
this.pollIntervalMs = config.pollIntervalMs
|
|
57
|
+
this.retention = config.retention
|
|
57
58
|
this.store = new BackgroundJobsStore({configuration, databaseIdentifier: config.databaseIdentifier})
|
|
58
59
|
this.logger = new Logger(this)
|
|
59
60
|
/**
|
|
@@ -88,6 +89,10 @@ export default class BackgroundJobsMain {
|
|
|
88
89
|
* Narrows the runtime value to the documented type.
|
|
89
90
|
* @type {ReturnType<typeof setTimeout> | undefined} */
|
|
90
91
|
this._orphanTimer = undefined
|
|
92
|
+
/**
|
|
93
|
+
* Narrows the runtime value to the documented type.
|
|
94
|
+
* @type {ReturnType<typeof setTimeout> | undefined} */
|
|
95
|
+
this._retentionTimer = undefined
|
|
91
96
|
/**
|
|
92
97
|
* Narrows the runtime value to the documented type.
|
|
93
98
|
* @type {BackgroundJobsScheduler | undefined} */
|
|
@@ -139,6 +144,10 @@ export default class BackgroundJobsMain {
|
|
|
139
144
|
void this._sweepOrphans()
|
|
140
145
|
}, 60000)
|
|
141
146
|
|
|
147
|
+
this._retentionTimer = setInterval(() => {
|
|
148
|
+
void this._sweepRetention()
|
|
149
|
+
}, this.retention.sweepIntervalMs)
|
|
150
|
+
|
|
142
151
|
this.scheduler = new BackgroundJobsScheduler({
|
|
143
152
|
configuration: this.configuration,
|
|
144
153
|
enqueueJob: async ({args, jobClass, options}) => {
|
|
@@ -197,10 +206,12 @@ export default class BackgroundJobsMain {
|
|
|
197
206
|
if (this._scheduledTimer) clearTimeout(this._scheduledTimer)
|
|
198
207
|
if (this._errorRetryTimer) clearTimeout(this._errorRetryTimer)
|
|
199
208
|
if (this._orphanTimer) clearInterval(this._orphanTimer)
|
|
209
|
+
if (this._retentionTimer) clearInterval(this._retentionTimer)
|
|
200
210
|
this._pollTimer = undefined
|
|
201
211
|
this._scheduledTimer = undefined
|
|
202
212
|
this._errorRetryTimer = undefined
|
|
203
213
|
this._orphanTimer = undefined
|
|
214
|
+
this._retentionTimer = undefined
|
|
204
215
|
}
|
|
205
216
|
|
|
206
217
|
/**
|
|
@@ -1085,4 +1096,24 @@ export default class BackgroundJobsMain {
|
|
|
1085
1096
|
this.logger.error(() => ["Failed to mark orphaned jobs:", error])
|
|
1086
1097
|
}
|
|
1087
1098
|
}
|
|
1099
|
+
|
|
1100
|
+
/**
|
|
1101
|
+
* Deletes terminal job rows past their retention window so the jobs table
|
|
1102
|
+
* does not grow unbounded. Runs on its own interval; failures are logged and
|
|
1103
|
+
* retried on the next tick.
|
|
1104
|
+
* @returns {Promise<void>} - Resolves after one retention sweep.
|
|
1105
|
+
*/
|
|
1106
|
+
async _sweepRetention() {
|
|
1107
|
+
try {
|
|
1108
|
+
const deleted = await this.store.pruneTerminalJobs({
|
|
1109
|
+
completedTtlMs: this.retention.completedTtlMs,
|
|
1110
|
+
failedTtlMs: this.retention.failedTtlMs,
|
|
1111
|
+
batchSize: this.retention.batchSize
|
|
1112
|
+
})
|
|
1113
|
+
|
|
1114
|
+
if (deleted > 0) this.logger.warn(() => ["Pruned terminal background jobs", deleted])
|
|
1115
|
+
} catch (error) {
|
|
1116
|
+
this.logger.error(() => ["Failed to prune terminal jobs:", error])
|
|
1117
|
+
}
|
|
1118
|
+
}
|
|
1088
1119
|
}
|
|
@@ -19,6 +19,10 @@ const ORPHANED_AFTER_MS = 2 * 60 * 60 * 1000
|
|
|
19
19
|
* @type {import("./types.js").BackgroundJobExecutionMode[]} */
|
|
20
20
|
const EXECUTION_MODES = ["inline", "forked", "spawned"]
|
|
21
21
|
const DEFAULT_EXECUTION_MODE = "forked"
|
|
22
|
+
const DEFAULT_QUEUE = "default"
|
|
23
|
+
// Queue-derived durable concurrency keys are namespaced so they can't collide
|
|
24
|
+
// with explicit caller-supplied concurrencyKeys.
|
|
25
|
+
const QUEUE_CONCURRENCY_KEY_PREFIX = "queue:"
|
|
22
26
|
|
|
23
27
|
/**
|
|
24
28
|
* Columns the dashboard is allowed to sort job listings by, mapped to their
|
|
@@ -95,10 +99,17 @@ export default class BackgroundJobsStore {
|
|
|
95
99
|
const executionMode = this._normalizeExecutionMode(options)
|
|
96
100
|
const maxRetries = this._normalizeMaxRetries(options?.maxRetries)
|
|
97
101
|
const argsJson = JSON.stringify(args || [])
|
|
98
|
-
const
|
|
102
|
+
const queue = this._normalizeQueue(options)
|
|
103
|
+
const concurrency = this._resolveConcurrency(options, queue)
|
|
99
104
|
|
|
100
105
|
await this._withDb(async (db) => {
|
|
101
|
-
if (concurrency)
|
|
106
|
+
if (concurrency) {
|
|
107
|
+
if (concurrency.queueDerived) {
|
|
108
|
+
await this._ensureQueueConcurrencyKey(db, concurrency)
|
|
109
|
+
} else {
|
|
110
|
+
await this._ensureConcurrencyKey(db, concurrency)
|
|
111
|
+
}
|
|
112
|
+
}
|
|
102
113
|
await db.insert({
|
|
103
114
|
tableName: JOBS_TABLE,
|
|
104
115
|
data: {
|
|
@@ -107,6 +118,7 @@ export default class BackgroundJobsStore {
|
|
|
107
118
|
args_json: argsJson,
|
|
108
119
|
forked: executionMode !== "inline",
|
|
109
120
|
execution_mode: executionMode,
|
|
121
|
+
queue,
|
|
110
122
|
max_retries: maxRetries,
|
|
111
123
|
attempts: 0,
|
|
112
124
|
status: "queued",
|
|
@@ -464,11 +476,25 @@ export default class BackgroundJobsStore {
|
|
|
464
476
|
for (const row of rows) {
|
|
465
477
|
const job = this._normalizeJobRow(row)
|
|
466
478
|
|
|
479
|
+
// Fence the reclaim on the exact handoff this sweep selected, using its
|
|
480
|
+
// `handed_off_at_ms` rather than its `handoff_id`. Two reasons:
|
|
481
|
+
// 1. Null-safe. Some rows have a null `handoff_id` (handed off by an
|
|
482
|
+
// older velocious before handoff-id fencing). `{handoff_id: null}`
|
|
483
|
+
// renders as `handoff_id = NULL`, which matches nothing, so those
|
|
484
|
+
// rows would be stranded in `handed_off` forever.
|
|
485
|
+
// 2. Race-safe. If the row is returned to the queue and re-handed-off
|
|
486
|
+
// between the SELECT above and this update, it gets a fresh
|
|
487
|
+
// `handed_off_at_ms` (always "now"), so this stale cutoff-era
|
|
488
|
+
// timestamp no longer matches and we won't fail/orphan — or
|
|
489
|
+
// wrongly release the concurrency reservation of — that new lease.
|
|
490
|
+
// `handed_off_at_ms` is always set on a handed-off row (and the SELECT
|
|
491
|
+
// required it `<= cutoff`), so it is a reliable null-safe lease pin.
|
|
467
492
|
const orphanedJob = await this._applyFailure({
|
|
468
493
|
db,
|
|
469
494
|
job,
|
|
470
495
|
error: "Job orphaned after timeout",
|
|
471
|
-
markOrphaned: true
|
|
496
|
+
markOrphaned: true,
|
|
497
|
+
conditions: {id: job.id, status: "handed_off", handed_off_at_ms: job.handedOffAtMs}
|
|
472
498
|
})
|
|
473
499
|
|
|
474
500
|
if (orphanedJob) orphanedCount += 1
|
|
@@ -478,6 +504,78 @@ export default class BackgroundJobsStore {
|
|
|
478
504
|
})
|
|
479
505
|
}
|
|
480
506
|
|
|
507
|
+
/**
|
|
508
|
+
* Deletes terminal job rows past their retention window so the jobs table
|
|
509
|
+
* does not grow unbounded (completed rows in particular accumulate forever
|
|
510
|
+
* otherwise). Batched by id — SELECT a page of ids, then
|
|
511
|
+
* `DELETE ... WHERE id IN (...)` — rather than `DELETE ... LIMIT`, which not
|
|
512
|
+
* every driver supports; each batch runs on its own connection so the sweep
|
|
513
|
+
* yields between batches instead of holding one long transaction.
|
|
514
|
+
* @param {object} [args] - Options.
|
|
515
|
+
* @param {number | null} [args.completedTtlMs] - Delete `completed` jobs whose `completed_at_ms` is older than this many ms. Falsy or `<= 0` disables completed pruning.
|
|
516
|
+
* @param {number | null} [args.failedTtlMs] - Delete terminal `failed`/`orphaned` jobs older than this many ms (by `failed_at_ms`/`orphaned_at_ms`). Falsy or `<= 0` disables.
|
|
517
|
+
* @param {number} [args.batchSize] - Max rows deleted per batch. Default `1000`.
|
|
518
|
+
* @returns {Promise<number>} - Total rows deleted.
|
|
519
|
+
*/
|
|
520
|
+
async pruneTerminalJobs({completedTtlMs = null, failedTtlMs = null, batchSize = 1000} = {}) {
|
|
521
|
+
await this.ensureReady()
|
|
522
|
+
|
|
523
|
+
const now = Date.now()
|
|
524
|
+
const size = batchSize > 0 ? batchSize : 1000
|
|
525
|
+
let deleted = 0
|
|
526
|
+
|
|
527
|
+
if (completedTtlMs && completedTtlMs > 0) {
|
|
528
|
+
deleted += await this._pruneStatusBatches({status: "completed", column: "completed_at_ms", cutoff: now - completedTtlMs, batchSize: size})
|
|
529
|
+
}
|
|
530
|
+
|
|
531
|
+
if (failedTtlMs && failedTtlMs > 0) {
|
|
532
|
+
deleted += await this._pruneStatusBatches({status: "failed", column: "failed_at_ms", cutoff: now - failedTtlMs, batchSize: size})
|
|
533
|
+
deleted += await this._pruneStatusBatches({status: "orphaned", column: "orphaned_at_ms", cutoff: now - failedTtlMs, batchSize: size})
|
|
534
|
+
}
|
|
535
|
+
|
|
536
|
+
return deleted
|
|
537
|
+
}
|
|
538
|
+
|
|
539
|
+
/**
|
|
540
|
+
* Deletes rows of one terminal status older than a cutoff, batch by batch,
|
|
541
|
+
* until a page returns fewer than `batchSize` rows.
|
|
542
|
+
* @param {object} args - Options.
|
|
543
|
+
* @param {string} args.status - Terminal status to prune.
|
|
544
|
+
* @param {string} args.column - Timestamp column compared against the cutoff.
|
|
545
|
+
* @param {number} args.cutoff - Delete rows whose column value is `<= cutoff`.
|
|
546
|
+
* @param {number} args.batchSize - Max rows per batch.
|
|
547
|
+
* @returns {Promise<number>} - Rows deleted for this status.
|
|
548
|
+
*/
|
|
549
|
+
async _pruneStatusBatches({status, column, cutoff, batchSize}) {
|
|
550
|
+
let deleted = 0
|
|
551
|
+
|
|
552
|
+
for (;;) {
|
|
553
|
+
const removed = await this._withDb(async (db) => {
|
|
554
|
+
const rows = await db
|
|
555
|
+
.newQuery()
|
|
556
|
+
.from(JOBS_TABLE)
|
|
557
|
+
.select("id")
|
|
558
|
+
.where({status})
|
|
559
|
+
.where(`${db.quoteColumn(column)} <= ${db.quote(cutoff)}`)
|
|
560
|
+
.limit(batchSize)
|
|
561
|
+
.results()
|
|
562
|
+
|
|
563
|
+
if (rows.length === 0) return 0
|
|
564
|
+
|
|
565
|
+
const ids = rows.map((/** @type {Record<string, ?>} */ row) => db.quote(String(row.id))).join(", ")
|
|
566
|
+
|
|
567
|
+
await db.query(`DELETE FROM ${db.quoteTable(JOBS_TABLE)} WHERE ${db.quoteColumn("id")} IN (${ids})`)
|
|
568
|
+
|
|
569
|
+
return rows.length
|
|
570
|
+
})
|
|
571
|
+
|
|
572
|
+
deleted += removed
|
|
573
|
+
if (removed < batchSize) break
|
|
574
|
+
}
|
|
575
|
+
|
|
576
|
+
return deleted
|
|
577
|
+
}
|
|
578
|
+
|
|
481
579
|
/**
|
|
482
580
|
* Runs clear all.
|
|
483
581
|
* @returns {Promise<void>} - Resolves when cleared.
|
|
@@ -621,6 +719,7 @@ export default class BackgroundJobsStore {
|
|
|
621
719
|
table.text("args_json", {null: false})
|
|
622
720
|
table.boolean("forked", {null: false})
|
|
623
721
|
table.string("execution_mode", {null: false})
|
|
722
|
+
table.string("queue", {null: true, index: true})
|
|
624
723
|
table.integer("max_retries", {null: false})
|
|
625
724
|
table.integer("attempts", {null: false})
|
|
626
725
|
table.string("status", {null: false, index: true})
|
|
@@ -723,6 +822,43 @@ export default class BackgroundJobsStore {
|
|
|
723
822
|
} finally {
|
|
724
823
|
await db.releaseAdvisoryLock(lockName)
|
|
725
824
|
}
|
|
825
|
+
|
|
826
|
+
await this._ensureQueueColumn(db)
|
|
827
|
+
}
|
|
828
|
+
|
|
829
|
+
/**
|
|
830
|
+
* Idempotently adds the `queue` column to an existing jobs table. Existing
|
|
831
|
+
* rows read back as the default queue (see {@link _normalizeJobRow}), so no
|
|
832
|
+
* data backfill is required.
|
|
833
|
+
* @param {import("../database/drivers/base.js").default} db - Database connection.
|
|
834
|
+
* @returns {Promise<void>} - Resolves when ensured.
|
|
835
|
+
*/
|
|
836
|
+
async _ensureQueueColumn(db) {
|
|
837
|
+
const table = await db.getTableByNameOrFail(JOBS_TABLE)
|
|
838
|
+
|
|
839
|
+
if (await table.getColumnByName("queue")) return
|
|
840
|
+
|
|
841
|
+
const lockName = `${MIGRATION_SCOPE}:queue_column`
|
|
842
|
+
const acquired = await db.acquireAdvisoryLock(lockName)
|
|
843
|
+
|
|
844
|
+
if (!acquired) throw new Error("Failed to acquire background jobs queue schema lock")
|
|
845
|
+
|
|
846
|
+
try {
|
|
847
|
+
db.clearSchemaCache()
|
|
848
|
+
const lockedTable = await db.getTableByNameOrFail(JOBS_TABLE)
|
|
849
|
+
|
|
850
|
+
if (!(await lockedTable.getColumnByName("queue"))) {
|
|
851
|
+
const tableData = new TableData(JOBS_TABLE)
|
|
852
|
+
|
|
853
|
+
tableData.string("queue", {null: true, index: true})
|
|
854
|
+
|
|
855
|
+
for (const sql of await db.alterTableSQLs(tableData)) await db.query(sql)
|
|
856
|
+
|
|
857
|
+
db.clearSchemaCache()
|
|
858
|
+
}
|
|
859
|
+
} finally {
|
|
860
|
+
await db.releaseAdvisoryLock(lockName)
|
|
861
|
+
}
|
|
726
862
|
}
|
|
727
863
|
|
|
728
864
|
/**
|
|
@@ -819,9 +955,10 @@ export default class BackgroundJobsStore {
|
|
|
819
955
|
* @param {import("./types.js").BackgroundJobRow} args.job - Job row.
|
|
820
956
|
* @param {?} args.error - Error.
|
|
821
957
|
* @param {boolean} args.markOrphaned - Whether marking orphaned.
|
|
958
|
+
* @param {Record<string, ?>} [args.conditions] - Update fencing conditions. Defaults to the active-handoff lease match; the time-based orphan sweep overrides this with an id/status match so it can reclaim rows whose `handoff_id` is null (e.g. handed off by an older velocious before handoff-id fencing existed).
|
|
822
959
|
* @returns {Promise<import("./types.js").BackgroundJobRow | null>} - Updated job row when the lease transition won.
|
|
823
960
|
*/
|
|
824
|
-
async _applyFailure({db, job, error, markOrphaned}) {
|
|
961
|
+
async _applyFailure({db, job, error, markOrphaned, conditions}) {
|
|
825
962
|
const now = Date.now()
|
|
826
963
|
const nextAttempt = (job.attempts || 0) + 1
|
|
827
964
|
const maxRetries = this._normalizeMaxRetries(job.maxRetries)
|
|
@@ -840,7 +977,7 @@ export default class BackgroundJobsStore {
|
|
|
840
977
|
const affectedRows = await this._updateAffectedRows(db, {
|
|
841
978
|
tableName: JOBS_TABLE,
|
|
842
979
|
data: update,
|
|
843
|
-
conditions: this._activeHandoffConditions(job)
|
|
980
|
+
conditions: conditions ?? this._activeHandoffConditions(job)
|
|
844
981
|
})
|
|
845
982
|
|
|
846
983
|
if (affectedRows !== 1) return null
|
|
@@ -934,6 +1071,7 @@ export default class BackgroundJobsStore {
|
|
|
934
1071
|
args: this._parseArgs(row.args_json),
|
|
935
1072
|
executionMode,
|
|
936
1073
|
forked: executionMode !== "inline",
|
|
1074
|
+
queue: row.queue ? String(row.queue) : DEFAULT_QUEUE,
|
|
937
1075
|
status: row.status ? String(row.status) : "queued",
|
|
938
1076
|
attempts: this._normalizeNumber(row.attempts),
|
|
939
1077
|
maxRetries: this._normalizeNumber(row.max_retries),
|
|
@@ -966,6 +1104,89 @@ export default class BackgroundJobsStore {
|
|
|
966
1104
|
return {concurrencyKey: key, maxConcurrency: Number(cap)}
|
|
967
1105
|
}
|
|
968
1106
|
|
|
1107
|
+
/**
|
|
1108
|
+
* Normalizes a job's queue name, defaulting to "default".
|
|
1109
|
+
* @param {import("./types.js").BackgroundJobOptions | undefined} options - Job options.
|
|
1110
|
+
* @returns {string} - Queue name.
|
|
1111
|
+
*/
|
|
1112
|
+
_normalizeQueue(options) {
|
|
1113
|
+
const queue = options?.queue
|
|
1114
|
+
|
|
1115
|
+
if (typeof queue === "string" && queue.trim().length > 0) return queue.trim()
|
|
1116
|
+
|
|
1117
|
+
return DEFAULT_QUEUE
|
|
1118
|
+
}
|
|
1119
|
+
|
|
1120
|
+
/**
|
|
1121
|
+
* Resolves a job's durable concurrency. An explicit concurrencyKey/maxConcurrency
|
|
1122
|
+
* pair always wins. Otherwise, when the job's queue has a configured cap
|
|
1123
|
+
* (`backgroundJobs.queues[queue].maxConcurrent`), derive a queue-scoped
|
|
1124
|
+
* concurrency key so the queue cap is enforced cluster-wide through the
|
|
1125
|
+
* existing durable concurrency mechanism.
|
|
1126
|
+
* @param {import("./types.js").BackgroundJobOptions | undefined} options - Job options.
|
|
1127
|
+
* @param {string} queue - Normalized queue name.
|
|
1128
|
+
* @returns {{concurrencyKey: string, maxConcurrency: number, queueDerived: boolean} | null} - Resolved concurrency.
|
|
1129
|
+
*/
|
|
1130
|
+
_resolveConcurrency(options, queue) {
|
|
1131
|
+
const explicit = this._normalizeConcurrencyOptions(options)
|
|
1132
|
+
|
|
1133
|
+
if (explicit) return {...explicit, queueDerived: false}
|
|
1134
|
+
|
|
1135
|
+
const cap = this._queueMaxConcurrency(queue)
|
|
1136
|
+
|
|
1137
|
+
if (cap === null) return null
|
|
1138
|
+
|
|
1139
|
+
return {concurrencyKey: `${QUEUE_CONCURRENCY_KEY_PREFIX}${queue}`, maxConcurrency: cap, queueDerived: true}
|
|
1140
|
+
}
|
|
1141
|
+
|
|
1142
|
+
/**
|
|
1143
|
+
* Reads the configured max concurrency for a queue from the background-jobs config.
|
|
1144
|
+
* @param {string} queue - Queue name.
|
|
1145
|
+
* @returns {number | null} - Positive integer cap, or null when the queue has no configured cap.
|
|
1146
|
+
*/
|
|
1147
|
+
_queueMaxConcurrency(queue) {
|
|
1148
|
+
const queues = this.configuration.getBackgroundJobsConfig().queues
|
|
1149
|
+
const cap = queues?.[queue]?.maxConcurrent
|
|
1150
|
+
|
|
1151
|
+
if (Number.isInteger(cap) && Number(cap) > 0) return Number(cap)
|
|
1152
|
+
|
|
1153
|
+
return null
|
|
1154
|
+
}
|
|
1155
|
+
|
|
1156
|
+
/**
|
|
1157
|
+
* Like {@link _ensureConcurrencyKey}, but for queue-derived keys the configured
|
|
1158
|
+
* queue cap is the source of truth: if it changed, update the stored cap
|
|
1159
|
+
* instead of throwing on conflict (config-driven caps must be tunable).
|
|
1160
|
+
* @param {import("../database/drivers/base.js").default} db - Database connection.
|
|
1161
|
+
* @param {{concurrencyKey: string, maxConcurrency: number}} concurrency - Concurrency configuration.
|
|
1162
|
+
* @returns {Promise<void>} - Resolves when ensured.
|
|
1163
|
+
*/
|
|
1164
|
+
async _ensureQueueConcurrencyKey(db, {concurrencyKey, maxConcurrency}) {
|
|
1165
|
+
const rows = await db.newQuery().from(CONCURRENCY_TABLE).where({concurrency_key: concurrencyKey}).limit(1).results()
|
|
1166
|
+
|
|
1167
|
+
if (!rows[0]) {
|
|
1168
|
+
try {
|
|
1169
|
+
await db.insert({tableName: CONCURRENCY_TABLE, data: {active_count: 0, concurrency_key: concurrencyKey, max_concurrency: maxConcurrency}})
|
|
1170
|
+
|
|
1171
|
+
return
|
|
1172
|
+
} catch (error) {
|
|
1173
|
+
const racedRows = await db.newQuery().from(CONCURRENCY_TABLE).where({concurrency_key: concurrencyKey}).limit(1).results()
|
|
1174
|
+
|
|
1175
|
+
if (!racedRows[0]) throw error
|
|
1176
|
+
|
|
1177
|
+
rows[0] = racedRows[0]
|
|
1178
|
+
}
|
|
1179
|
+
}
|
|
1180
|
+
|
|
1181
|
+
const configured = /** @type {{max_concurrency?: number | string}} */ (rows[0])
|
|
1182
|
+
|
|
1183
|
+
if (this._normalizeNumber(configured.max_concurrency) !== maxConcurrency) {
|
|
1184
|
+
const table = db.quoteTable(CONCURRENCY_TABLE)
|
|
1185
|
+
|
|
1186
|
+
await db.query(`UPDATE ${table} SET ${db.quoteColumn("max_concurrency")} = ${Number(maxConcurrency)} WHERE ${db.quoteColumn("concurrency_key")} = ${db.quote(concurrencyKey)}`)
|
|
1187
|
+
}
|
|
1188
|
+
}
|
|
1189
|
+
|
|
969
1190
|
/**
|
|
970
1191
|
* Ensures the concurrency state table exists.
|
|
971
1192
|
* @param {import("../database/drivers/base.js").default} db - Database connection.
|
|
@@ -13,7 +13,8 @@
|
|
|
13
13
|
* @property {BackgroundJobExecutionMode} [executionMode] - How the job should run. Defaults to `"forked"`.
|
|
14
14
|
* @property {boolean} [forked] - Compatibility alias: `false` maps to `"inline"` and `true` maps to `"forked"`.
|
|
15
15
|
* @property {number} [maxRetries] - Max retries for a failed job before it is marked failed.
|
|
16
|
-
* @property {string} [
|
|
16
|
+
* @property {string} [queue] - Queue name. Defaults to `"default"`. When the queue has a configured cap in `backgroundJobs.queues`, that cap is enforced cluster-wide.
|
|
17
|
+
* @property {string} [concurrencyKey] - Opaque non-empty key used to share a concurrency cap. Overrides any queue-derived cap.
|
|
17
18
|
* @property {number} [maxConcurrency] - Positive integer cap; must be paired with `concurrencyKey`.
|
|
18
19
|
*/
|
|
19
20
|
/**
|
|
@@ -33,6 +34,7 @@
|
|
|
33
34
|
* @property {Array<?>} args - Serialized job arguments.
|
|
34
35
|
* @property {BackgroundJobExecutionMode} executionMode - How the job should run.
|
|
35
36
|
* @property {boolean} forked - Compatibility flag; true for non-inline execution.
|
|
37
|
+
* @property {string} queue - Queue name (defaults to `"default"`).
|
|
36
38
|
* @property {string} status - Current job status.
|
|
37
39
|
* @property {number | null} attempts - Failure attempts count.
|
|
38
40
|
* @property {number | null} maxRetries - Max retry attempts.
|
|
@@ -161,12 +161,41 @@
|
|
|
161
161
|
* long-running jobs and for using more cores. Default: `4`.
|
|
162
162
|
* @property {number} [maxConcurrentForkedJobs] - How many out-of-process
|
|
163
163
|
* `"forked"` or `"spawned"` jobs a single `background-jobs-worker` is
|
|
164
|
-
* allowed to keep in flight. Default: `4`.
|
|
164
|
+
* allowed to keep in flight. Default: `4`. This is a per-worker safety cap;
|
|
165
|
+
* for workload-shaped limits use per-queue caps (`queues`) instead, which are
|
|
166
|
+
* enforced cluster-wide and are immune to duplicate worker processes.
|
|
167
|
+
* @property {Record<string, {maxConcurrent?: number}>} [queues] - Per-queue
|
|
168
|
+
* concurrency caps, Sidekiq-style. A job declares its queue (static `queue` on
|
|
169
|
+
* the job class, or the `queue` enqueue option; defaults to `"default"`), and
|
|
170
|
+
* `queues[name].maxConcurrent` bounds how many jobs from that queue may be
|
|
171
|
+
* in flight across the whole cluster (enforced via durable per-key
|
|
172
|
+
* concurrency, so it holds regardless of how many worker processes run).
|
|
173
|
+
* Size each queue to its workload: I/O-bound queues (e.g. build runners
|
|
174
|
+
* waiting on remote Docker servers) can run far above the core count, while
|
|
175
|
+
* CPU-bound queues should stay near it. Default: `{}` (no queue caps).
|
|
165
176
|
* @property {BackgroundJobsDispatchStrategy} [dispatchStrategy] - How the main process
|
|
166
177
|
* detects new work. Defaults to `"beacon"` (event-driven). Set to `"polling"`
|
|
167
178
|
* to restore the legacy fixed-interval poll.
|
|
168
179
|
* @property {number} [pollIntervalMs] - Poll interval in milliseconds. Only used
|
|
169
180
|
* when `dispatchStrategy === "polling"`. Default: `1000`.
|
|
181
|
+
* @property {BackgroundJobsRetentionConfiguration} [retention] - Retention/pruning
|
|
182
|
+
* of terminal job rows. Without pruning the jobs table grows unbounded
|
|
183
|
+
* (completed rows accumulate forever), which bloats storage and indexes and
|
|
184
|
+
* eventually slows dispatch. The main process sweeps terminal rows past their
|
|
185
|
+
* window on an interval.
|
|
186
|
+
*/
|
|
187
|
+
|
|
188
|
+
/**
|
|
189
|
+
* @typedef {object} BackgroundJobsRetentionConfiguration
|
|
190
|
+
* @property {number | null} [completedTtlMs] - Delete `completed` jobs whose
|
|
191
|
+
* `completed_at_ms` is older than this many ms. `null` or `<= 0` disables
|
|
192
|
+
* completed pruning. Default: `604800000` (7 days).
|
|
193
|
+
* @property {number | null} [failedTtlMs] - Delete terminal `failed`/`orphaned`
|
|
194
|
+
* jobs older than this many ms. `null` or `<= 0` disables (keeps them for
|
|
195
|
+
* debugging). Default: `2592000000` (30 days).
|
|
196
|
+
* @property {number} [batchSize] - Rows deleted per batch. Default: `1000`.
|
|
197
|
+
* @property {number} [sweepIntervalMs] - How often the main process runs the
|
|
198
|
+
* retention sweep. Default: `3600000` (1 hour).
|
|
170
199
|
*/
|
|
171
200
|
|
|
172
201
|
/**
|
package/build/configuration.js
CHANGED
|
@@ -1291,8 +1291,24 @@ export default class VelociousConfiguration {
|
|
|
1291
1291
|
const pollIntervalMs = typeof configured.pollIntervalMs === "number" && configured.pollIntervalMs >= 1
|
|
1292
1292
|
? configured.pollIntervalMs
|
|
1293
1293
|
: (typeof envPollInterval === "number" && Number.isFinite(envPollInterval) && envPollInterval >= 1 ? envPollInterval : 1000)
|
|
1294
|
-
|
|
1295
|
-
|
|
1294
|
+
const queues = configured.queues && typeof configured.queues === "object" ? configured.queues : {}
|
|
1295
|
+
const configuredRetention = configured.retention && typeof configured.retention === "object" ? configured.retention : {}
|
|
1296
|
+
const retention = {
|
|
1297
|
+
completedTtlMs: typeof configuredRetention.completedTtlMs === "number" || configuredRetention.completedTtlMs === null
|
|
1298
|
+
? configuredRetention.completedTtlMs
|
|
1299
|
+
: 7 * 24 * 60 * 60 * 1000,
|
|
1300
|
+
failedTtlMs: typeof configuredRetention.failedTtlMs === "number" || configuredRetention.failedTtlMs === null
|
|
1301
|
+
? configuredRetention.failedTtlMs
|
|
1302
|
+
: 30 * 24 * 60 * 60 * 1000,
|
|
1303
|
+
batchSize: typeof configuredRetention.batchSize === "number" && configuredRetention.batchSize > 0
|
|
1304
|
+
? configuredRetention.batchSize
|
|
1305
|
+
: 1000,
|
|
1306
|
+
sweepIntervalMs: typeof configuredRetention.sweepIntervalMs === "number" && configuredRetention.sweepIntervalMs > 0
|
|
1307
|
+
? configuredRetention.sweepIntervalMs
|
|
1308
|
+
: 60 * 60 * 1000
|
|
1309
|
+
}
|
|
1310
|
+
|
|
1311
|
+
return {host, port, databaseIdentifier, maxConcurrentForkedJobs, maxConcurrentInlineJobs, dispatchStrategy, pollIntervalMs, queues, retention}
|
|
1296
1312
|
}
|
|
1297
1313
|
|
|
1298
1314
|
/**
|
|
@@ -217,4 +217,22 @@ export default class RecordAttachmentHandle {
|
|
|
217
217
|
|
|
218
218
|
return await store.attachmentRowUrl({model: this.model, name: this.name, row})
|
|
219
219
|
}
|
|
220
|
+
|
|
221
|
+
/**
|
|
222
|
+
* Purges every attachment under this (record, name): deletes the backing
|
|
223
|
+
* storage for each and removes the attachment rows. A no-op for unpersisted
|
|
224
|
+
* records. Only the attachments present when the purge starts are removed, so a
|
|
225
|
+
* concurrent attach for the same (record, name) is left intact. Throws (without
|
|
226
|
+
* deleting any rows) if a storage driver cannot delete its object, so a driver
|
|
227
|
+
* configured without a `delete` operation can never leak storage. Callers use
|
|
228
|
+
* this to clean up attachments before destroying the owner record.
|
|
229
|
+
* @returns {Promise<number>} - Number of attachments purged.
|
|
230
|
+
*/
|
|
231
|
+
async purgeAll() {
|
|
232
|
+
if (!this.model.isPersisted()) return 0
|
|
233
|
+
|
|
234
|
+
const store = recordAttachmentsStoreForModel(this.model)
|
|
235
|
+
|
|
236
|
+
return await store.purgeAll({model: this.model, name: this.name})
|
|
237
|
+
}
|
|
220
238
|
}
|
|
@@ -391,6 +391,51 @@ export default class RecordAttachmentsStore {
|
|
|
391
391
|
})
|
|
392
392
|
}
|
|
393
393
|
|
|
394
|
+
/**
|
|
395
|
+
* Purges every attachment stored under (model, name): deletes each row's
|
|
396
|
+
* backing storage and then removes the attachment rows. Used to clean up an
|
|
397
|
+
* owner record's attachments before/when the owner is destroyed.
|
|
398
|
+
* @param {object} args - Options.
|
|
399
|
+
* @param {import("../index.js").default} args.model - Model instance.
|
|
400
|
+
* @param {string} args.name - Attachment name.
|
|
401
|
+
* @returns {Promise<number>} - Number of attachments purged.
|
|
402
|
+
*/
|
|
403
|
+
async purgeAll({model, name}) {
|
|
404
|
+
await this.ensureReady()
|
|
405
|
+
|
|
406
|
+
return await this._withDb(async (db) => {
|
|
407
|
+
const recordType = model.getModelClass().getModelName()
|
|
408
|
+
const recordId = String(model.id())
|
|
409
|
+
/** @type {Array<Record<string, ?>>} */
|
|
410
|
+
const rows = await db
|
|
411
|
+
.newQuery()
|
|
412
|
+
.from(ATTACHMENTS_TABLE)
|
|
413
|
+
.where({name, record_id: recordId, record_type: recordType})
|
|
414
|
+
.results()
|
|
415
|
+
|
|
416
|
+
// Refuse to purge when any row's driver cannot delete its backing storage:
|
|
417
|
+
// removing the row while the object stays behind would leak storage and
|
|
418
|
+
// discard the metadata needed to retry cleanup. Fail loudly instead.
|
|
419
|
+
for (const row of rows) {
|
|
420
|
+
const attachmentDriver = await this.resolveAttachmentDriver({model, name, row})
|
|
421
|
+
|
|
422
|
+
if (typeof attachmentDriver.delete !== "function") {
|
|
423
|
+
throw new Error(`Cannot purge attachment ${row.id} for ${recordType}#${recordId} (${name}): its storage driver does not support deletion.`)
|
|
424
|
+
}
|
|
425
|
+
}
|
|
426
|
+
|
|
427
|
+
for (const row of rows) {
|
|
428
|
+
await this.deleteAttachmentRowStorage({model, name, row})
|
|
429
|
+
// Delete only the snapshotted row by id, so an attachment inserted for the
|
|
430
|
+
// same (record, name) after the snapshot is not removed with its storage
|
|
431
|
+
// still present (which would leave it as unreachable storage).
|
|
432
|
+
await db.delete({conditions: {id: row.id}, tableName: ATTACHMENTS_TABLE})
|
|
433
|
+
}
|
|
434
|
+
|
|
435
|
+
return rows.length
|
|
436
|
+
})
|
|
437
|
+
}
|
|
438
|
+
|
|
394
439
|
/**
|
|
395
440
|
* Runs attachment driver by name.
|
|
396
441
|
* @param {string} driverName - Driver name.
|
|
@@ -400,7 +400,7 @@ export default class VeoliciousHttpServerClient {
|
|
|
400
400
|
const stats = await fs.stat(filePath)
|
|
401
401
|
contentLength = stats.size
|
|
402
402
|
} else {
|
|
403
|
-
contentLength = bodyIsString ?
|
|
403
|
+
contentLength = bodyIsString ? Buffer.byteLength(body, "utf8") : body.byteLength
|
|
404
404
|
}
|
|
405
405
|
|
|
406
406
|
response.setHeader("Content-Length", contentLength)
|