velocious 1.0.623 → 1.0.625
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +12 -0
- package/build/background-jobs/store.js +136 -131
- package/build/background-jobs/worker.js +57 -4
- package/build/src/background-jobs/store.d.ts +21 -13
- package/build/src/background-jobs/store.d.ts.map +1 -1
- package/build/src/background-jobs/store.js +136 -130
- package/build/src/background-jobs/worker.d.ts +22 -1
- package/build/src/background-jobs/worker.d.ts.map +1 -1
- package/build/src/background-jobs/worker.js +58 -5
- package/package.json +10 -2
- package/src/background-jobs/store.js +136 -131
- package/src/background-jobs/worker.js +57 -4
package/README.md
CHANGED
|
@@ -60,6 +60,16 @@ npx velocious init
|
|
|
60
60
|
|
|
61
61
|
By default, Velocious looks for your configuration in `src/config/configuration.js`. If you keep the configuration elsewhere, make sure your app imports it early and calls `configuration.setCurrent()`.
|
|
62
62
|
|
|
63
|
+
# Node SQLite driver
|
|
64
|
+
|
|
65
|
+
Projects using `velocious/build/src/database/drivers/sqlite/index.js` must install its optional peer dependencies:
|
|
66
|
+
|
|
67
|
+
```bash
|
|
68
|
+
npm install sqlite sqlite3
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
Projects that do not use the Node SQLite driver do not need these packages. Browser and Expo SQLite drivers use their platform-specific dependencies instead.
|
|
72
|
+
|
|
63
73
|
# Operation-scoped transactions
|
|
64
74
|
|
|
65
75
|
Use `configuration.withTransaction` for an atomic unit of model work on one database:
|
|
@@ -2506,6 +2516,8 @@ delay and never returns: it stops the current `perform`, releases its worker and
|
|
|
2506
2516
|
concurrency slots, and makes the same job eligible again after the delay. This is
|
|
2507
2517
|
normal control flow, not failure retry: attempts and failure metadata remain
|
|
2508
2518
|
unchanged, retries are not consumed, and failure/error events are not emitted.
|
|
2519
|
+
Pooled workers serialize repeated leases for that same durable row through the
|
|
2520
|
+
prior terminal-acknowledgement boundary; other job IDs remain concurrent.
|
|
2509
2521
|
See [Rescheduling a running job](docs/background-jobs.md#rescheduling-a-running-job).
|
|
2510
2522
|
|
|
2511
2523
|
Use a durable stable key when the same logical one-off schedule must be moved or cancelled without retaining its transient job id:
|
|
@@ -43,6 +43,12 @@ import {
|
|
|
43
43
|
* @property {number | null} timeoutMs - Per-job timeout override, or null when omitted.
|
|
44
44
|
*/
|
|
45
45
|
|
|
46
|
+
/**
|
|
47
|
+
* BackgroundJobTransactionSerializationOptions type.
|
|
48
|
+
* @typedef {object} BackgroundJobTransactionSerializationOptions
|
|
49
|
+
* @property {{failureMessage: string, name: string}} [advisoryLock] - Session lock held around the transaction.
|
|
50
|
+
*/
|
|
51
|
+
|
|
46
52
|
const MIGRATIONS_TABLE = "velocious_internal_migrations"
|
|
47
53
|
const MIGRATION_SCOPE = "background_jobs"
|
|
48
54
|
const MIGRATION_VERSION = "20250215000000"
|
|
@@ -241,7 +247,7 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
|
|
|
241
247
|
/** @type {string} */
|
|
242
248
|
let resultJobId = preparedJob.jobId
|
|
243
249
|
|
|
244
|
-
await this.
|
|
250
|
+
await this._serializedCountMutation(async (db) => {
|
|
245
251
|
if (options?.deduplicateWhileQueued) {
|
|
246
252
|
// Dedupe on the job's identity (name + args + queue), NOT its concurrency key, so a job
|
|
247
253
|
// keeps whatever concurrency it resolves to. Only an existing job scheduled no later than
|
|
@@ -266,7 +272,7 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
|
|
|
266
272
|
|
|
267
273
|
await this._insertPreparedJob(db, {preparedJob, scheduleKey: null})
|
|
268
274
|
await this._recordCountDelta(db, {all: 1, queued: 1})
|
|
269
|
-
})
|
|
275
|
+
})
|
|
270
276
|
|
|
271
277
|
return resultJobId
|
|
272
278
|
}
|
|
@@ -303,7 +309,7 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
|
|
|
303
309
|
// Reuse ordinary enqueue transaction admission because this path changes
|
|
304
310
|
// the same durable count revision. The scope primary key remains the
|
|
305
311
|
// cross-process convergence owner.
|
|
306
|
-
return await this.
|
|
312
|
+
return await this._idempotentEnqueueTransaction(async (db) => {
|
|
307
313
|
const existing = await this._idempotencyOwnership(db, scopeDigest)
|
|
308
314
|
|
|
309
315
|
if (existing) {
|
|
@@ -326,19 +332,18 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
|
|
|
326
332
|
await this._recordCountDelta(db, {all: 1, queued: 1})
|
|
327
333
|
|
|
328
334
|
return preparedJob.jobId
|
|
329
|
-
})
|
|
335
|
+
})
|
|
330
336
|
}
|
|
331
337
|
|
|
332
338
|
/**
|
|
333
339
|
* Serializes one physical connection locally without taking ownership away
|
|
334
340
|
* from the database uniqueness constraint shared by all processes.
|
|
335
341
|
* @template T
|
|
336
|
-
* @param {import("../database/drivers/base.js").default}
|
|
337
|
-
* @param {() => Promise<T>} callback - Transaction work.
|
|
342
|
+
* @param {(db: import("../database/drivers/base.js").default) => Promise<T>} callback - Transaction work.
|
|
338
343
|
* @returns {Promise<T>} - Callback result.
|
|
339
344
|
*/
|
|
340
|
-
async _idempotentEnqueueTransaction(
|
|
341
|
-
return await this._serializedTransactionMutation(
|
|
345
|
+
async _idempotentEnqueueTransaction(callback) {
|
|
346
|
+
return await this._serializedTransactionMutation(callback)
|
|
342
347
|
}
|
|
343
348
|
|
|
344
349
|
/**
|
|
@@ -564,63 +569,56 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
|
|
|
564
569
|
const normalizedScheduleKey = this._normalizeScheduleKey(scheduleKey)
|
|
565
570
|
const preparedJob = this._prepareJob({jobName, args, options})
|
|
566
571
|
|
|
567
|
-
return await this.
|
|
568
|
-
const
|
|
569
|
-
|
|
572
|
+
return await this._serializedCountMutation(async (db) => {
|
|
573
|
+
const ownerRows = await db
|
|
574
|
+
.newQuery()
|
|
575
|
+
.from(SCHEDULE_KEYS_TABLE)
|
|
576
|
+
.where({schedule_key: normalizedScheduleKey})
|
|
577
|
+
.limit(1)
|
|
578
|
+
.results()
|
|
579
|
+
const ownerJobId = ownerRows[0] ? String(/** @type {Record<string, ReturnType<typeof JSON.parse>>} */ (ownerRows[0]).job_id) : null
|
|
580
|
+
const ownerJob = ownerJobId ? await this._getJobRowById(db, ownerJobId) : null
|
|
581
|
+
/** @type {import("./types.js").BackgroundJobReplacementPreviousStatus} */
|
|
582
|
+
let previousStatus = null
|
|
583
|
+
let previousJobId = null
|
|
584
|
+
|
|
585
|
+
if (ownerJob?.status === "queued") {
|
|
586
|
+
const affectedRows = await this._updateAffectedRows(db, {
|
|
587
|
+
tableName: JOBS_TABLE,
|
|
588
|
+
data: {status: "cancelled"},
|
|
589
|
+
conditions: {id: ownerJob.id, status: "queued"}
|
|
590
|
+
})
|
|
570
591
|
|
|
571
|
-
|
|
592
|
+
if (affectedRows === 1) {
|
|
593
|
+
previousJobId = ownerJob.id
|
|
594
|
+
previousStatus = "queued"
|
|
595
|
+
} else {
|
|
596
|
+
const currentOwnerJob = await this._getJobRowById(db, ownerJob.id)
|
|
572
597
|
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
const ownerRows = await db
|
|
576
|
-
.newQuery()
|
|
577
|
-
.from(SCHEDULE_KEYS_TABLE)
|
|
578
|
-
.where({schedule_key: normalizedScheduleKey})
|
|
579
|
-
.limit(1)
|
|
580
|
-
.results()
|
|
581
|
-
const ownerJobId = ownerRows[0] ? String(/** @type {Record<string, ReturnType<typeof JSON.parse>>} */ (ownerRows[0]).job_id) : null
|
|
582
|
-
const ownerJob = ownerJobId ? await this._getJobRowById(db, ownerJobId) : null
|
|
583
|
-
/** @type {import("./types.js").BackgroundJobReplacementPreviousStatus} */
|
|
584
|
-
let previousStatus = null
|
|
585
|
-
let previousJobId = null
|
|
586
|
-
|
|
587
|
-
if (ownerJob?.status === "queued") {
|
|
588
|
-
const affectedRows = await this._updateAffectedRows(db, {
|
|
589
|
-
tableName: JOBS_TABLE,
|
|
590
|
-
data: {status: "cancelled"},
|
|
591
|
-
conditions: {id: ownerJob.id, status: "queued"}
|
|
592
|
-
})
|
|
593
|
-
|
|
594
|
-
if (affectedRows === 1) {
|
|
595
|
-
previousJobId = ownerJob.id
|
|
596
|
-
previousStatus = "queued"
|
|
597
|
-
} else {
|
|
598
|
-
const currentOwnerJob = await this._getJobRowById(db, ownerJob.id)
|
|
599
|
-
|
|
600
|
-
if (currentOwnerJob?.status === "handed_off") {
|
|
601
|
-
previousJobId = currentOwnerJob.id
|
|
602
|
-
previousStatus = "handed_off"
|
|
603
|
-
}
|
|
604
|
-
}
|
|
605
|
-
} else if (ownerJob?.status === "handed_off") {
|
|
606
|
-
previousJobId = ownerJob.id
|
|
598
|
+
if (currentOwnerJob?.status === "handed_off") {
|
|
599
|
+
previousJobId = currentOwnerJob.id
|
|
607
600
|
previousStatus = "handed_off"
|
|
608
601
|
}
|
|
602
|
+
}
|
|
603
|
+
} else if (ownerJob?.status === "handed_off") {
|
|
604
|
+
previousJobId = ownerJob.id
|
|
605
|
+
previousStatus = "handed_off"
|
|
606
|
+
}
|
|
609
607
|
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
if (previousStatus !== "queued") await this._recordCountDelta(db, {all: 1, queued: 1})
|
|
608
|
+
await this._insertPreparedJob(db, {preparedJob, scheduleKey: normalizedScheduleKey})
|
|
609
|
+
await db.upsert({
|
|
610
|
+
tableName: SCHEDULE_KEYS_TABLE,
|
|
611
|
+
data: {schedule_key: normalizedScheduleKey, job_id: preparedJob.jobId},
|
|
612
|
+
conflictColumns: ["schedule_key"],
|
|
613
|
+
updateColumns: ["job_id"]
|
|
614
|
+
})
|
|
619
615
|
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
616
|
+
if (previousStatus !== "queued") await this._recordCountDelta(db, {all: 1, queued: 1})
|
|
617
|
+
return {jobId: preparedJob.jobId, previousJobId, previousStatus}
|
|
618
|
+
}, {
|
|
619
|
+
advisoryLock: {
|
|
620
|
+
failureMessage: "Failed to acquire background job schedule-key lock",
|
|
621
|
+
name: this._scheduleKeyLockName(normalizedScheduleKey)
|
|
624
622
|
}
|
|
625
623
|
})
|
|
626
624
|
}
|
|
@@ -636,51 +634,44 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
|
|
|
636
634
|
|
|
637
635
|
const normalizedScheduleKey = this._normalizeScheduleKey(scheduleKey)
|
|
638
636
|
|
|
639
|
-
return await this.
|
|
640
|
-
const
|
|
641
|
-
|
|
637
|
+
return await this._serializedCountMutation(async (db) => {
|
|
638
|
+
const ownerRows = await db
|
|
639
|
+
.newQuery()
|
|
640
|
+
.from(SCHEDULE_KEYS_TABLE)
|
|
641
|
+
.where({schedule_key: normalizedScheduleKey})
|
|
642
|
+
.limit(1)
|
|
643
|
+
.results()
|
|
642
644
|
|
|
643
|
-
if (!
|
|
645
|
+
if (!ownerRows[0]) return {jobId: null, outcome: "not_found"}
|
|
644
646
|
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
const ownerRows = await db
|
|
648
|
-
.newQuery()
|
|
649
|
-
.from(SCHEDULE_KEYS_TABLE)
|
|
650
|
-
.where({schedule_key: normalizedScheduleKey})
|
|
651
|
-
.limit(1)
|
|
652
|
-
.results()
|
|
653
|
-
|
|
654
|
-
if (!ownerRows[0]) return {jobId: null, outcome: "not_found"}
|
|
655
|
-
|
|
656
|
-
const jobId = String(/** @type {Record<string, ReturnType<typeof JSON.parse>>} */ (ownerRows[0]).job_id)
|
|
657
|
-
const job = await this._getJobRowById(db, jobId)
|
|
658
|
-
|
|
659
|
-
if (job?.status === "queued") {
|
|
660
|
-
const affectedRows = await this._updateAffectedRows(db, {
|
|
661
|
-
tableName: JOBS_TABLE,
|
|
662
|
-
data: {status: "cancelled"},
|
|
663
|
-
conditions: {id: job.id, status: "queued"}
|
|
664
|
-
})
|
|
665
|
-
|
|
666
|
-
if (affectedRows === 1) {
|
|
667
|
-
await this._releaseScheduleOwnership(db, {jobId, scheduleKey: normalizedScheduleKey})
|
|
668
|
-
await this._recordStatusTransition(db, "queued", "cancelled")
|
|
669
|
-
|
|
670
|
-
return {jobId, outcome: "cancelled"}
|
|
671
|
-
}
|
|
672
|
-
}
|
|
647
|
+
const jobId = String(/** @type {Record<string, ReturnType<typeof JSON.parse>>} */ (ownerRows[0]).job_id)
|
|
648
|
+
const job = await this._getJobRowById(db, jobId)
|
|
673
649
|
|
|
674
|
-
|
|
650
|
+
if (job?.status === "queued") {
|
|
651
|
+
const affectedRows = await this._updateAffectedRows(db, {
|
|
652
|
+
tableName: JOBS_TABLE,
|
|
653
|
+
data: {status: "cancelled"},
|
|
654
|
+
conditions: {id: job.id, status: "queued"}
|
|
655
|
+
})
|
|
675
656
|
|
|
657
|
+
if (affectedRows === 1) {
|
|
676
658
|
await this._releaseScheduleOwnership(db, {jobId, scheduleKey: normalizedScheduleKey})
|
|
659
|
+
await this._recordStatusTransition(db, "queued", "cancelled")
|
|
677
660
|
|
|
678
|
-
|
|
661
|
+
return {jobId, outcome: "cancelled"}
|
|
662
|
+
}
|
|
663
|
+
}
|
|
679
664
|
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
665
|
+
const currentJob = await this._getJobRowById(db, jobId)
|
|
666
|
+
|
|
667
|
+
await this._releaseScheduleOwnership(db, {jobId, scheduleKey: normalizedScheduleKey})
|
|
668
|
+
|
|
669
|
+
if (currentJob?.status === "handed_off") return {jobId, outcome: "handed_off"}
|
|
670
|
+
return {jobId: null, outcome: "not_found"}
|
|
671
|
+
}, {
|
|
672
|
+
advisoryLock: {
|
|
673
|
+
failureMessage: "Failed to acquire background job schedule-key lock",
|
|
674
|
+
name: this._scheduleKeyLockName(normalizedScheduleKey)
|
|
684
675
|
}
|
|
685
676
|
})
|
|
686
677
|
}
|
|
@@ -865,9 +856,9 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
|
|
|
865
856
|
async countSnapshot() {
|
|
866
857
|
await this.ensureReady()
|
|
867
858
|
|
|
868
|
-
return await this.
|
|
859
|
+
return await this._serializedCountMutation(async (db) => {
|
|
869
860
|
return await this._countSnapshotOnLockedConnection(db)
|
|
870
|
-
})
|
|
861
|
+
})
|
|
871
862
|
}
|
|
872
863
|
|
|
873
864
|
/**
|
|
@@ -938,7 +929,7 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
|
|
|
938
929
|
|
|
939
930
|
const handedOffAtMs = Date.now()
|
|
940
931
|
|
|
941
|
-
return await this.
|
|
932
|
+
return await this._serializedCountMutation(async (db) => {
|
|
942
933
|
const queuedJob = await this._getJobRowById(db, jobId)
|
|
943
934
|
if (!queuedJob || queuedJob.status !== "queued") return null
|
|
944
935
|
if (queuedJob.concurrencyKey && !(await this._reserveConcurrency(db, queuedJob.concurrencyKey))) return null
|
|
@@ -960,7 +951,7 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
|
|
|
960
951
|
|
|
961
952
|
await this._recordStatusTransition(db, "queued", "handed_off")
|
|
962
953
|
return {handedOffAtMs, handoffId}
|
|
963
|
-
})
|
|
954
|
+
})
|
|
964
955
|
}
|
|
965
956
|
|
|
966
957
|
/**
|
|
@@ -975,7 +966,7 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
|
|
|
975
966
|
async markCompleted({jobId, handoffId, workerId, handedOffAtMs}) {
|
|
976
967
|
await this.ensureReady()
|
|
977
968
|
|
|
978
|
-
return await this.
|
|
969
|
+
return await this._serializedCountMutation(async (db) => {
|
|
979
970
|
const job = await this._getJobRowById(db, jobId)
|
|
980
971
|
|
|
981
972
|
if (!job) return false
|
|
@@ -996,7 +987,7 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
|
|
|
996
987
|
await this._releaseConcurrency(db, job.concurrencyKey)
|
|
997
988
|
await this._recordStatusTransition(db, "handed_off", "completed")
|
|
998
989
|
return true
|
|
999
|
-
})
|
|
990
|
+
})
|
|
1000
991
|
}
|
|
1001
992
|
|
|
1002
993
|
/**
|
|
@@ -1014,7 +1005,7 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
|
|
|
1014
1005
|
await this.ensureReady()
|
|
1015
1006
|
this._validateRescheduleDelayMs(delayMs)
|
|
1016
1007
|
|
|
1017
|
-
return await this.
|
|
1008
|
+
return await this._serializedCountMutation(async (db) => {
|
|
1018
1009
|
const job = await this._getJobRowById(db, jobId)
|
|
1019
1010
|
|
|
1020
1011
|
if (!job) return false
|
|
@@ -1038,7 +1029,7 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
|
|
|
1038
1029
|
await this._releaseConcurrency(db, job.concurrencyKey)
|
|
1039
1030
|
await this._recordStatusTransition(db, "handed_off", "queued")
|
|
1040
1031
|
return true
|
|
1041
|
-
})
|
|
1032
|
+
})
|
|
1042
1033
|
}
|
|
1043
1034
|
|
|
1044
1035
|
/**
|
|
@@ -1051,7 +1042,7 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
|
|
|
1051
1042
|
async markReturnedToQueue({jobId, handoffId}) {
|
|
1052
1043
|
await this.ensureReady()
|
|
1053
1044
|
|
|
1054
|
-
await this.
|
|
1045
|
+
await this._serializedCountMutation(async (db) => {
|
|
1055
1046
|
const job = await this._getJobRowById(db, jobId)
|
|
1056
1047
|
if (!job || job.handoffId !== handoffId || job.status !== "handed_off") return
|
|
1057
1048
|
await this._lockConcurrencyRow(db, job.concurrencyKey)
|
|
@@ -1070,7 +1061,7 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
|
|
|
1070
1061
|
await this._releaseConcurrency(db, job.concurrencyKey)
|
|
1071
1062
|
await this._recordStatusTransition(db, "handed_off", "queued")
|
|
1072
1063
|
}
|
|
1073
|
-
})
|
|
1064
|
+
})
|
|
1074
1065
|
}
|
|
1075
1066
|
|
|
1076
1067
|
/**
|
|
@@ -1118,7 +1109,7 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
|
|
|
1118
1109
|
async markFailed({jobId, error, handoffId, workerId, handedOffAtMs}) {
|
|
1119
1110
|
await this.ensureReady()
|
|
1120
1111
|
|
|
1121
|
-
return await this.
|
|
1112
|
+
return await this._serializedCountMutation(async (db) => {
|
|
1122
1113
|
const job = await this._getJobRowById(db, jobId)
|
|
1123
1114
|
|
|
1124
1115
|
if (!job) return null
|
|
@@ -1128,7 +1119,7 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
|
|
|
1128
1119
|
|
|
1129
1120
|
if (updatedJob) await this._recordStatusTransition(db, job.status, updatedJob.status)
|
|
1130
1121
|
return updatedJob
|
|
1131
|
-
})
|
|
1122
|
+
})
|
|
1132
1123
|
}
|
|
1133
1124
|
|
|
1134
1125
|
/**
|
|
@@ -1140,7 +1131,7 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
|
|
|
1140
1131
|
async markOrphanedJobs({orphanedAfterMs = ORPHANED_AFTER_MS} = {}) {
|
|
1141
1132
|
await this.ensureReady()
|
|
1142
1133
|
|
|
1143
|
-
return await this.
|
|
1134
|
+
return await this._serializedCountMutation(async (db) => {
|
|
1144
1135
|
const cutoff = Date.now() - orphanedAfterMs
|
|
1145
1136
|
const query = db
|
|
1146
1137
|
.newQuery()
|
|
@@ -1190,7 +1181,7 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
|
|
|
1190
1181
|
await this._recordCountDelta(db, deltas)
|
|
1191
1182
|
|
|
1192
1183
|
return orphanedJobs
|
|
1193
|
-
})
|
|
1184
|
+
})
|
|
1194
1185
|
}
|
|
1195
1186
|
|
|
1196
1187
|
/**
|
|
@@ -1239,7 +1230,7 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
|
|
|
1239
1230
|
let deleted = 0
|
|
1240
1231
|
|
|
1241
1232
|
for (;;) {
|
|
1242
|
-
const removed = await this.
|
|
1233
|
+
const removed = await this._serializedCountMutation(async (db) => {
|
|
1243
1234
|
const rows = await db
|
|
1244
1235
|
.newQuery()
|
|
1245
1236
|
.from(JOBS_TABLE)
|
|
@@ -1260,7 +1251,7 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
|
|
|
1260
1251
|
await this._recordCountDelta(db, {all: -removed, [status]: -removed})
|
|
1261
1252
|
|
|
1262
1253
|
return removed
|
|
1263
|
-
})
|
|
1254
|
+
})
|
|
1264
1255
|
|
|
1265
1256
|
deleted += removed
|
|
1266
1257
|
if (removed < batchSize) break
|
|
@@ -1276,7 +1267,7 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
|
|
|
1276
1267
|
async clearAll() {
|
|
1277
1268
|
await this.ensureReady()
|
|
1278
1269
|
|
|
1279
|
-
await this.
|
|
1270
|
+
await this._serializedCountMutation(async (db) => {
|
|
1280
1271
|
const snapshot = await this._countSnapshotOnLockedConnection(db)
|
|
1281
1272
|
if (await db.tableExists(MAIL_DELIVERY_OPERATIONS_TABLE)) await db.query(`DELETE FROM ${db.quoteTable(MAIL_DELIVERY_OPERATIONS_TABLE)}`)
|
|
1282
1273
|
if (await db.tableExists(IDEMPOTENCY_KEYS_TABLE)) await db.query(`DELETE FROM ${db.quoteTable(IDEMPOTENCY_KEYS_TABLE)}`)
|
|
@@ -1285,7 +1276,7 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
|
|
|
1285
1276
|
if (await db.tableExists(CONCURRENCY_TABLE)) await db.query(`DELETE FROM ${db.quoteTable(CONCURRENCY_TABLE)}`)
|
|
1286
1277
|
const deltas = Object.fromEntries(Object.entries(snapshot.counts).map(([key, value]) => [key, -value]))
|
|
1287
1278
|
await this._recordCountDelta(db, deltas)
|
|
1288
|
-
})
|
|
1279
|
+
})
|
|
1289
1280
|
}
|
|
1290
1281
|
|
|
1291
1282
|
/**
|
|
@@ -1295,7 +1286,7 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
|
|
|
1295
1286
|
*/
|
|
1296
1287
|
async cancel(jobId) {
|
|
1297
1288
|
await this.ensureReady()
|
|
1298
|
-
return await this.
|
|
1289
|
+
return await this._serializedCountMutation(async (db) => {
|
|
1299
1290
|
const job = await this._getJobRowById(db, jobId)
|
|
1300
1291
|
if (!job || (job.status !== "queued" && job.status !== "handed_off")) return false
|
|
1301
1292
|
// Only a handed_off job holds a concurrency reservation, so only that case touches the
|
|
@@ -1307,7 +1298,7 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
|
|
|
1307
1298
|
if (job.status === "handed_off") await this._releaseConcurrency(db, job.concurrencyKey)
|
|
1308
1299
|
await this._recordStatusTransition(db, job.status, "cancelled")
|
|
1309
1300
|
return true
|
|
1310
|
-
})
|
|
1301
|
+
})
|
|
1311
1302
|
}
|
|
1312
1303
|
|
|
1313
1304
|
/**
|
|
@@ -2798,33 +2789,33 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
|
|
|
2798
2789
|
}
|
|
2799
2790
|
|
|
2800
2791
|
/**
|
|
2801
|
-
* Serializes count-changing transactions
|
|
2792
|
+
* Serializes count-changing transactions before checking out their connection.
|
|
2802
2793
|
* Database row locking still provides cross-process ordering; this guard
|
|
2803
2794
|
* prevents concurrent callers on SQLite's shared connection from attempting
|
|
2804
2795
|
* overlapping top-level transactions.
|
|
2805
2796
|
* @template T
|
|
2806
|
-
* @param {import("../database/drivers/base.js").default}
|
|
2807
|
-
* @param {
|
|
2797
|
+
* @param {(db: import("../database/drivers/base.js").default) => Promise<T>} callback - Transaction callback.
|
|
2798
|
+
* @param {BackgroundJobTransactionSerializationOptions} [options] - Serialization options.
|
|
2808
2799
|
* @returns {Promise<T>} Callback result.
|
|
2809
2800
|
*/
|
|
2810
|
-
async _serializedCountMutation(
|
|
2811
|
-
return await this._serializedTransactionMutation(
|
|
2801
|
+
async _serializedCountMutation(callback, options = {}) {
|
|
2802
|
+
return await this._serializedTransactionMutation(async (db) => {
|
|
2812
2803
|
await this._lockCountRevision(db)
|
|
2813
2804
|
|
|
2814
|
-
return await callback()
|
|
2815
|
-
})
|
|
2805
|
+
return await callback(db)
|
|
2806
|
+
}, options)
|
|
2816
2807
|
}
|
|
2817
2808
|
|
|
2818
2809
|
/**
|
|
2819
|
-
*
|
|
2820
|
-
*
|
|
2821
|
-
* locks and unique constraints acquired
|
|
2810
|
+
* Admits transactions to the process-local FIFO before they check out a
|
|
2811
|
+
* connection. Cross-process ordering remains the responsibility of durable
|
|
2812
|
+
* row/advisory locks and unique constraints acquired around the callback.
|
|
2822
2813
|
* @template T
|
|
2823
|
-
* @param {import("../database/drivers/base.js").default}
|
|
2824
|
-
* @param {
|
|
2814
|
+
* @param {(db: import("../database/drivers/base.js").default) => Promise<T>} callback - Transaction callback.
|
|
2815
|
+
* @param {BackgroundJobTransactionSerializationOptions} [options] - Serialization options.
|
|
2825
2816
|
* @returns {Promise<T>} Callback result.
|
|
2826
2817
|
*/
|
|
2827
|
-
async _serializedTransactionMutation(
|
|
2818
|
+
async _serializedTransactionMutation(callback, options = {}) {
|
|
2828
2819
|
const identifier = this.getDatabaseIdentifier() || "default"
|
|
2829
2820
|
const previous = transactionMutationChains.get(identifier) || Promise.resolve()
|
|
2830
2821
|
let resolveRun = () => {}
|
|
@@ -2838,7 +2829,21 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
|
|
|
2838
2829
|
await previous
|
|
2839
2830
|
|
|
2840
2831
|
try {
|
|
2841
|
-
return await this.
|
|
2832
|
+
return await this._withDb(async (db) => {
|
|
2833
|
+
const {advisoryLock} = options
|
|
2834
|
+
|
|
2835
|
+
if (advisoryLock) {
|
|
2836
|
+
const acquired = await db.acquireAdvisoryLock(advisoryLock.name)
|
|
2837
|
+
|
|
2838
|
+
if (!acquired) throw new Error(advisoryLock.failureMessage)
|
|
2839
|
+
}
|
|
2840
|
+
|
|
2841
|
+
try {
|
|
2842
|
+
return await this._transactionResult(db, async () => await callback(db))
|
|
2843
|
+
} finally {
|
|
2844
|
+
if (advisoryLock) await db.releaseAdvisoryLock(advisoryLock.name)
|
|
2845
|
+
}
|
|
2846
|
+
})
|
|
2842
2847
|
} finally {
|
|
2843
2848
|
resolveRun()
|
|
2844
2849
|
if (transactionMutationChains.get(identifier) === chain) transactionMutationChains.delete(identifier)
|
|
@@ -192,6 +192,10 @@ export default class BackgroundJobsWorker {
|
|
|
192
192
|
this.inflightProcessChildren = new Set()
|
|
193
193
|
/** @type {Set<Promise<void>>} */
|
|
194
194
|
this.inflightPooledJobs = new Set()
|
|
195
|
+
/** @type {Map<string, Array<import("./types.js").BackgroundJobPayload & {id: string}>>} */
|
|
196
|
+
this.pooledJobQueues = new Map()
|
|
197
|
+
/** @type {Map<string, Promise<void>>} - Per-id outer queue trackers. */
|
|
198
|
+
this.pooledJobQueueTrackers = new Map()
|
|
195
199
|
/** @type {Set<import("node:child_process").ChildProcess>} */
|
|
196
200
|
this.pooledChildren = new Set()
|
|
197
201
|
/** @type {Map<import("node:child_process").ChildProcess, {createdAtMs: number, jobsRun: number, inflight: Map<string, {payload: import("./types.js").BackgroundJobPayload & {id: string}, resolve?: (value: void) => void, pooledJob?: Promise<void>, timeoutTimer?: ReturnType<typeof setTimeout> | null}>, lastDispatchSeq: number, retiring: boolean, started?: boolean, settling?: boolean, timeoutSigkillTimer?: ReturnType<typeof setTimeout> | null}>} */
|
|
@@ -443,7 +447,7 @@ export default class BackgroundJobsWorker {
|
|
|
443
447
|
const executionMode = this._executionModeForPayload(identifiedPayload)
|
|
444
448
|
|
|
445
449
|
if (executionMode === "pooled") {
|
|
446
|
-
this.
|
|
450
|
+
this._queuePooledJob(identifiedPayload)
|
|
447
451
|
return
|
|
448
452
|
}
|
|
449
453
|
|
|
@@ -644,16 +648,60 @@ export default class BackgroundJobsWorker {
|
|
|
644
648
|
/**
|
|
645
649
|
* Tracks a pooled job and re-advertises capacity.
|
|
646
650
|
* @param {Promise<void>} pooledJob - Pooled job promise.
|
|
647
|
-
* @returns {void}
|
|
651
|
+
* @returns {Promise<void>} - The tracked in-flight promise.
|
|
648
652
|
*/
|
|
649
653
|
_trackPooledJob(pooledJob) {
|
|
650
654
|
/** @type {Promise<void>} */
|
|
651
655
|
let inflight
|
|
652
656
|
inflight = pooledJob.finally(() => {
|
|
653
657
|
this.inflightPooledJobs.delete(inflight)
|
|
654
|
-
if (!this.shouldStop && !this._pooledStartupFailureJobs.has(pooledJob)) this._sendReadyIfRunning()
|
|
658
|
+
if (!this.shouldStop && !this._pooledStartupFailureJobs.has(pooledJob) && !this._pooledStartupFailureJobs.has(inflight)) this._sendReadyIfRunning()
|
|
655
659
|
})
|
|
656
660
|
this.inflightPooledJobs.add(inflight)
|
|
661
|
+
return inflight
|
|
662
|
+
}
|
|
663
|
+
|
|
664
|
+
/**
|
|
665
|
+
* Serializes repeated leases for one durable row while preserving pooled
|
|
666
|
+
* concurrency across different job ids.
|
|
667
|
+
* @param {import("./types.js").BackgroundJobPayload & {id: string}} payload - Pooled job payload.
|
|
668
|
+
* @returns {void}
|
|
669
|
+
*/
|
|
670
|
+
_queuePooledJob(payload) {
|
|
671
|
+
const queue = this.pooledJobQueues.get(payload.id)
|
|
672
|
+
if (queue) {
|
|
673
|
+
queue.push(payload)
|
|
674
|
+
return
|
|
675
|
+
}
|
|
676
|
+
|
|
677
|
+
this.pooledJobQueues.set(payload.id, [payload])
|
|
678
|
+
const tracker = this._trackPooledJob(this._runPooledJobQueue(payload.id))
|
|
679
|
+
this.pooledJobQueueTrackers.set(payload.id, tracker)
|
|
680
|
+
}
|
|
681
|
+
|
|
682
|
+
/**
|
|
683
|
+
* Runs admitted leases for one durable job id in arrival order.
|
|
684
|
+
* @param {string} jobId - Durable job id.
|
|
685
|
+
* @returns {Promise<void>} - Resolves after the per-id queue drains.
|
|
686
|
+
*/
|
|
687
|
+
async _runPooledJobQueue(jobId) {
|
|
688
|
+
const queue = this.pooledJobQueues.get(jobId)
|
|
689
|
+
if (!queue) throw new Error(`Pooled job queue missing for job: ${jobId}`)
|
|
690
|
+
|
|
691
|
+
try {
|
|
692
|
+
while (queue.length > 0) {
|
|
693
|
+
const payload = queue.shift()
|
|
694
|
+
if (!payload) throw new Error(`Pooled job queue contained an empty payload for job: ${jobId}`)
|
|
695
|
+
await this._runPooledJob(payload)
|
|
696
|
+
}
|
|
697
|
+
} finally {
|
|
698
|
+
const tracker = this.pooledJobQueueTrackers.get(jobId)
|
|
699
|
+
if (tracker) {
|
|
700
|
+
this.inflightPooledJobs.delete(tracker)
|
|
701
|
+
this.pooledJobQueueTrackers.delete(jobId)
|
|
702
|
+
}
|
|
703
|
+
this.pooledJobQueues.delete(jobId)
|
|
704
|
+
}
|
|
657
705
|
}
|
|
658
706
|
|
|
659
707
|
/**
|
|
@@ -665,6 +713,7 @@ export default class BackgroundJobsWorker {
|
|
|
665
713
|
_availablePooledSlots() {
|
|
666
714
|
let openInExisting = 0
|
|
667
715
|
let nonRetiringChildren = 0
|
|
716
|
+
let queuedReservations = 0
|
|
668
717
|
|
|
669
718
|
for (const child of this.pooledChildren) {
|
|
670
719
|
const state = this.pooledChildStates.get(child)
|
|
@@ -673,9 +722,11 @@ export default class BackgroundJobsWorker {
|
|
|
673
722
|
openInExisting += this.pooledRunnerConcurrency - state.inflight.size
|
|
674
723
|
}
|
|
675
724
|
|
|
725
|
+
for (const queue of this.pooledJobQueues.values()) queuedReservations += queue.length
|
|
726
|
+
|
|
676
727
|
const spawnableChildren = Math.max(0, this.pooledRunnerCount - nonRetiringChildren)
|
|
677
728
|
|
|
678
|
-
return openInExisting + spawnableChildren * this.pooledRunnerConcurrency
|
|
729
|
+
return Math.max(0, openInExisting + spawnableChildren * this.pooledRunnerConcurrency - queuedReservations)
|
|
679
730
|
}
|
|
680
731
|
|
|
681
732
|
/**
|
|
@@ -974,6 +1025,8 @@ export default class BackgroundJobsWorker {
|
|
|
974
1025
|
} else if (state) {
|
|
975
1026
|
for (const entry of entries) {
|
|
976
1027
|
if (entry.pooledJob) this._pooledStartupFailureJobs.add(entry.pooledJob)
|
|
1028
|
+
const queueTracker = this.pooledJobQueueTrackers.get(entry.payload.id)
|
|
1029
|
+
if (queueTracker) this._pooledStartupFailureJobs.add(queueTracker)
|
|
977
1030
|
}
|
|
978
1031
|
// A previous ready message may still have unconsumed pooled credits at the
|
|
979
1032
|
// main. Revoke them authoritatively without suppressing valid inline or
|