velocious 1.0.558 → 1.0.559
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/build/background-jobs/store.js +262 -16
- package/build/background-jobs/web/authorization.js +5 -4
- package/build/background-jobs/web/controller.js +13 -16
- package/build/background-jobs/web/counts-channel.js +79 -0
- package/build/background-jobs/web/index.js +2 -0
- package/build/background-jobs/web/registry.js +1 -1
- package/build/src/background-jobs/store.d.ts +79 -0
- package/build/src/background-jobs/store.d.ts.map +1 -1
- package/build/src/background-jobs/store.js +235 -17
- package/build/src/background-jobs/web/authorization.d.ts +5 -3
- package/build/src/background-jobs/web/authorization.d.ts.map +1 -1
- package/build/src/background-jobs/web/authorization.js +6 -5
- package/build/src/background-jobs/web/controller.d.ts.map +1 -1
- package/build/src/background-jobs/web/controller.js +14 -15
- package/build/src/background-jobs/web/counts-channel.d.ts +31 -0
- package/build/src/background-jobs/web/counts-channel.d.ts.map +1 -0
- package/build/src/background-jobs/web/counts-channel.js +71 -0
- package/build/src/background-jobs/web/index.d.ts.map +1 -1
- package/build/src/background-jobs/web/index.js +3 -1
- package/build/src/background-jobs/web/registry.d.ts +1 -1
- package/build/src/background-jobs/web/registry.d.ts.map +1 -1
- package/build/src/background-jobs/web/registry.js +2 -2
- package/build/tsconfig.tsbuildinfo +1 -1
- package/package.json +1 -1
- package/src/background-jobs/store.js +262 -16
- package/src/background-jobs/web/authorization.js +5 -4
- package/src/background-jobs/web/controller.js +13 -16
- package/src/background-jobs/web/counts-channel.js +79 -0
- package/src/background-jobs/web/index.js +2 -0
- package/src/background-jobs/web/registry.js +1 -1
package/README.md
CHANGED
|
@@ -26,7 +26,7 @@
|
|
|
26
26
|
* Translated model attributes with current-locale relationship sorting (see [docs/translations.md](docs/translations.md))
|
|
27
27
|
* Cross-process broadcast bus for `broadcastToChannel` via `velocious beacon`, including background job runner processes (see [docs/beacon.md](docs/beacon.md))
|
|
28
28
|
* Configurable HTTP server worker handlers plus backpressured, descriptor-only file responses with completion callbacks (see [docs/http-server.md](docs/http-server.md))
|
|
29
|
-
* Background jobs with failure events for production reporting (see [docs/background-jobs.md](docs/background-jobs.md))
|
|
29
|
+
* Background jobs with failure events for production reporting and authorized database-scoped dashboard count snapshots/deltas (see [docs/background-jobs.md](docs/background-jobs.md) and [docs/background-jobs-dashboard.md](docs/background-jobs-dashboard.md))
|
|
30
30
|
* Durable one-off background-job scheduling with exact epoch timestamps (see [docs/scheduled-background-job-enqueue.md](docs/scheduled-background-job-enqueue.md))
|
|
31
31
|
* Rails-style request and database query logging (see [docs/logging.md](docs/logging.md))
|
|
32
32
|
* EJS-backed mailers with delivery, queueing, and payload rendering support (see [docs/mailers.md](docs/mailers.md))
|
|
@@ -23,6 +23,11 @@ const LEGACY_POOLED_HANDOFF_ID_PREFIX = "velocious-pooled:"
|
|
|
23
23
|
const LEGACY_POOLED_QUEUED_HANDOFF_ID = `${LEGACY_POOLED_HANDOFF_ID_PREFIX}queued`
|
|
24
24
|
const JOBS_TABLE = "background_jobs"
|
|
25
25
|
const CONCURRENCY_TABLE = "background_job_concurrency"
|
|
26
|
+
const COUNTS_REVISION_TABLE = "background_job_count_revisions"
|
|
27
|
+
const COUNTS_REVISION_KEY = "counts"
|
|
28
|
+
export const BACKGROUND_JOB_COUNTS_CHANNEL = "velocious-background-job-counts"
|
|
29
|
+
export const BACKGROUND_JOB_COUNT_BUCKETS = ["all", "queued", "handed_off", "completed", "failed", "orphaned"]
|
|
30
|
+
const COUNTED_JOB_STATUSES = BACKGROUND_JOB_COUNT_BUCKETS.slice(1)
|
|
26
31
|
const DEFAULT_MAX_RETRIES = 10
|
|
27
32
|
const ORPHANED_AFTER_MS = 2 * 60 * 60 * 1000
|
|
28
33
|
/**
|
|
@@ -68,6 +73,8 @@ const SORTABLE_COLUMNS = {
|
|
|
68
73
|
* @type {Map<string, Promise<void>>}
|
|
69
74
|
*/
|
|
70
75
|
const schemaApplyChains = new Map()
|
|
76
|
+
/** @type {Map<string, Promise<void>>} */
|
|
77
|
+
const countMutationChains = new Map()
|
|
71
78
|
|
|
72
79
|
export default class BackgroundJobsStore {
|
|
73
80
|
/**
|
|
@@ -157,7 +164,7 @@ export default class BackgroundJobsStore {
|
|
|
157
164
|
/** @type {string} */
|
|
158
165
|
let resultJobId = jobId
|
|
159
166
|
|
|
160
|
-
await this._withDb(async (db) => {
|
|
167
|
+
await this._withDb(async (db) => await this._serializedCountMutation(db, async () => {
|
|
161
168
|
if (options?.deduplicateWhileQueued) {
|
|
162
169
|
// Dedupe on the job's identity (name + args + queue), NOT its concurrency key, so a job
|
|
163
170
|
// keeps whatever concurrency it resolves to. Only an existing job scheduled no later than
|
|
@@ -205,7 +212,8 @@ export default class BackgroundJobsStore {
|
|
|
205
212
|
handoff_id: null
|
|
206
213
|
}
|
|
207
214
|
})
|
|
208
|
-
|
|
215
|
+
await this._recordCountDelta(db, {all: 1, queued: 1})
|
|
216
|
+
}))
|
|
209
217
|
|
|
210
218
|
return resultJobId
|
|
211
219
|
}
|
|
@@ -381,6 +389,20 @@ export default class BackgroundJobsStore {
|
|
|
381
389
|
})
|
|
382
390
|
}
|
|
383
391
|
|
|
392
|
+
/**
|
|
393
|
+
* Returns the authoritative dashboard count snapshot and its matching durable
|
|
394
|
+
* revision. Locking the revision row before counting prevents a writer from
|
|
395
|
+
* committing between the count query and revision read.
|
|
396
|
+
* @returns {Promise<{counts: Record<string, number>, revision: number, total: number}>} Snapshot.
|
|
397
|
+
*/
|
|
398
|
+
async countSnapshot() {
|
|
399
|
+
await this.ensureReady()
|
|
400
|
+
|
|
401
|
+
return await this._withDb(async (db) => await this._serializedCountMutation(db, async () => {
|
|
402
|
+
return await this._countSnapshotOnLockedConnection(db)
|
|
403
|
+
}))
|
|
404
|
+
}
|
|
405
|
+
|
|
384
406
|
/**
|
|
385
407
|
* Counts jobs matching the given filters.
|
|
386
408
|
* @param {object} [args] - Options.
|
|
@@ -448,7 +470,7 @@ export default class BackgroundJobsStore {
|
|
|
448
470
|
|
|
449
471
|
const handedOffAtMs = Date.now()
|
|
450
472
|
|
|
451
|
-
return await this._withDb(async (db) => await this.
|
|
473
|
+
return await this._withDb(async (db) => await this._serializedCountMutation(db, async () => {
|
|
452
474
|
const queuedJob = await this._getJobRowById(db, jobId)
|
|
453
475
|
if (!queuedJob || queuedJob.status !== "queued") return null
|
|
454
476
|
if (queuedJob.concurrencyKey && !(await this._reserveConcurrency(db, queuedJob.concurrencyKey))) return null
|
|
@@ -470,6 +492,7 @@ export default class BackgroundJobsStore {
|
|
|
470
492
|
return null
|
|
471
493
|
}
|
|
472
494
|
|
|
495
|
+
await this._recordStatusTransition(db, "queued", "handed_off")
|
|
473
496
|
return {handedOffAtMs, handoffId}
|
|
474
497
|
}))
|
|
475
498
|
}
|
|
@@ -486,7 +509,7 @@ export default class BackgroundJobsStore {
|
|
|
486
509
|
async markCompleted({jobId, handoffId, workerId, handedOffAtMs}) {
|
|
487
510
|
await this.ensureReady()
|
|
488
511
|
|
|
489
|
-
return await this._withDb(async (db) => await this.
|
|
512
|
+
return await this._withDb(async (db) => await this._serializedCountMutation(db, async () => {
|
|
490
513
|
const job = await this._getJobRowById(db, jobId)
|
|
491
514
|
|
|
492
515
|
if (!job) return false
|
|
@@ -504,6 +527,7 @@ export default class BackgroundJobsStore {
|
|
|
504
527
|
|
|
505
528
|
if (affectedRows !== 1) return false
|
|
506
529
|
await this._releaseConcurrency(db, job.concurrencyKey)
|
|
530
|
+
await this._recordStatusTransition(db, "handed_off", "completed")
|
|
507
531
|
return true
|
|
508
532
|
}))
|
|
509
533
|
}
|
|
@@ -518,7 +542,7 @@ export default class BackgroundJobsStore {
|
|
|
518
542
|
async markReturnedToQueue({jobId, handoffId}) {
|
|
519
543
|
await this.ensureReady()
|
|
520
544
|
|
|
521
|
-
await this._withDb(async (db) => await
|
|
545
|
+
await this._withDb(async (db) => await this._serializedCountMutation(db, async () => {
|
|
522
546
|
const job = await this._getJobRowById(db, jobId)
|
|
523
547
|
if (!job || job.handoffId !== handoffId || job.status !== "handed_off") return
|
|
524
548
|
await this._lockConcurrencyRow(db, job.concurrencyKey)
|
|
@@ -533,7 +557,10 @@ export default class BackgroundJobsStore {
|
|
|
533
557
|
},
|
|
534
558
|
conditions: {handoff_id: handoffId, id: jobId, status: "handed_off"}
|
|
535
559
|
})
|
|
536
|
-
if (affectedRows === 1)
|
|
560
|
+
if (affectedRows === 1) {
|
|
561
|
+
await this._releaseConcurrency(db, job.concurrencyKey)
|
|
562
|
+
await this._recordStatusTransition(db, "handed_off", "queued")
|
|
563
|
+
}
|
|
537
564
|
}))
|
|
538
565
|
}
|
|
539
566
|
|
|
@@ -582,13 +609,15 @@ export default class BackgroundJobsStore {
|
|
|
582
609
|
async markFailed({jobId, error, handoffId, workerId, handedOffAtMs}) {
|
|
583
610
|
await this.ensureReady()
|
|
584
611
|
|
|
585
|
-
return await this._withDb(async (db) => await this.
|
|
612
|
+
return await this._withDb(async (db) => await this._serializedCountMutation(db, async () => {
|
|
586
613
|
const job = await this._getJobRowById(db, jobId)
|
|
587
614
|
|
|
588
615
|
if (!job) return null
|
|
589
616
|
if (!this._shouldAcceptReport({job, handoffId, workerId, handedOffAtMs})) return null
|
|
590
617
|
|
|
591
618
|
const updatedJob = await this._applyFailure({db, job, error, markOrphaned: false})
|
|
619
|
+
|
|
620
|
+
if (updatedJob) await this._recordStatusTransition(db, job.status, updatedJob.status)
|
|
592
621
|
return updatedJob
|
|
593
622
|
}))
|
|
594
623
|
}
|
|
@@ -602,7 +631,7 @@ export default class BackgroundJobsStore {
|
|
|
602
631
|
async markOrphanedJobs({orphanedAfterMs = ORPHANED_AFTER_MS} = {}) {
|
|
603
632
|
await this.ensureReady()
|
|
604
633
|
|
|
605
|
-
return await this._withDb(async (db) => {
|
|
634
|
+
return await this._withDb(async (db) => await this._serializedCountMutation(db, async () => {
|
|
606
635
|
const cutoff = Date.now() - orphanedAfterMs
|
|
607
636
|
const query = db
|
|
608
637
|
.newQuery()
|
|
@@ -642,8 +671,17 @@ export default class BackgroundJobsStore {
|
|
|
642
671
|
if (orphanedJob) orphanedJobs.push(orphanedJob)
|
|
643
672
|
}
|
|
644
673
|
|
|
674
|
+
const statusCounts = this._statusCounts(orphanedJobs)
|
|
675
|
+
const deltas = this._emptyCountBuckets()
|
|
676
|
+
|
|
677
|
+
for (const [status, count] of Object.entries(statusCounts)) {
|
|
678
|
+
deltas.handed_off -= count
|
|
679
|
+
deltas[status] += count
|
|
680
|
+
}
|
|
681
|
+
await this._recordCountDelta(db, deltas)
|
|
682
|
+
|
|
645
683
|
return orphanedJobs
|
|
646
|
-
})
|
|
684
|
+
}))
|
|
647
685
|
}
|
|
648
686
|
|
|
649
687
|
/**
|
|
@@ -692,7 +730,7 @@ export default class BackgroundJobsStore {
|
|
|
692
730
|
let deleted = 0
|
|
693
731
|
|
|
694
732
|
for (;;) {
|
|
695
|
-
const removed = await this._withDb(async (db) => {
|
|
733
|
+
const removed = await this._withDb(async (db) => await this._serializedCountMutation(db, async () => {
|
|
696
734
|
const rows = await db
|
|
697
735
|
.newQuery()
|
|
698
736
|
.from(JOBS_TABLE)
|
|
@@ -706,10 +744,14 @@ export default class BackgroundJobsStore {
|
|
|
706
744
|
|
|
707
745
|
const ids = rows.map((/** @type {Record<string, ?>} */ row) => db.quote(String(row.id))).join(", ")
|
|
708
746
|
|
|
709
|
-
|
|
747
|
+
const removed = await db.affectedRows(
|
|
748
|
+
`DELETE FROM ${db.quoteTable(JOBS_TABLE)} WHERE ${db.quoteColumn("id")} IN (${ids})`
|
|
749
|
+
)
|
|
710
750
|
|
|
711
|
-
|
|
712
|
-
|
|
751
|
+
await this._recordCountDelta(db, {all: -removed, [status]: -removed})
|
|
752
|
+
|
|
753
|
+
return removed
|
|
754
|
+
}))
|
|
713
755
|
|
|
714
756
|
deleted += removed
|
|
715
757
|
if (removed < batchSize) break
|
|
@@ -725,10 +767,13 @@ export default class BackgroundJobsStore {
|
|
|
725
767
|
async clearAll() {
|
|
726
768
|
await this.ensureReady()
|
|
727
769
|
|
|
728
|
-
await this._withDb(async (db) => {
|
|
770
|
+
await this._withDb(async (db) => await this._serializedCountMutation(db, async () => {
|
|
771
|
+
const snapshot = await this._countSnapshotOnLockedConnection(db)
|
|
729
772
|
await db.query(`DELETE FROM ${db.quoteTable(JOBS_TABLE)}`)
|
|
730
773
|
if (await db.tableExists(CONCURRENCY_TABLE)) await db.query(`DELETE FROM ${db.quoteTable(CONCURRENCY_TABLE)}`)
|
|
731
|
-
|
|
774
|
+
const deltas = Object.fromEntries(Object.entries(snapshot.counts).map(([key, value]) => [key, -value]))
|
|
775
|
+
await this._recordCountDelta(db, deltas)
|
|
776
|
+
}))
|
|
732
777
|
}
|
|
733
778
|
|
|
734
779
|
/**
|
|
@@ -738,7 +783,7 @@ export default class BackgroundJobsStore {
|
|
|
738
783
|
*/
|
|
739
784
|
async cancel(jobId) {
|
|
740
785
|
await this.ensureReady()
|
|
741
|
-
return await this._withDb(async (db) => await this.
|
|
786
|
+
return await this._withDb(async (db) => await this._serializedCountMutation(db, async () => {
|
|
742
787
|
const job = await this._getJobRowById(db, jobId)
|
|
743
788
|
if (!job || (job.status !== "queued" && job.status !== "handed_off")) return false
|
|
744
789
|
// Only a handed_off job holds a concurrency reservation, so only that case touches the
|
|
@@ -747,6 +792,7 @@ export default class BackgroundJobsStore {
|
|
|
747
792
|
const affectedRows = await this._updateAffectedRows(db, {tableName: JOBS_TABLE, data: {status: "cancelled"}, conditions: {id: job.id, status: job.status}})
|
|
748
793
|
if (affectedRows !== 1) return false
|
|
749
794
|
if (job.status === "handed_off") await this._releaseConcurrency(db, job.concurrencyKey)
|
|
795
|
+
await this._recordStatusTransition(db, job.status, "cancelled")
|
|
750
796
|
return true
|
|
751
797
|
}))
|
|
752
798
|
}
|
|
@@ -855,6 +901,7 @@ export default class BackgroundJobsStore {
|
|
|
855
901
|
if (alreadyApplied && await db.tableExists(JOBS_TABLE)) {
|
|
856
902
|
await this._ensureJobsTableColumns(db)
|
|
857
903
|
await this._ensureConcurrencyTable(db)
|
|
904
|
+
await this._ensureCountRevisionTable(db)
|
|
858
905
|
await this._reconcileQueueConcurrency(db)
|
|
859
906
|
await this._reconcileConcurrency(db)
|
|
860
907
|
|
|
@@ -864,6 +911,7 @@ export default class BackgroundJobsStore {
|
|
|
864
911
|
await this._applyMigrations(db)
|
|
865
912
|
await this._ensureJobsTableColumns(db)
|
|
866
913
|
await this._ensureConcurrencyTable(db)
|
|
914
|
+
await this._ensureCountRevisionTable(db)
|
|
867
915
|
await this._reconcileQueueConcurrency(db)
|
|
868
916
|
await this._reconcileConcurrency(db)
|
|
869
917
|
|
|
@@ -1496,6 +1544,173 @@ export default class BackgroundJobsStore {
|
|
|
1496
1544
|
await db.createTable(table)
|
|
1497
1545
|
}
|
|
1498
1546
|
|
|
1547
|
+
/**
|
|
1548
|
+
* Ensures the singleton durable count-revision row exists.
|
|
1549
|
+
* @param {import("../database/drivers/base.js").default} db - Database connection.
|
|
1550
|
+
* @returns {Promise<void>} Resolves when ready.
|
|
1551
|
+
*/
|
|
1552
|
+
async _ensureCountRevisionTable(db) {
|
|
1553
|
+
if (!(await db.tableExists(COUNTS_REVISION_TABLE))) {
|
|
1554
|
+
const table = new TableData(COUNTS_REVISION_TABLE, {ifNotExists: true})
|
|
1555
|
+
|
|
1556
|
+
table.string("key", {primaryKey: true})
|
|
1557
|
+
table.bigint("revision", {null: false})
|
|
1558
|
+
await db.createTable(table)
|
|
1559
|
+
}
|
|
1560
|
+
|
|
1561
|
+
const rows = await db.newQuery().from(COUNTS_REVISION_TABLE).where({key: COUNTS_REVISION_KEY}).limit(1).results()
|
|
1562
|
+
|
|
1563
|
+
if (rows.length > 0) return
|
|
1564
|
+
|
|
1565
|
+
try {
|
|
1566
|
+
await db.insert({tableName: COUNTS_REVISION_TABLE, data: {key: COUNTS_REVISION_KEY, revision: 0}})
|
|
1567
|
+
} catch (error) {
|
|
1568
|
+
const racedRows = await db.newQuery().from(COUNTS_REVISION_TABLE).where({key: COUNTS_REVISION_KEY}).limit(1).results()
|
|
1569
|
+
|
|
1570
|
+
if (racedRows.length === 0) throw error
|
|
1571
|
+
}
|
|
1572
|
+
}
|
|
1573
|
+
|
|
1574
|
+
/**
|
|
1575
|
+
* Records one logical count mutation atomically and broadcasts it after commit.
|
|
1576
|
+
* Zero entries are omitted; a wholly zero-net mutation does not consume a revision.
|
|
1577
|
+
* @param {import("../database/drivers/base.js").default} db - Transaction connection.
|
|
1578
|
+
* @param {Record<string, number>} requestedDeltas - Signed bucket changes.
|
|
1579
|
+
* @returns {Promise<void>} Resolves when recorded.
|
|
1580
|
+
*/
|
|
1581
|
+
async _recordCountDelta(db, requestedDeltas) {
|
|
1582
|
+
/** @type {Record<string, number>} */
|
|
1583
|
+
const deltas = {}
|
|
1584
|
+
|
|
1585
|
+
for (const bucket of BACKGROUND_JOB_COUNT_BUCKETS) {
|
|
1586
|
+
const amount = requestedDeltas[bucket] || 0
|
|
1587
|
+
|
|
1588
|
+
if (!Number.isInteger(amount)) throw new Error(`Invalid background job count delta for ${bucket}: ${amount}`)
|
|
1589
|
+
if (amount !== 0) deltas[bucket] = amount
|
|
1590
|
+
}
|
|
1591
|
+
|
|
1592
|
+
if (Object.keys(deltas).length === 0) return
|
|
1593
|
+
|
|
1594
|
+
const table = db.quoteTable(COUNTS_REVISION_TABLE)
|
|
1595
|
+
const revisionColumn = db.quoteColumn("revision")
|
|
1596
|
+
const affectedRows = await db.affectedRows(
|
|
1597
|
+
`UPDATE ${table} SET ${revisionColumn} = ${revisionColumn} + 1 WHERE ${db.quoteColumn("key")} = ${db.quote(COUNTS_REVISION_KEY)}`
|
|
1598
|
+
)
|
|
1599
|
+
|
|
1600
|
+
if (affectedRows !== 1) throw new Error("Background job count revision row is missing")
|
|
1601
|
+
|
|
1602
|
+
const revision = await this._countRevision(db)
|
|
1603
|
+
const body = {deltas, revision, type: "background-job-count-delta"}
|
|
1604
|
+
const databaseIdentifier = this.getDatabaseIdentifier() || "default"
|
|
1605
|
+
|
|
1606
|
+
await db.afterCommit(() => {
|
|
1607
|
+
this.configuration.broadcastToChannel(BACKGROUND_JOB_COUNTS_CHANNEL, {databaseIdentifier}, body)
|
|
1608
|
+
})
|
|
1609
|
+
}
|
|
1610
|
+
|
|
1611
|
+
/**
|
|
1612
|
+
* Records a transition between persisted statuses.
|
|
1613
|
+
* @param {import("../database/drivers/base.js").default} db - Transaction connection.
|
|
1614
|
+
* @param {string} oldStatus - Previous status.
|
|
1615
|
+
* @param {string} newStatus - New status.
|
|
1616
|
+
* @returns {Promise<void>} Resolves when recorded.
|
|
1617
|
+
*/
|
|
1618
|
+
async _recordStatusTransition(db, oldStatus, newStatus) {
|
|
1619
|
+
const oldCounted = COUNTED_JOB_STATUSES.includes(oldStatus)
|
|
1620
|
+
const newCounted = COUNTED_JOB_STATUSES.includes(newStatus)
|
|
1621
|
+
|
|
1622
|
+
if (!oldCounted && oldStatus !== "cancelled") throw new Error(`Unknown previous background job status: ${oldStatus}`)
|
|
1623
|
+
if (!newCounted && newStatus !== "cancelled") throw new Error(`Unknown next background job status: ${newStatus}`)
|
|
1624
|
+
if (oldStatus === newStatus) return
|
|
1625
|
+
|
|
1626
|
+
/** @type {Record<string, number>} */
|
|
1627
|
+
const deltas = {}
|
|
1628
|
+
|
|
1629
|
+
if (oldCounted) deltas[oldStatus] = -1
|
|
1630
|
+
if (newCounted) deltas[newStatus] = 1
|
|
1631
|
+
if (oldCounted !== newCounted) deltas.all = newCounted ? 1 : -1
|
|
1632
|
+
await this._recordCountDelta(db, deltas)
|
|
1633
|
+
}
|
|
1634
|
+
|
|
1635
|
+
/**
|
|
1636
|
+
* Reads the locked revision.
|
|
1637
|
+
* @param {import("../database/drivers/base.js").default} db - Database connection.
|
|
1638
|
+
* @returns {Promise<number>} Revision.
|
|
1639
|
+
*/
|
|
1640
|
+
async _countRevision(db) {
|
|
1641
|
+
const rows = await db.newQuery().from(COUNTS_REVISION_TABLE).select("revision").where({key: COUNTS_REVISION_KEY}).limit(1).results()
|
|
1642
|
+
const revision = this._normalizeNumber(/** @type {Record<string, ?>} */ (rows[0] || {}).revision)
|
|
1643
|
+
|
|
1644
|
+
if (revision === null || !Number.isSafeInteger(revision) || revision < 0) {
|
|
1645
|
+
throw new Error(`Invalid background job count revision: ${revision}`)
|
|
1646
|
+
}
|
|
1647
|
+
|
|
1648
|
+
return revision
|
|
1649
|
+
}
|
|
1650
|
+
|
|
1651
|
+
/**
|
|
1652
|
+
* Takes a portable write lock on the singleton revision row.
|
|
1653
|
+
* @param {import("../database/drivers/base.js").default} db - Database connection.
|
|
1654
|
+
* @returns {Promise<void>} Resolves when locked.
|
|
1655
|
+
*/
|
|
1656
|
+
async _lockCountRevision(db) {
|
|
1657
|
+
const table = db.quoteTable(COUNTS_REVISION_TABLE)
|
|
1658
|
+
const revision = db.quoteColumn("revision")
|
|
1659
|
+
|
|
1660
|
+
await db.query(`UPDATE ${table} SET ${revision} = ${revision} WHERE ${db.quoteColumn("key")} = ${db.quote(COUNTS_REVISION_KEY)}`)
|
|
1661
|
+
}
|
|
1662
|
+
|
|
1663
|
+
/**
|
|
1664
|
+
* Builds zeroed canonical buckets.
|
|
1665
|
+
* @returns {Record<string, number>} Zeroed canonical buckets.
|
|
1666
|
+
*/
|
|
1667
|
+
_emptyCountBuckets() {
|
|
1668
|
+
return Object.fromEntries(BACKGROUND_JOB_COUNT_BUCKETS.map((bucket) => [bucket, 0]))
|
|
1669
|
+
}
|
|
1670
|
+
|
|
1671
|
+
/**
|
|
1672
|
+
* Counts normalized rows by canonical status.
|
|
1673
|
+
* @param {import("./types.js").BackgroundJobRow[]} jobs - Jobs.
|
|
1674
|
+
* @returns {Record<string, number>} Counts.
|
|
1675
|
+
*/
|
|
1676
|
+
_statusCounts(jobs) {
|
|
1677
|
+
/** @type {Record<string, number>} */
|
|
1678
|
+
const counts = {}
|
|
1679
|
+
|
|
1680
|
+
for (const job of jobs) {
|
|
1681
|
+
if (!COUNTED_JOB_STATUSES.includes(job.status)) throw new Error(`Unknown background job status: ${job.status}`)
|
|
1682
|
+
counts[job.status] = (counts[job.status] || 0) + 1
|
|
1683
|
+
}
|
|
1684
|
+
|
|
1685
|
+
return counts
|
|
1686
|
+
}
|
|
1687
|
+
|
|
1688
|
+
/**
|
|
1689
|
+
* Reads a canonical snapshot after locking the revision row.
|
|
1690
|
+
* @param {import("../database/drivers/base.js").default} db - Transaction connection.
|
|
1691
|
+
* @returns {Promise<{counts: Record<string, number>, revision: number, total: number}>} Snapshot.
|
|
1692
|
+
*/
|
|
1693
|
+
async _countSnapshotOnLockedConnection(db) {
|
|
1694
|
+
await this._lockCountRevision(db)
|
|
1695
|
+
const rows = await db.newQuery().from(JOBS_TABLE).select("status").select("COUNT(*) AS count").group("status").results()
|
|
1696
|
+
const counts = this._emptyCountBuckets()
|
|
1697
|
+
let total = 0
|
|
1698
|
+
|
|
1699
|
+
for (const row of rows) {
|
|
1700
|
+
const typedRow = /** @type {Record<string, ?>} */ (row)
|
|
1701
|
+
const status = String(typedRow.status)
|
|
1702
|
+
const count = this._normalizeNumber(typedRow.count) || 0
|
|
1703
|
+
|
|
1704
|
+
total += count
|
|
1705
|
+
|
|
1706
|
+
if (!COUNTED_JOB_STATUSES.includes(status)) continue
|
|
1707
|
+
counts[status] = count
|
|
1708
|
+
counts.all += counts[status]
|
|
1709
|
+
}
|
|
1710
|
+
|
|
1711
|
+
return {counts, revision: await this._countRevision(db), total}
|
|
1712
|
+
}
|
|
1713
|
+
|
|
1499
1714
|
/**
|
|
1500
1715
|
* Registers or verifies a stable key configuration.
|
|
1501
1716
|
* @param {import("../database/drivers/base.js").default} db - Database connection.
|
|
@@ -1785,6 +2000,37 @@ export default class BackgroundJobsStore {
|
|
|
1785
2000
|
return /** @type {T} */ (result)
|
|
1786
2001
|
}
|
|
1787
2002
|
|
|
2003
|
+
/**
|
|
2004
|
+
* Serializes count-changing transactions sharing a process-local connection.
|
|
2005
|
+
* Database row locking still provides cross-process ordering; this guard
|
|
2006
|
+
* prevents concurrent callers on SQLite's shared connection from attempting
|
|
2007
|
+
* overlapping top-level transactions.
|
|
2008
|
+
* @template T
|
|
2009
|
+
* @param {import("../database/drivers/base.js").default} db - Database connection.
|
|
2010
|
+
* @param {() => Promise<T>} callback - Transaction callback.
|
|
2011
|
+
* @returns {Promise<T>} Callback result.
|
|
2012
|
+
*/
|
|
2013
|
+
async _serializedCountMutation(db, callback) {
|
|
2014
|
+
const identifier = this.getDatabaseIdentifier() || "default"
|
|
2015
|
+
const previous = countMutationChains.get(identifier) || Promise.resolve()
|
|
2016
|
+
let resolveRun = () => {}
|
|
2017
|
+
/** @type {Promise<void>} */
|
|
2018
|
+
const run = new Promise((resolve) => {
|
|
2019
|
+
resolveRun = () => resolve(undefined)
|
|
2020
|
+
})
|
|
2021
|
+
const chain = previous.then(() => run)
|
|
2022
|
+
|
|
2023
|
+
countMutationChains.set(identifier, chain)
|
|
2024
|
+
await previous
|
|
2025
|
+
|
|
2026
|
+
try {
|
|
2027
|
+
return await this._transactionResult(db, callback)
|
|
2028
|
+
} finally {
|
|
2029
|
+
resolveRun()
|
|
2030
|
+
if (countMutationChains.get(identifier) === chain) countMutationChains.delete(identifier)
|
|
2031
|
+
}
|
|
2032
|
+
}
|
|
2033
|
+
|
|
1788
2034
|
/**
|
|
1789
2035
|
* Runs should accept report.
|
|
1790
2036
|
* @param {object} args - Options.
|
|
@@ -20,7 +20,7 @@ function safeEqual(a, b) {
|
|
|
20
20
|
|
|
21
21
|
/**
|
|
22
22
|
* Runs bearer token.
|
|
23
|
-
* @param {import("../../http-server/client/request.js").default} request - Request object.
|
|
23
|
+
* @param {import("../../http-server/client/request.js").default | import("../../http-server/client/websocket-request.js").default} request - Request object.
|
|
24
24
|
* @returns {string | null} - Bearer token from the Authorization header, if any.
|
|
25
25
|
*/
|
|
26
26
|
function bearerToken(request) {
|
|
@@ -57,17 +57,18 @@ function isLoopback(remoteAddress) {
|
|
|
57
57
|
* during development without being exposed to the network.
|
|
58
58
|
* @param {object} args - Options.
|
|
59
59
|
* @param {import("./registry.js").JobsMountOptions} args.options - Mount options.
|
|
60
|
-
* @param {import("../../http-server/client/request.js").default} args.request - Request object.
|
|
60
|
+
* @param {import("../../http-server/client/request.js").default | import("../../http-server/client/websocket-request.js").default} args.request - Request object.
|
|
61
61
|
* @param {import("../../configuration.js").default} args.configuration - Configuration instance.
|
|
62
62
|
* @param {import("../../authorization/ability.js").default | undefined} args.ability - Current ability.
|
|
63
|
+
* @param {string | null} [args.token] - Explicit websocket subscription token.
|
|
63
64
|
* @returns {Promise<boolean>} - Whether the request is authorized.
|
|
64
65
|
*/
|
|
65
|
-
export async function authorizeJobsRequest({ability, configuration, options, request}) {
|
|
66
|
+
export async function authorizeJobsRequest({ability, configuration, options, request, token: explicitToken}) {
|
|
66
67
|
const accessTokens = Array.isArray(options.accessTokens)
|
|
67
68
|
? options.accessTokens.filter((token) => typeof token === "string" && token.length > 0)
|
|
68
69
|
: []
|
|
69
70
|
const authorize = typeof options.authorize === "function" ? options.authorize : null
|
|
70
|
-
const token = bearerToken(request)
|
|
71
|
+
const token = explicitToken ?? bearerToken(request)
|
|
71
72
|
|
|
72
73
|
if (accessTokens.length > 0 && token) {
|
|
73
74
|
for (const accessToken of accessTokens) {
|
|
@@ -99,7 +99,11 @@ export default class VelociousBackgroundJobsWebController extends Controller {
|
|
|
99
99
|
*/
|
|
100
100
|
async health() {
|
|
101
101
|
await this._respond(async () => {
|
|
102
|
-
await this.render({json: {
|
|
102
|
+
await this.render({json: {
|
|
103
|
+
capabilities: {backgroundJobCountDeltas: 1},
|
|
104
|
+
ok: true,
|
|
105
|
+
service: "velocious-background-jobs"
|
|
106
|
+
}})
|
|
103
107
|
})
|
|
104
108
|
}
|
|
105
109
|
|
|
@@ -109,22 +113,15 @@ export default class VelociousBackgroundJobsWebController extends Controller {
|
|
|
109
113
|
*/
|
|
110
114
|
async stats() {
|
|
111
115
|
await this._respond(async () => {
|
|
112
|
-
const
|
|
113
|
-
/**
|
|
114
|
-
* By status.
|
|
115
|
-
* @type {Record<string, number>} */
|
|
116
|
-
const byStatus = {}
|
|
117
|
-
let total = 0
|
|
118
|
-
|
|
119
|
-
for (const status of DASHBOARD_STATUSES) {
|
|
120
|
-
byStatus[status] = counts[status] || 0
|
|
121
|
-
}
|
|
116
|
+
const snapshot = await this._store().countSnapshot()
|
|
122
117
|
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
118
|
+
await this.render({json: {
|
|
119
|
+
capabilities: {backgroundJobCountDeltas: 1},
|
|
120
|
+
counts: snapshot.counts,
|
|
121
|
+
generatedAtMs: Date.now(),
|
|
122
|
+
revision: snapshot.revision,
|
|
123
|
+
total: snapshot.total
|
|
124
|
+
}})
|
|
128
125
|
})
|
|
129
126
|
}
|
|
130
127
|
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
// @ts-check
|
|
2
|
+
|
|
3
|
+
import {BACKGROUND_JOB_COUNTS_CHANNEL} from "../store.js"
|
|
4
|
+
import VelociousWebsocketChannel from "../../http-server/websocket-channel.js"
|
|
5
|
+
import {authorizeJobsRequest} from "./authorization.js"
|
|
6
|
+
import {getJobsMount} from "./registry.js"
|
|
7
|
+
import {normalizeMountPrefix} from "./path-matcher.js"
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Authorized dashboard count-delta channel. Clients subscribe with the mount
|
|
11
|
+
* path and their normal bearer token as `authenticationToken`.
|
|
12
|
+
*/
|
|
13
|
+
export default class BackgroundJobCountsChannel extends VelociousWebsocketChannel {
|
|
14
|
+
/**
|
|
15
|
+
* Authorizes the subscription.
|
|
16
|
+
* @returns {Promise<boolean>} Whether the mount's normal dashboard authorization allows the subscription.
|
|
17
|
+
*/
|
|
18
|
+
async canSubscribe() {
|
|
19
|
+
if (typeof this.params.mountAt !== "string") return false
|
|
20
|
+
|
|
21
|
+
const mountAt = normalizeMountPrefix(this.params.mountAt)
|
|
22
|
+
const options = getJobsMount(this.session.configuration, mountAt)
|
|
23
|
+
|
|
24
|
+
if (!options || !this.session.upgradeRequest) return false
|
|
25
|
+
|
|
26
|
+
const token = typeof this.params.authenticationToken === "string"
|
|
27
|
+
? this.params.authenticationToken
|
|
28
|
+
: null
|
|
29
|
+
const ability = await this.session.configuration.resolveAbility({
|
|
30
|
+
params: this.params,
|
|
31
|
+
request: this.session.upgradeRequest
|
|
32
|
+
})
|
|
33
|
+
const authorized = await authorizeJobsRequest({
|
|
34
|
+
ability,
|
|
35
|
+
configuration: this.session.configuration,
|
|
36
|
+
options,
|
|
37
|
+
request: this.session.upgradeRequest,
|
|
38
|
+
token
|
|
39
|
+
})
|
|
40
|
+
|
|
41
|
+
if (!authorized) return false
|
|
42
|
+
|
|
43
|
+
this.databaseIdentifier = options.databaseIdentifier
|
|
44
|
+
|| this.session.configuration.getBackgroundJobsConfig().databaseIdentifier
|
|
45
|
+
|| "default"
|
|
46
|
+
|
|
47
|
+
return true
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* Matches only events from the database selected by the authorized mount.
|
|
52
|
+
* @param {import("../../http-server/websocket-channel.js").WebsocketJsonValue} broadcastParams - Publisher scope.
|
|
53
|
+
* @returns {boolean} Whether this subscription should receive the event.
|
|
54
|
+
*/
|
|
55
|
+
matches(broadcastParams) {
|
|
56
|
+
if (!broadcastParams || typeof broadcastParams !== "object" || Array.isArray(broadcastParams)) return false
|
|
57
|
+
|
|
58
|
+
return String(/** @type {Record<string, ?>} */ (broadcastParams).databaseIdentifier) === this.databaseIdentifier
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* Builds diagnostics.
|
|
63
|
+
* @returns {Record<string, string>} Non-sensitive diagnostics.
|
|
64
|
+
*/
|
|
65
|
+
debugSnapshot() {
|
|
66
|
+
return {databaseIdentifier: this.databaseIdentifier}
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/** @type {string} */
|
|
70
|
+
databaseIdentifier = ""
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* Registers the framework channel used by mounted jobs dashboards.
|
|
74
|
+
* @param {import("../../configuration.js").default} configuration - Configuration.
|
|
75
|
+
*/
|
|
76
|
+
static register(configuration) {
|
|
77
|
+
configuration.registerWebsocketChannel(BACKGROUND_JOB_COUNTS_CHANNEL, this)
|
|
78
|
+
}
|
|
79
|
+
}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
// @ts-check
|
|
2
2
|
|
|
3
3
|
import VelociousBackgroundJobsWebController from "./controller.js"
|
|
4
|
+
import BackgroundJobCountsChannel from "./counts-channel.js"
|
|
4
5
|
import {matchJobsApiPath, normalizeMountPrefix} from "./path-matcher.js"
|
|
5
6
|
import {registerJobsMount} from "./registry.js"
|
|
6
7
|
|
|
@@ -40,6 +41,7 @@ export default class VelociousBackgroundJobsApi {
|
|
|
40
41
|
const prefix = normalizeMountPrefix(at)
|
|
41
42
|
|
|
42
43
|
registerJobsMount(configuration, prefix, {accessTokens, allowedOrigins, authorize, databaseIdentifier, redactArgs})
|
|
44
|
+
BackgroundJobCountsChannel.register(configuration)
|
|
43
45
|
|
|
44
46
|
configuration.addRouteResolverHook(({currentPath, request}) => {
|
|
45
47
|
const match = matchJobsApiPath({method: request.httpMethod(), path: currentPath, prefix})
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
/**
|
|
4
4
|
* JobsMountOptions type.
|
|
5
5
|
* @typedef {object} JobsMountOptions
|
|
6
|
-
* @property {(args: {request: import("../../http-server/client/request.js").default, ability: (import("../../authorization/ability.js").default | undefined), token: (string | null), configuration: import("../../configuration.js").default}) => (boolean | void | Promise<boolean | void>)} [authorize] - Authorization callback. Return true to allow the request.
|
|
6
|
+
* @property {(args: {request: import("../../http-server/client/request.js").default | import("../../http-server/client/websocket-request.js").default, ability: (import("../../authorization/ability.js").default | undefined), token: (string | null), configuration: import("../../configuration.js").default}) => (boolean | void | Promise<boolean | void>)} [authorize] - Authorization callback. Return true to allow the request.
|
|
7
7
|
* @property {string[]} [accessTokens] - Bearer tokens accepted for cross-origin/native access.
|
|
8
8
|
* @property {string[]} [allowedOrigins] - Origins allowed for cross-origin browser access.
|
|
9
9
|
* @property {boolean} [redactArgs] - When true, job arguments are omitted from API responses.
|