velocious 1.0.574 → 1.0.576
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -3
- package/build/background-jobs/client.js +71 -3
- package/build/background-jobs/job.js +30 -0
- package/build/background-jobs/main.js +100 -14
- package/build/background-jobs/store.js +344 -41
- package/build/background-jobs/types.js +25 -1
- package/build/background-jobs/web/controller.js +1 -0
- package/build/database/advisory-lock-runner.js +27 -17
- package/build/database/drivers/mssql/index.js +98 -3
- package/build/database/record/index.js +10 -8
- package/build/database/table-data/table-column.js +2 -0
- package/build/frontend-models/resource-definition.js +1 -138
- package/build/src/background-jobs/client.d.ts +30 -0
- package/build/src/background-jobs/client.d.ts.map +1 -1
- package/build/src/background-jobs/client.js +64 -4
- package/build/src/background-jobs/job.d.ts +19 -0
- package/build/src/background-jobs/job.d.ts.map +1 -1
- package/build/src/background-jobs/job.js +27 -1
- package/build/src/background-jobs/main.d.ts +41 -0
- package/build/src/background-jobs/main.d.ts.map +1 -1
- package/build/src/background-jobs/main.js +95 -14
- package/build/src/background-jobs/store.d.ts +136 -0
- package/build/src/background-jobs/store.d.ts.map +1 -1
- package/build/src/background-jobs/store.js +314 -42
- package/build/src/background-jobs/types.d.ts +86 -2
- package/build/src/background-jobs/types.d.ts.map +1 -1
- package/build/src/background-jobs/types.js +26 -2
- package/build/src/background-jobs/web/controller.d.ts.map +1 -1
- package/build/src/background-jobs/web/controller.js +2 -1
- package/build/src/database/advisory-lock-runner.d.ts +27 -14
- package/build/src/database/advisory-lock-runner.d.ts.map +1 -1
- package/build/src/database/advisory-lock-runner.js +27 -18
- package/build/src/database/drivers/mssql/index.d.ts +32 -0
- package/build/src/database/drivers/mssql/index.d.ts.map +1 -1
- package/build/src/database/drivers/mssql/index.js +86 -4
- package/build/src/database/record/index.d.ts +12 -8
- package/build/src/database/record/index.d.ts.map +1 -1
- package/build/src/database/record/index.js +11 -9
- package/build/src/database/table-data/table-column.d.ts.map +1 -1
- package/build/src/database/table-data/table-column.js +4 -1
- package/build/src/frontend-models/resource-definition.d.ts.map +1 -1
- package/build/src/frontend-models/resource-definition.js +2 -124
- package/build/src/sync/local-mutation-log.d.ts +7 -1
- package/build/src/sync/local-mutation-log.d.ts.map +1 -1
- package/build/src/sync/local-mutation-log.js +18 -7
- package/build/src/sync/peer-mutation-bundle.d.ts.map +1 -1
- package/build/src/sync/peer-mutation-bundle.js +9 -7
- package/build/src/sync/signed-sync-envelope-replay-service.d.ts.map +1 -1
- package/build/src/sync/signed-sync-envelope-replay-service.js +3 -1
- package/build/src/sync/sync-envelope-replay-service.d.ts +69 -0
- package/build/src/sync/sync-envelope-replay-service.d.ts.map +1 -1
- package/build/src/sync/sync-envelope-replay-service.js +186 -18
- package/build/src/testing/test-runner.d.ts.map +1 -1
- package/build/src/testing/test-runner.js +17 -22
- package/build/src/utils/sha256-hex.d.ts +10 -0
- package/build/src/utils/sha256-hex.d.ts.map +1 -0
- package/build/src/utils/sha256-hex.js +126 -0
- package/build/sync/local-mutation-log.js +19 -5
- package/build/sync/peer-mutation-bundle.js +8 -6
- package/build/sync/signed-sync-envelope-replay-service.js +2 -0
- package/build/sync/sync-envelope-replay-service.js +205 -17
- package/build/testing/test-runner.js +17 -22
- package/build/tsconfig.tsbuildinfo +1 -1
- package/build/utils/sha256-hex.js +141 -0
- package/package.json +1 -1
- package/src/background-jobs/client.js +71 -3
- package/src/background-jobs/job.js +30 -0
- package/src/background-jobs/main.js +100 -14
- package/src/background-jobs/store.js +344 -41
- package/src/background-jobs/types.js +25 -1
- package/src/background-jobs/web/controller.js +1 -0
- package/src/database/advisory-lock-runner.js +27 -17
- package/src/database/drivers/mssql/index.js +98 -3
- package/src/database/record/index.js +10 -8
- package/src/database/table-data/table-column.js +2 -0
- package/src/frontend-models/resource-definition.js +1 -138
- package/src/sync/local-mutation-log.js +19 -5
- package/src/sync/peer-mutation-bundle.js +8 -6
- package/src/sync/signed-sync-envelope-replay-service.js +2 -0
- package/src/sync/sync-envelope-replay-service.js +205 -17
- package/src/testing/test-runner.js +17 -22
- package/src/utils/sha256-hex.js +141 -0
|
@@ -1,12 +1,26 @@
|
|
|
1
1
|
// @ts-check
|
|
2
2
|
|
|
3
|
-
import {randomUUID} from "crypto"
|
|
3
|
+
import {createHash, randomUUID} from "crypto"
|
|
4
4
|
import Logger from "../logger.js"
|
|
5
5
|
import TableData from "../database/table-data/index.js"
|
|
6
6
|
import VelociousError from "../velocious-error.js"
|
|
7
7
|
import BackgroundJobRecord from "./job-record.js"
|
|
8
8
|
import normalizeBackgroundJobError from "./normalize-error.js"
|
|
9
9
|
|
|
10
|
+
/**
|
|
11
|
+
* PreparedBackgroundJob type.
|
|
12
|
+
* @typedef {object} PreparedBackgroundJob
|
|
13
|
+
* @property {string} argsJson - Serialized arguments.
|
|
14
|
+
* @property {{concurrencyKey: string, maxConcurrency: number, queueDerived: boolean} | null} concurrency - Resolved concurrency.
|
|
15
|
+
* @property {number} createdAtMs - Creation timestamp.
|
|
16
|
+
* @property {import("./types.js").BackgroundJobExecutionMode} executionMode - Execution mode.
|
|
17
|
+
* @property {string} jobId - New job id.
|
|
18
|
+
* @property {string} jobName - Job name.
|
|
19
|
+
* @property {number} maxRetries - Retry cap.
|
|
20
|
+
* @property {string} queue - Queue name.
|
|
21
|
+
* @property {number} scheduledAtMs - Eligibility timestamp.
|
|
22
|
+
*/
|
|
23
|
+
|
|
10
24
|
const MIGRATIONS_TABLE = "velocious_internal_migrations"
|
|
11
25
|
const MIGRATION_SCOPE = "background_jobs"
|
|
12
26
|
const MIGRATION_VERSION = "20250215000000"
|
|
@@ -22,6 +36,7 @@ const DROP_FORKED_COLUMN_MIGRATION_VERSION = "20260719000000"
|
|
|
22
36
|
const LEGACY_POOLED_HANDOFF_ID_PREFIX = "velocious-pooled:"
|
|
23
37
|
const LEGACY_POOLED_QUEUED_HANDOFF_ID = `${LEGACY_POOLED_HANDOFF_ID_PREFIX}queued`
|
|
24
38
|
const JOBS_TABLE = "background_jobs"
|
|
39
|
+
const SCHEDULE_KEYS_TABLE = "background_job_schedule_keys"
|
|
25
40
|
const CONCURRENCY_TABLE = "background_job_concurrency"
|
|
26
41
|
const COUNTS_REVISION_TABLE = "background_job_count_revisions"
|
|
27
42
|
const COUNTS_REVISION_KEY = "counts"
|
|
@@ -153,16 +168,9 @@ export default class BackgroundJobsStore {
|
|
|
153
168
|
async enqueue({jobName, args, options}) {
|
|
154
169
|
await this.ensureReady()
|
|
155
170
|
|
|
156
|
-
const
|
|
157
|
-
const now = Date.now()
|
|
158
|
-
const executionMode = this._normalizeExecutionMode(options)
|
|
159
|
-
const maxRetries = this._normalizeMaxRetries(options?.maxRetries)
|
|
160
|
-
const scheduledAtMs = this._normalizeScheduledAtMs(options?.scheduledAtMs, now)
|
|
161
|
-
const argsJson = JSON.stringify(args || [])
|
|
162
|
-
const queue = this._normalizeQueue(options)
|
|
163
|
-
const concurrency = this._resolveConcurrency(options, queue)
|
|
171
|
+
const preparedJob = this._prepareJob({jobName, args, options})
|
|
164
172
|
/** @type {string} */
|
|
165
|
-
let resultJobId = jobId
|
|
173
|
+
let resultJobId = preparedJob.jobId
|
|
166
174
|
|
|
167
175
|
await this._withDb(async (db) => await this._serializedCountMutation(db, async () => {
|
|
168
176
|
if (options?.deduplicateWhileQueued) {
|
|
@@ -174,8 +182,8 @@ export default class BackgroundJobsStore {
|
|
|
174
182
|
.newQuery()
|
|
175
183
|
.from(JOBS_TABLE)
|
|
176
184
|
.select("id")
|
|
177
|
-
.where({status: "queued", job_name: jobName, args_json: argsJson, queue})
|
|
178
|
-
.where(`scheduled_at_ms <= ${db.quote(scheduledAtMs)}`)
|
|
185
|
+
.where({status: "queued", job_name: jobName, args_json: preparedJob.argsJson, queue: preparedJob.queue})
|
|
186
|
+
.where(`scheduled_at_ms <= ${db.quote(preparedJob.scheduledAtMs)}`)
|
|
179
187
|
.order("scheduled_at_ms ASC")
|
|
180
188
|
.limit(1)
|
|
181
189
|
.results()
|
|
@@ -187,37 +195,150 @@ export default class BackgroundJobsStore {
|
|
|
187
195
|
}
|
|
188
196
|
}
|
|
189
197
|
|
|
190
|
-
|
|
191
|
-
if (concurrency.queueDerived) {
|
|
192
|
-
await this._ensureQueueConcurrencyKey(db, concurrency)
|
|
193
|
-
} else {
|
|
194
|
-
await this._ensureConcurrencyKey(db, concurrency)
|
|
195
|
-
}
|
|
196
|
-
}
|
|
197
|
-
await db.insert({
|
|
198
|
-
tableName: JOBS_TABLE,
|
|
199
|
-
data: {
|
|
200
|
-
id: jobId,
|
|
201
|
-
job_name: jobName,
|
|
202
|
-
args_json: argsJson,
|
|
203
|
-
execution_mode: executionMode,
|
|
204
|
-
queue,
|
|
205
|
-
max_retries: maxRetries,
|
|
206
|
-
attempts: 0,
|
|
207
|
-
status: "queued",
|
|
208
|
-
scheduled_at_ms: scheduledAtMs,
|
|
209
|
-
created_at_ms: now,
|
|
210
|
-
concurrency_key: concurrency?.concurrencyKey || null,
|
|
211
|
-
max_concurrency: concurrency?.maxConcurrency || null,
|
|
212
|
-
handoff_id: null
|
|
213
|
-
}
|
|
214
|
-
})
|
|
198
|
+
await this._insertPreparedJob(db, {preparedJob, scheduleKey: null})
|
|
215
199
|
await this._recordCountDelta(db, {all: 1, queued: 1})
|
|
216
200
|
}))
|
|
217
201
|
|
|
218
202
|
return resultJobId
|
|
219
203
|
}
|
|
220
204
|
|
|
205
|
+
/**
|
|
206
|
+
* Replaces the queued owner of a stable schedule key with a new one-off job.
|
|
207
|
+
* A handed-off owner is left running and reported truthfully.
|
|
208
|
+
* @param {object} args - Options.
|
|
209
|
+
* @param {string} args.scheduleKey - Stable logical schedule key.
|
|
210
|
+
* @param {string} args.jobName - Job name.
|
|
211
|
+
* @param {Array<?>} args.args - Arguments.
|
|
212
|
+
* @param {import("./types.js").BackgroundJobOptions} [args.options] - Options.
|
|
213
|
+
* @returns {Promise<import("./types.js").BackgroundJobReplacementResult>} - Replacement result.
|
|
214
|
+
*/
|
|
215
|
+
async replaceScheduled({scheduleKey, jobName, args, options}) {
|
|
216
|
+
await this.ensureReady()
|
|
217
|
+
|
|
218
|
+
const normalizedScheduleKey = this._normalizeScheduleKey(scheduleKey)
|
|
219
|
+
const preparedJob = this._prepareJob({jobName, args, options})
|
|
220
|
+
|
|
221
|
+
return await this._withDb(async (db) => {
|
|
222
|
+
const lockName = this._scheduleKeyLockName(normalizedScheduleKey)
|
|
223
|
+
const acquired = await db.acquireAdvisoryLock(lockName)
|
|
224
|
+
|
|
225
|
+
if (!acquired) throw new Error("Failed to acquire background job schedule-key lock")
|
|
226
|
+
|
|
227
|
+
try {
|
|
228
|
+
return await this._serializedCountMutation(db, async () => {
|
|
229
|
+
const ownerRows = await db
|
|
230
|
+
.newQuery()
|
|
231
|
+
.from(SCHEDULE_KEYS_TABLE)
|
|
232
|
+
.where({schedule_key: normalizedScheduleKey})
|
|
233
|
+
.limit(1)
|
|
234
|
+
.results()
|
|
235
|
+
const ownerJobId = ownerRows[0] ? String(/** @type {Record<string, ?>} */ (ownerRows[0]).job_id) : null
|
|
236
|
+
const ownerJob = ownerJobId ? await this._getJobRowById(db, ownerJobId) : null
|
|
237
|
+
/** @type {import("./types.js").BackgroundJobReplacementPreviousStatus} */
|
|
238
|
+
let previousStatus = null
|
|
239
|
+
let previousJobId = null
|
|
240
|
+
|
|
241
|
+
if (ownerJob?.status === "queued") {
|
|
242
|
+
const affectedRows = await this._updateAffectedRows(db, {
|
|
243
|
+
tableName: JOBS_TABLE,
|
|
244
|
+
data: {status: "cancelled"},
|
|
245
|
+
conditions: {id: ownerJob.id, status: "queued"}
|
|
246
|
+
})
|
|
247
|
+
|
|
248
|
+
if (affectedRows === 1) {
|
|
249
|
+
previousJobId = ownerJob.id
|
|
250
|
+
previousStatus = "queued"
|
|
251
|
+
} else {
|
|
252
|
+
const currentOwnerJob = await this._getJobRowById(db, ownerJob.id)
|
|
253
|
+
|
|
254
|
+
if (currentOwnerJob?.status === "handed_off") {
|
|
255
|
+
previousJobId = currentOwnerJob.id
|
|
256
|
+
previousStatus = "handed_off"
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
} else if (ownerJob?.status === "handed_off") {
|
|
260
|
+
previousJobId = ownerJob.id
|
|
261
|
+
previousStatus = "handed_off"
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
await this._insertPreparedJob(db, {preparedJob, scheduleKey: normalizedScheduleKey})
|
|
265
|
+
await db.upsert({
|
|
266
|
+
tableName: SCHEDULE_KEYS_TABLE,
|
|
267
|
+
data: {schedule_key: normalizedScheduleKey, job_id: preparedJob.jobId},
|
|
268
|
+
conflictColumns: ["schedule_key"],
|
|
269
|
+
updateColumns: ["job_id"]
|
|
270
|
+
})
|
|
271
|
+
|
|
272
|
+
if (previousStatus !== "queued") await this._recordCountDelta(db, {all: 1, queued: 1})
|
|
273
|
+
|
|
274
|
+
return {jobId: preparedJob.jobId, previousJobId, previousStatus}
|
|
275
|
+
})
|
|
276
|
+
} finally {
|
|
277
|
+
await db.releaseAdvisoryLock(lockName)
|
|
278
|
+
}
|
|
279
|
+
})
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
/**
|
|
283
|
+
* Cancels the queued owner of a stable schedule key. A handed-off owner is
|
|
284
|
+
* detached but not marked stopped because execution may already be running.
|
|
285
|
+
* @param {string} scheduleKey - Stable logical schedule key.
|
|
286
|
+
* @returns {Promise<import("./types.js").BackgroundJobCancellationResult>} - Cancellation result.
|
|
287
|
+
*/
|
|
288
|
+
async cancelScheduled(scheduleKey) {
|
|
289
|
+
await this.ensureReady()
|
|
290
|
+
|
|
291
|
+
const normalizedScheduleKey = this._normalizeScheduleKey(scheduleKey)
|
|
292
|
+
|
|
293
|
+
return await this._withDb(async (db) => {
|
|
294
|
+
const lockName = this._scheduleKeyLockName(normalizedScheduleKey)
|
|
295
|
+
const acquired = await db.acquireAdvisoryLock(lockName)
|
|
296
|
+
|
|
297
|
+
if (!acquired) throw new Error("Failed to acquire background job schedule-key lock")
|
|
298
|
+
|
|
299
|
+
try {
|
|
300
|
+
return await this._serializedCountMutation(db, async () => {
|
|
301
|
+
const ownerRows = await db
|
|
302
|
+
.newQuery()
|
|
303
|
+
.from(SCHEDULE_KEYS_TABLE)
|
|
304
|
+
.where({schedule_key: normalizedScheduleKey})
|
|
305
|
+
.limit(1)
|
|
306
|
+
.results()
|
|
307
|
+
|
|
308
|
+
if (!ownerRows[0]) return {jobId: null, outcome: "not_found"}
|
|
309
|
+
|
|
310
|
+
const jobId = String(/** @type {Record<string, ?>} */ (ownerRows[0]).job_id)
|
|
311
|
+
const job = await this._getJobRowById(db, jobId)
|
|
312
|
+
|
|
313
|
+
if (job?.status === "queued") {
|
|
314
|
+
const affectedRows = await this._updateAffectedRows(db, {
|
|
315
|
+
tableName: JOBS_TABLE,
|
|
316
|
+
data: {status: "cancelled"},
|
|
317
|
+
conditions: {id: job.id, status: "queued"}
|
|
318
|
+
})
|
|
319
|
+
|
|
320
|
+
if (affectedRows === 1) {
|
|
321
|
+
await this._releaseScheduleOwnership(db, {jobId, scheduleKey: normalizedScheduleKey})
|
|
322
|
+
await this._recordStatusTransition(db, "queued", "cancelled")
|
|
323
|
+
|
|
324
|
+
return {jobId, outcome: "cancelled"}
|
|
325
|
+
}
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
const currentJob = await this._getJobRowById(db, jobId)
|
|
329
|
+
|
|
330
|
+
await this._releaseScheduleOwnership(db, {jobId, scheduleKey: normalizedScheduleKey})
|
|
331
|
+
|
|
332
|
+
if (currentJob?.status === "handed_off") return {jobId, outcome: "handed_off"}
|
|
333
|
+
|
|
334
|
+
return {jobId: null, outcome: "not_found"}
|
|
335
|
+
})
|
|
336
|
+
} finally {
|
|
337
|
+
await db.releaseAdvisoryLock(lockName)
|
|
338
|
+
}
|
|
339
|
+
})
|
|
340
|
+
}
|
|
341
|
+
|
|
221
342
|
/**
|
|
222
343
|
* Runs next available job.
|
|
223
344
|
* @param {object} [args] - Options.
|
|
@@ -526,6 +647,7 @@ export default class BackgroundJobsStore {
|
|
|
526
647
|
})
|
|
527
648
|
|
|
528
649
|
if (affectedRows !== 1) return false
|
|
650
|
+
await this._releaseScheduleOwnershipForJob(db, job)
|
|
529
651
|
await this._releaseConcurrency(db, job.concurrencyKey)
|
|
530
652
|
await this._recordStatusTransition(db, "handed_off", "completed")
|
|
531
653
|
return true
|
|
@@ -769,6 +891,7 @@ export default class BackgroundJobsStore {
|
|
|
769
891
|
|
|
770
892
|
await this._withDb(async (db) => await this._serializedCountMutation(db, async () => {
|
|
771
893
|
const snapshot = await this._countSnapshotOnLockedConnection(db)
|
|
894
|
+
if (await db.tableExists(SCHEDULE_KEYS_TABLE)) await db.query(`DELETE FROM ${db.quoteTable(SCHEDULE_KEYS_TABLE)}`)
|
|
772
895
|
await db.query(`DELETE FROM ${db.quoteTable(JOBS_TABLE)}`)
|
|
773
896
|
if (await db.tableExists(CONCURRENCY_TABLE)) await db.query(`DELETE FROM ${db.quoteTable(CONCURRENCY_TABLE)}`)
|
|
774
897
|
const deltas = Object.fromEntries(Object.entries(snapshot.counts).map(([key, value]) => [key, -value]))
|
|
@@ -791,6 +914,7 @@ export default class BackgroundJobsStore {
|
|
|
791
914
|
if (job.status === "handed_off") await this._lockConcurrencyRow(db, job.concurrencyKey)
|
|
792
915
|
const affectedRows = await this._updateAffectedRows(db, {tableName: JOBS_TABLE, data: {status: "cancelled"}, conditions: {id: job.id, status: job.status}})
|
|
793
916
|
if (affectedRows !== 1) return false
|
|
917
|
+
await this._releaseScheduleOwnershipForJob(db, job)
|
|
794
918
|
if (job.status === "handed_off") await this._releaseConcurrency(db, job.concurrencyKey)
|
|
795
919
|
await this._recordStatusTransition(db, job.status, "cancelled")
|
|
796
920
|
return true
|
|
@@ -812,6 +936,71 @@ export default class BackgroundJobsStore {
|
|
|
812
936
|
return (retryCount - 3) * 60 * 60 * 1000
|
|
813
937
|
}
|
|
814
938
|
|
|
939
|
+
/**
|
|
940
|
+
* Normalizes one new job before entering its persistence transaction.
|
|
941
|
+
* @param {object} args - Job input.
|
|
942
|
+
* @param {Array<?>} args.args - Job arguments.
|
|
943
|
+
* @param {string} args.jobName - Job name.
|
|
944
|
+
* @param {import("./types.js").BackgroundJobOptions} [args.options] - Job options.
|
|
945
|
+
* @returns {PreparedBackgroundJob} - Prepared job.
|
|
946
|
+
*/
|
|
947
|
+
_prepareJob({args, jobName, options}) {
|
|
948
|
+
const createdAtMs = Date.now()
|
|
949
|
+
const queue = this._normalizeQueue(options)
|
|
950
|
+
|
|
951
|
+
return {
|
|
952
|
+
argsJson: JSON.stringify(args || []),
|
|
953
|
+
concurrency: this._resolveConcurrency(options, queue),
|
|
954
|
+
createdAtMs,
|
|
955
|
+
executionMode: this._normalizeExecutionMode(options),
|
|
956
|
+
jobId: randomUUID(),
|
|
957
|
+
jobName,
|
|
958
|
+
maxRetries: this._normalizeMaxRetries(options?.maxRetries),
|
|
959
|
+
queue,
|
|
960
|
+
scheduledAtMs: this._normalizeScheduledAtMs(options?.scheduledAtMs, createdAtMs)
|
|
961
|
+
}
|
|
962
|
+
}
|
|
963
|
+
|
|
964
|
+
/**
|
|
965
|
+
* Inserts one prepared queued job, including its concurrency registration.
|
|
966
|
+
* @param {import("../database/drivers/base.js").default} db - Database connection.
|
|
967
|
+
* @param {object} args - Insert input.
|
|
968
|
+
* @param {PreparedBackgroundJob} args.preparedJob - Prepared job.
|
|
969
|
+
* @param {string | null} args.scheduleKey - Historical stable key.
|
|
970
|
+
* @returns {Promise<void>} - Resolves after insertion.
|
|
971
|
+
*/
|
|
972
|
+
async _insertPreparedJob(db, {preparedJob, scheduleKey}) {
|
|
973
|
+
const {concurrency} = preparedJob
|
|
974
|
+
|
|
975
|
+
if (concurrency) {
|
|
976
|
+
if (concurrency.queueDerived) {
|
|
977
|
+
await this._ensureQueueConcurrencyKey(db, concurrency)
|
|
978
|
+
} else {
|
|
979
|
+
await this._ensureConcurrencyKey(db, concurrency)
|
|
980
|
+
}
|
|
981
|
+
}
|
|
982
|
+
|
|
983
|
+
await db.insert({
|
|
984
|
+
tableName: JOBS_TABLE,
|
|
985
|
+
data: {
|
|
986
|
+
id: preparedJob.jobId,
|
|
987
|
+
job_name: preparedJob.jobName,
|
|
988
|
+
args_json: preparedJob.argsJson,
|
|
989
|
+
execution_mode: preparedJob.executionMode,
|
|
990
|
+
queue: preparedJob.queue,
|
|
991
|
+
max_retries: preparedJob.maxRetries,
|
|
992
|
+
attempts: 0,
|
|
993
|
+
status: "queued",
|
|
994
|
+
scheduled_at_ms: preparedJob.scheduledAtMs,
|
|
995
|
+
created_at_ms: preparedJob.createdAtMs,
|
|
996
|
+
schedule_key: scheduleKey,
|
|
997
|
+
concurrency_key: concurrency?.concurrencyKey || null,
|
|
998
|
+
max_concurrency: concurrency?.maxConcurrency || null,
|
|
999
|
+
handoff_id: null
|
|
1000
|
+
}
|
|
1001
|
+
})
|
|
1002
|
+
}
|
|
1003
|
+
|
|
815
1004
|
/**
|
|
816
1005
|
* Runs normalize max retries.
|
|
817
1006
|
* @param {number | null | undefined} maxRetries - Input.
|
|
@@ -838,6 +1027,28 @@ export default class BackgroundJobsStore {
|
|
|
838
1027
|
throw VelociousError.safe("background job scheduledAtMs must be a non-negative safe integer")
|
|
839
1028
|
}
|
|
840
1029
|
|
|
1030
|
+
/**
|
|
1031
|
+
* Validates a stable schedule key at the public storage boundary.
|
|
1032
|
+
* @param {string} scheduleKey - Stable logical schedule key.
|
|
1033
|
+
* @returns {string} - Validated key.
|
|
1034
|
+
*/
|
|
1035
|
+
_normalizeScheduleKey(scheduleKey) {
|
|
1036
|
+
if (typeof scheduleKey === "string" && scheduleKey.length > 0 && scheduleKey.length <= 255) return scheduleKey
|
|
1037
|
+
|
|
1038
|
+
throw VelociousError.safe("background job scheduleKey must be a non-empty string of at most 255 characters")
|
|
1039
|
+
}
|
|
1040
|
+
|
|
1041
|
+
/**
|
|
1042
|
+
* Builds a bounded advisory-lock name for one stable schedule key.
|
|
1043
|
+
* @param {string} scheduleKey - Validated stable schedule key.
|
|
1044
|
+
* @returns {string} - Advisory-lock name.
|
|
1045
|
+
*/
|
|
1046
|
+
_scheduleKeyLockName(scheduleKey) {
|
|
1047
|
+
const hash = createHash("sha256").update(scheduleKey).digest("hex").slice(0, 32)
|
|
1048
|
+
|
|
1049
|
+
return `background-jobs:schedule:${hash}`
|
|
1050
|
+
}
|
|
1051
|
+
|
|
841
1052
|
/**
|
|
842
1053
|
* Ensures the background-jobs schema exists, reusing a caller-held connection when
|
|
843
1054
|
* one is given rather than checking out its own.
|
|
@@ -900,6 +1111,7 @@ export default class BackgroundJobsStore {
|
|
|
900
1111
|
// row alone, otherwise later callers fail with "no such table".
|
|
901
1112
|
if (alreadyApplied && await db.tableExists(JOBS_TABLE)) {
|
|
902
1113
|
await this._ensureJobsTableColumns(db)
|
|
1114
|
+
await this._ensureScheduleKeysTable(db)
|
|
903
1115
|
await this._ensureConcurrencyTable(db)
|
|
904
1116
|
await this._ensureCountRevisionTable(db)
|
|
905
1117
|
await this._reconcileQueueConcurrency(db)
|
|
@@ -910,6 +1122,7 @@ export default class BackgroundJobsStore {
|
|
|
910
1122
|
|
|
911
1123
|
await this._applyMigrations(db)
|
|
912
1124
|
await this._ensureJobsTableColumns(db)
|
|
1125
|
+
await this._ensureScheduleKeysTable(db)
|
|
913
1126
|
await this._ensureConcurrencyTable(db)
|
|
914
1127
|
await this._ensureCountRevisionTable(db)
|
|
915
1128
|
await this._reconcileQueueConcurrency(db)
|
|
@@ -981,6 +1194,7 @@ export default class BackgroundJobsStore {
|
|
|
981
1194
|
table.string("status", {null: false, index: true})
|
|
982
1195
|
table.bigint("scheduled_at_ms", {null: false, index: true})
|
|
983
1196
|
table.bigint("created_at_ms", {null: false, index: true})
|
|
1197
|
+
table.string("schedule_key", {null: true, index: true})
|
|
984
1198
|
table.bigint("handed_off_at_ms", {null: true, index: true})
|
|
985
1199
|
table.string("handoff_id", {null: true})
|
|
986
1200
|
table.bigint("completed_at_ms", {null: true})
|
|
@@ -1081,6 +1295,36 @@ export default class BackgroundJobsStore {
|
|
|
1081
1295
|
}
|
|
1082
1296
|
|
|
1083
1297
|
await this._ensureQueueColumn(db)
|
|
1298
|
+
await this._ensureScheduleKeyColumn(db)
|
|
1299
|
+
}
|
|
1300
|
+
|
|
1301
|
+
/**
|
|
1302
|
+
* Idempotently adds the historical stable schedule key to existing jobs.
|
|
1303
|
+
* @param {import("../database/drivers/base.js").default} db - Database connection.
|
|
1304
|
+
* @returns {Promise<void>} - Resolves when ensured.
|
|
1305
|
+
*/
|
|
1306
|
+
async _ensureScheduleKeyColumn(db) {
|
|
1307
|
+
const lockName = `${MIGRATION_SCOPE}:schedule_key_column`
|
|
1308
|
+
const acquired = await db.acquireAdvisoryLock(lockName)
|
|
1309
|
+
|
|
1310
|
+
if (!acquired) throw new Error("Failed to acquire background jobs schedule-key schema lock")
|
|
1311
|
+
|
|
1312
|
+
try {
|
|
1313
|
+
db.clearSchemaCache()
|
|
1314
|
+
const lockedTable = await db.getTableByNameOrFail(JOBS_TABLE)
|
|
1315
|
+
|
|
1316
|
+
if (!(await lockedTable.getColumnByName("schedule_key"))) {
|
|
1317
|
+
const tableData = new TableData(JOBS_TABLE)
|
|
1318
|
+
|
|
1319
|
+
tableData.string("schedule_key", {null: true, index: true})
|
|
1320
|
+
|
|
1321
|
+
for (const sql of await db.alterTableSQLs(tableData)) await db.query(sql)
|
|
1322
|
+
|
|
1323
|
+
db.clearSchemaCache()
|
|
1324
|
+
}
|
|
1325
|
+
} finally {
|
|
1326
|
+
await db.releaseAdvisoryLock(lockName)
|
|
1327
|
+
}
|
|
1084
1328
|
}
|
|
1085
1329
|
|
|
1086
1330
|
/**
|
|
@@ -1235,10 +1479,9 @@ export default class BackgroundJobsStore {
|
|
|
1235
1479
|
}
|
|
1236
1480
|
|
|
1237
1481
|
async _initializeModel() {
|
|
1238
|
-
BackgroundJobRecord.setDatabaseIdentifier(this.getDatabaseIdentifier())
|
|
1239
|
-
|
|
1240
1482
|
if (BackgroundJobRecord.isInitialized()) return
|
|
1241
1483
|
|
|
1484
|
+
BackgroundJobRecord.setDatabaseIdentifier(this.getDatabaseIdentifier())
|
|
1242
1485
|
const pool = this.configuration.getDatabasePool(this.getDatabaseIdentifier())
|
|
1243
1486
|
|
|
1244
1487
|
await pool.withConnection({name: "Background jobs store initialize model"}, async () => {
|
|
@@ -1266,6 +1509,33 @@ export default class BackgroundJobsStore {
|
|
|
1266
1509
|
return this._normalizeJobRow(rows[0])
|
|
1267
1510
|
}
|
|
1268
1511
|
|
|
1512
|
+
/**
|
|
1513
|
+
* Releases ownership only when the key still points at the expected job.
|
|
1514
|
+
* @param {import("../database/drivers/base.js").default} db - Database connection.
|
|
1515
|
+
* @param {object} args - Ownership identity.
|
|
1516
|
+
* @param {string} args.jobId - Expected owner job id.
|
|
1517
|
+
* @param {string} args.scheduleKey - Stable schedule key.
|
|
1518
|
+
* @returns {Promise<void>} - Resolves when deleted or already superseded.
|
|
1519
|
+
*/
|
|
1520
|
+
async _releaseScheduleOwnership(db, {jobId, scheduleKey}) {
|
|
1521
|
+
await db.delete({
|
|
1522
|
+
tableName: SCHEDULE_KEYS_TABLE,
|
|
1523
|
+
conditions: {job_id: jobId, schedule_key: scheduleKey}
|
|
1524
|
+
})
|
|
1525
|
+
}
|
|
1526
|
+
|
|
1527
|
+
/**
|
|
1528
|
+
* Releases a job's ownership when it has a historical schedule key.
|
|
1529
|
+
* @param {import("../database/drivers/base.js").default} db - Database connection.
|
|
1530
|
+
* @param {import("./types.js").BackgroundJobRow} job - Terminal job.
|
|
1531
|
+
* @returns {Promise<void>} - Resolves when deleted or not applicable.
|
|
1532
|
+
*/
|
|
1533
|
+
async _releaseScheduleOwnershipForJob(db, job) {
|
|
1534
|
+
if (!job.scheduleKey) return
|
|
1535
|
+
|
|
1536
|
+
await this._releaseScheduleOwnership(db, {jobId: job.id, scheduleKey: job.scheduleKey})
|
|
1537
|
+
}
|
|
1538
|
+
|
|
1269
1539
|
/**
|
|
1270
1540
|
* Runs apply failure.
|
|
1271
1541
|
* @param {object} args - Options.
|
|
@@ -1300,6 +1570,7 @@ export default class BackgroundJobsStore {
|
|
|
1300
1570
|
})
|
|
1301
1571
|
|
|
1302
1572
|
if (affectedRows !== 1) return null
|
|
1573
|
+
if (!shouldRetry) await this._releaseScheduleOwnershipForJob(db, job)
|
|
1303
1574
|
await this._releaseConcurrency(db, job.concurrencyKey)
|
|
1304
1575
|
|
|
1305
1576
|
// Return a snapshot of the transition this update just applied rather than re-reading the row.
|
|
@@ -1412,6 +1683,7 @@ export default class BackgroundJobsStore {
|
|
|
1412
1683
|
args: this._parseArgs(row.args_json),
|
|
1413
1684
|
executionMode,
|
|
1414
1685
|
queue: row.queue ? String(row.queue) : DEFAULT_QUEUE,
|
|
1686
|
+
scheduleKey: row.schedule_key ? String(row.schedule_key) : null,
|
|
1415
1687
|
status: row.status ? String(row.status) : "queued",
|
|
1416
1688
|
attempts: this._normalizeNumber(row.attempts),
|
|
1417
1689
|
maxRetries: this._normalizeNumber(row.max_retries),
|
|
@@ -1544,6 +1816,34 @@ export default class BackgroundJobsStore {
|
|
|
1544
1816
|
await db.createTable(table)
|
|
1545
1817
|
}
|
|
1546
1818
|
|
|
1819
|
+
/**
|
|
1820
|
+
* Ensures the stable schedule-key ownership table exists.
|
|
1821
|
+
* @param {import("../database/drivers/base.js").default} db - Database connection.
|
|
1822
|
+
* @returns {Promise<void>} - Resolves when ready.
|
|
1823
|
+
*/
|
|
1824
|
+
async _ensureScheduleKeysTable(db) {
|
|
1825
|
+
if (await db.tableExists(SCHEDULE_KEYS_TABLE)) return
|
|
1826
|
+
|
|
1827
|
+
const lockName = `${MIGRATION_SCOPE}:schedule_keys_table`
|
|
1828
|
+
const acquired = await db.acquireAdvisoryLock(lockName)
|
|
1829
|
+
|
|
1830
|
+
if (!acquired) throw new Error("Failed to acquire background jobs schedule-key table schema lock")
|
|
1831
|
+
|
|
1832
|
+
try {
|
|
1833
|
+
db.clearSchemaCache()
|
|
1834
|
+
if (await db.tableExists(SCHEDULE_KEYS_TABLE)) return
|
|
1835
|
+
|
|
1836
|
+
const table = new TableData(SCHEDULE_KEYS_TABLE, {ifNotExists: true})
|
|
1837
|
+
|
|
1838
|
+
table.string("schedule_key", {primaryKey: true})
|
|
1839
|
+
table.string("job_id", {null: false, index: true})
|
|
1840
|
+
await db.createTable(table)
|
|
1841
|
+
db.clearSchemaCache()
|
|
1842
|
+
} finally {
|
|
1843
|
+
await db.releaseAdvisoryLock(lockName)
|
|
1844
|
+
}
|
|
1845
|
+
}
|
|
1846
|
+
|
|
1547
1847
|
/**
|
|
1548
1848
|
* Ensures the singleton durable count-revision row exists.
|
|
1549
1849
|
* @param {import("../database/drivers/base.js").default} db - Database connection.
|
|
@@ -1691,7 +1991,6 @@ export default class BackgroundJobsStore {
|
|
|
1691
1991
|
* @returns {Promise<{counts: Record<string, number>, revision: number, total: number}>} Snapshot.
|
|
1692
1992
|
*/
|
|
1693
1993
|
async _countSnapshotOnLockedConnection(db) {
|
|
1694
|
-
await this._lockCountRevision(db)
|
|
1695
1994
|
const rows = await db.newQuery().from(JOBS_TABLE).select("status").select("COUNT(*) AS count").group("status").results()
|
|
1696
1995
|
const counts = this._emptyCountBuckets()
|
|
1697
1996
|
let total = 0
|
|
@@ -2024,7 +2323,11 @@ export default class BackgroundJobsStore {
|
|
|
2024
2323
|
await previous
|
|
2025
2324
|
|
|
2026
2325
|
try {
|
|
2027
|
-
return await this._transactionResult(db,
|
|
2326
|
+
return await this._transactionResult(db, async () => {
|
|
2327
|
+
await this._lockCountRevision(db)
|
|
2328
|
+
|
|
2329
|
+
return await callback()
|
|
2330
|
+
})
|
|
2028
2331
|
} finally {
|
|
2029
2332
|
resolveRun()
|
|
2030
2333
|
if (countMutationChains.get(identifier) === chain) countMutationChains.delete(identifier)
|
|
@@ -35,6 +35,7 @@
|
|
|
35
35
|
* @property {Array<?>} args - Serialized job arguments.
|
|
36
36
|
* @property {BackgroundJobExecutionMode} executionMode - How the job should run.
|
|
37
37
|
* @property {string} queue - Queue name (defaults to `"default"`).
|
|
38
|
+
* @property {string | null} scheduleKey - Stable logical schedule key retained for history.
|
|
38
39
|
* @property {string} status - Current job status.
|
|
39
40
|
* @property {number | null} attempts - Failure attempts count.
|
|
40
41
|
* @property {number | null} maxRetries - Max retry attempts.
|
|
@@ -50,6 +51,23 @@
|
|
|
50
51
|
* @property {string | null} concurrencyKey - Durable concurrency key.
|
|
51
52
|
* @property {number | null} maxConcurrency - Durable per-key cap.
|
|
52
53
|
*/
|
|
54
|
+
/**
|
|
55
|
+
* @typedef {"queued" | "handed_off" | null} BackgroundJobReplacementPreviousStatus
|
|
56
|
+
*/
|
|
57
|
+
/**
|
|
58
|
+
* @typedef {object} BackgroundJobReplacementResult
|
|
59
|
+
* @property {string} jobId - Newly queued job id.
|
|
60
|
+
* @property {string | null} previousJobId - Previous active owner's job id.
|
|
61
|
+
* @property {BackgroundJobReplacementPreviousStatus} previousStatus - Previous owner's observed state.
|
|
62
|
+
*/
|
|
63
|
+
/**
|
|
64
|
+
* @typedef {"cancelled" | "handed_off" | "not_found"} BackgroundJobCancellationOutcome
|
|
65
|
+
*/
|
|
66
|
+
/**
|
|
67
|
+
* @typedef {object} BackgroundJobCancellationResult
|
|
68
|
+
* @property {string | null} jobId - Detached owner's job id, when one was active.
|
|
69
|
+
* @property {BackgroundJobCancellationOutcome} outcome - Truthful best-effort outcome.
|
|
70
|
+
*/
|
|
53
71
|
/**
|
|
54
72
|
* @typedef {object} BackgroundJobFailureEvent
|
|
55
73
|
* @property {BackgroundJobRow} job - Updated job row after failure handling.
|
|
@@ -72,6 +90,12 @@
|
|
|
72
90
|
* @typedef {{type: "enqueue", jobName: string, args?: Array<?>, options?: BackgroundJobOptions}} BackgroundJobEnqueueMessage
|
|
73
91
|
* @typedef {{type: "enqueued", jobId: string}} BackgroundJobEnqueuedMessage
|
|
74
92
|
* @typedef {{type: "enqueue-error", error?: string}} BackgroundJobEnqueueErrorMessage
|
|
93
|
+
* @typedef {{type: "replace-scheduled", scheduleKey: string, jobName: string, args?: Array<?>, options?: BackgroundJobOptions}} BackgroundJobReplaceScheduledMessage
|
|
94
|
+
* @typedef {{type: "schedule-replaced", jobId: string, previousJobId: string | null, previousStatus: BackgroundJobReplacementPreviousStatus}} BackgroundJobScheduleReplacedMessage
|
|
95
|
+
* @typedef {{type: "replace-scheduled-error", error?: string}} BackgroundJobReplaceScheduledErrorMessage
|
|
96
|
+
* @typedef {{type: "cancel-scheduled", scheduleKey: string}} BackgroundJobCancelScheduledMessage
|
|
97
|
+
* @typedef {{type: "schedule-cancelled", jobId: string | null, outcome: BackgroundJobCancellationOutcome}} BackgroundJobScheduleCancelledMessage
|
|
98
|
+
* @typedef {{type: "cancel-scheduled-error", error?: string}} BackgroundJobCancelScheduledErrorMessage
|
|
75
99
|
* @typedef {{type: "job", payload: BackgroundJobPayload}} BackgroundJobJobMessage
|
|
76
100
|
* @typedef {{type: "job-complete", jobId: string, handoffId?: string, workerId?: string, handedOffAtMs?: number}} BackgroundJobCompleteMessage
|
|
77
101
|
* @typedef {{type: "job-failed", jobId: string, error?: ?, handoffId?: string, workerId?: string, handedOffAtMs?: number}} BackgroundJobFailedMessage
|
|
@@ -79,7 +103,7 @@
|
|
|
79
103
|
* @typedef {{type: "job-update-error", jobId: string, error?: string}} BackgroundJobUpdateErrorMessage
|
|
80
104
|
*/
|
|
81
105
|
/**
|
|
82
|
-
* @typedef {BackgroundJobHelloMessage | BackgroundJobReadyMessage | BackgroundJobDrainingMessage | BackgroundJobHeartbeatMessage | BackgroundJobEnqueueMessage | BackgroundJobEnqueuedMessage | BackgroundJobEnqueueErrorMessage | BackgroundJobJobMessage | BackgroundJobCompleteMessage | BackgroundJobFailedMessage | BackgroundJobUpdatedMessage | BackgroundJobUpdateErrorMessage} BackgroundJobSocketMessage
|
|
106
|
+
* @typedef {BackgroundJobHelloMessage | BackgroundJobReadyMessage | BackgroundJobDrainingMessage | BackgroundJobHeartbeatMessage | BackgroundJobEnqueueMessage | BackgroundJobEnqueuedMessage | BackgroundJobEnqueueErrorMessage | BackgroundJobReplaceScheduledMessage | BackgroundJobScheduleReplacedMessage | BackgroundJobReplaceScheduledErrorMessage | BackgroundJobCancelScheduledMessage | BackgroundJobScheduleCancelledMessage | BackgroundJobCancelScheduledErrorMessage | BackgroundJobJobMessage | BackgroundJobCompleteMessage | BackgroundJobFailedMessage | BackgroundJobUpdatedMessage | BackgroundJobUpdateErrorMessage} BackgroundJobSocketMessage
|
|
83
107
|
*/
|
|
84
108
|
|
|
85
109
|
export const nothing = {}
|
|
@@ -199,6 +199,7 @@ export default class VelociousBackgroundJobsWebController extends Controller {
|
|
|
199
199
|
lastError: job.lastError,
|
|
200
200
|
maxRetries: job.maxRetries,
|
|
201
201
|
orphanedAtMs: job.orphanedAtMs,
|
|
202
|
+
scheduleKey: job.scheduleKey,
|
|
202
203
|
scheduledAtMs: job.scheduledAtMs,
|
|
203
204
|
status: job.status,
|
|
204
205
|
workerId: job.workerId
|