velocious 1.0.748 → 1.0.750

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/build/background-jobs/adapter.js +7 -0
  2. package/build/background-jobs/queue-priority-order.js +32 -23
  3. package/build/background-jobs/store.js +443 -15
  4. package/build/background-jobs/types.js +15 -0
  5. package/build/configuration-types.js +16 -0
  6. package/build/configuration.js +14 -1
  7. package/build/database/query/model-class-query.js +6 -0
  8. package/build/database/table-data/table-column.js +5 -1
  9. package/build/jobs/prune-terminal-background-jobs.js +15 -1
  10. package/build/src/background-jobs/adapter.d.ts +11 -0
  11. package/build/src/background-jobs/adapter.d.ts.map +1 -1
  12. package/build/src/background-jobs/adapter.js +7 -1
  13. package/build/src/background-jobs/queue-priority-order.d.ts +28 -14
  14. package/build/src/background-jobs/queue-priority-order.d.ts.map +1 -1
  15. package/build/src/background-jobs/queue-priority-order.js +33 -23
  16. package/build/src/background-jobs/store.d.ts +155 -3
  17. package/build/src/background-jobs/store.d.ts.map +1 -1
  18. package/build/src/background-jobs/store.js +386 -14
  19. package/build/src/background-jobs/types.d.ts +55 -0
  20. package/build/src/background-jobs/types.d.ts.map +1 -1
  21. package/build/src/background-jobs/types.js +16 -1
  22. package/build/src/configuration-types.d.ts +48 -0
  23. package/build/src/configuration-types.d.ts.map +1 -1
  24. package/build/src/configuration-types.js +16 -1
  25. package/build/src/configuration.d.ts.map +1 -1
  26. package/build/src/configuration.js +14 -2
  27. package/build/src/database/query/model-class-query.d.ts.map +1 -1
  28. package/build/src/database/query/model-class-query.js +4 -1
  29. package/build/src/database/table-data/table-column.d.ts.map +1 -1
  30. package/build/src/database/table-data/table-column.js +6 -2
  31. package/build/src/jobs/prune-terminal-background-jobs.d.ts +4 -1
  32. package/build/src/jobs/prune-terminal-background-jobs.d.ts.map +1 -1
  33. package/build/src/jobs/prune-terminal-background-jobs.js +14 -2
  34. package/package.json +1 -1
  35. package/src/background-jobs/adapter.js +7 -0
  36. package/src/background-jobs/queue-priority-order.js +32 -23
  37. package/src/background-jobs/store.js +443 -15
  38. package/src/background-jobs/types.js +15 -0
  39. package/src/configuration-types.js +16 -0
  40. package/src/configuration.js +14 -1
  41. package/src/database/query/model-class-query.js +6 -0
  42. package/src/database/table-data/table-column.js +5 -1
  43. package/src/jobs/prune-terminal-background-jobs.js +15 -1
@@ -238,4 +238,11 @@ export default class BackgroundJobsAdapter {
238
238
  * @returns {Promise<number>} - Deleted rows.
239
239
  */
240
240
  async pruneTerminalJobs(_args = {}) { throw new Error("BackgroundJobsAdapter#pruneTerminalJobs is not implemented") }
241
+
242
+ /**
243
+ * Moves terminal jobs past their retention windows into monthly archive tables.
244
+ * @param {{completedTtlMs?: number | null, failedTtlMs?: number | null, batchSize?: number, timeBudgetMs?: number}} [_args] - Retention options.
245
+ * @returns {Promise<import("./types.js").BackgroundJobsArchiveResult>} - Archived rows and per-month totals.
246
+ */
247
+ async archiveTerminalJobs(_args = {}) { throw new Error("BackgroundJobsAdapter#archiveTerminalJobs is not implemented") }
241
248
  }
@@ -3,11 +3,14 @@
3
3
  import {DEFAULT_BACKGROUND_JOB_QUEUE, QUEUE_CONCURRENCY_KEY_PREFIX} from "./job-semantics.js"
4
4
 
5
5
  /**
6
- * How many milliseconds of eligible waiting cancel one unit of `queues[name].priority`.
7
- * `1` priority unit buys this much queue-jumping: with the default, a lane ranked 10 points
8
- * below another is overtaken once the lower lane has an eligible job that has been waiting
9
- * 10 minutes. Time is the only condition — nothing here runs on a cadence, and a lane that
10
- * is dispatched normally never ages far enough for the term to matter.
6
+ * How long a lane may age before its configured priority stops buying queue-jumping.
7
+ * `1` priority unit still buys this much queue-jumping for fresh work — a priority-`10` lane is
8
+ * served before a priority-`0` lane while both are younger than one unit — but the aged row's
9
+ * wait is multiplied by the lane's gap to the highest priority, so every lane reaches the shared
10
+ * ceiling after exactly one unit of eligible waiting. With the default, no lane is starved for
11
+ * longer than a minute of contention regardless of how deep the priority ladder is. Time is the
12
+ * only condition — nothing here runs on a cadence, and a lane that is dispatched normally never
13
+ * ages far enough for the term to matter.
11
14
  */
12
15
  export const DEFAULT_QUEUE_PRIORITY_AGING_UNIT_MS = 60000
13
16
 
@@ -44,11 +47,21 @@ export function queuePriorityAgingUnitMs(configuration) {
44
47
  * never skipped, and every lane below it receives zero claims for as long as it stays
45
48
  * non-empty.
46
49
  *
47
- * Aging adds the row's own eligible wait, scaled by the priority unit, and caps that wait per
48
- * lane at the gap between the lane and the highest configured priority plus one unit. The lane
49
- * service age is folded into this same primary rank, so a lane that has gone without a claim can
50
- * cross the global ceiling before its own row-age cap. The queue-keyed service term remains
51
- * constant within a lane, preserving the existing intra-lane ordering and ceiling alternation.
50
+ * Aging adds the row's own eligible wait, multiplied by the lane's gap to the highest
51
+ * configured priority, and caps that wait at one `queuePriorityAgingUnitMs` for every lane. The
52
+ * lane service age is folded into this same primary rank, so a lane that has gone without a
53
+ * claim can cross the global ceiling before its own row-age cap. The queue-keyed service term
54
+ * remains constant within a lane, preserving the existing intra-lane ordering and ceiling
55
+ * alternation.
56
+ *
57
+ * The multiplier is what makes that ceiling reachable. Capping the raw wait at the gap instead
58
+ * (`(highest priority − priority + 1) × unitMs`) makes a lane wait as long as the gap it has to
59
+ * cancel: TensorBuzz's `auditRetention` lane (priority `-10` under a highest priority of `100`)
60
+ * needed ~111 minutes of eligible waiting, and its prune job was measured at 284–1,388 s per
61
+ * claim on 2026-11-19 while the prune itself took 5–19 s. Multiplying a wait bounded by one unit
62
+ * by the same gap reaches the identical ceiling after one unit for every lane, so the configured
63
+ * priority still orders fresh work and still buys exactly one unit of queue-jumping, but no lane
64
+ * has to age for the width of the whole priority ladder before it becomes claimable.
52
65
  *
53
66
  * Without a prior lane claim, the legacy row-age behavior remains the fallback. After a lane has
54
67
  * a durable claim stamp, its service age contributes directly to the rank, reducing the bound
@@ -80,19 +93,16 @@ export function buildQueuePriorityOrderSql({configuration, db, nowMs, laneFairne
80
93
 
81
94
  const queueExpression = `COALESCE(${db.quoteColumn("queue")}, ${db.quote(DEFAULT_BACKGROUND_JOB_QUEUE)})`
82
95
  const priorityRank = `CASE ${queueExpression} ${prioritized.map(([queue, priority]) => `WHEN ${db.quote(queue)} THEN ${priority}`).join(" ")} ELSE 0 END`
96
+ const priorityGap = `CASE ${queueExpression} ${prioritized.map(([queue, priority]) => `WHEN ${db.quote(queue)} THEN ${highestPriority - priority + 1}`).join(" ")} ELSE ${highestPriority + 1} END`
83
97
  const unitMs = queuePriorityAgingUnitMs(configuration)
84
98
 
85
99
  if (unitMs === null) return priorityRank
86
100
 
87
- const highestCeilingMs = (highestPriority + 1) * unitMs
88
- const agingCap = `CASE ${queueExpression} ${prioritized.map(([queue, priority]) => `WHEN ${db.quote(queue)} THEN ${(highestPriority - priority + 1) * unitMs}`).join(" ")} ELSE ${highestCeilingMs} END`
89
101
  const scheduledAtColumn = db.quoteColumn("scheduled_at_ms")
90
-
102
+ const agingWindow = `CASE WHEN ${scheduledAtColumn} <= (${db.quote(nowMs)} - ${unitMs}) THEN ${unitMs} ELSE (${db.quote(nowMs)} - ${scheduledAtColumn}) END`
91
103
  const laneFairness = laneFairnessRank ? ` + (${laneFairnessRank})` : ""
92
104
 
93
- return `(${priorityRank} * ${unitMs} + ` +
94
- `CASE WHEN ${scheduledAtColumn} <= (${db.quote(nowMs)} - ${agingCap}) THEN ${agingCap} ` +
95
- `ELSE (${db.quote(nowMs)} - ${scheduledAtColumn}) END${laneFairness})`
105
+ return `(${priorityRank} * ${unitMs} + ${priorityGap} * ${agingWindow}${laneFairness})`
96
106
  }
97
107
 
98
108
  /**
@@ -154,10 +164,11 @@ export function buildLaneFairnessOrderSql({configuration, db, nowMs, jobsTable,
154
164
  /**
155
165
  * Builds the raw lane-fairness rank used to compose the primary queue rank. A configured capped
156
166
  * lane without a claim stamp contributes no service-age bonus: without a claim there is no
157
- * lane-service history to rank. Once its row reaches its own cap, its configured priority is
158
- * retained as a small primary-key tie signal so a deep backlog cannot win solely by row age. A
159
- * long-ago claim stamp contributes its full starvation age to the primary rank. A lane without a
160
- * durable queue counter contributes zero and keeps the historical row-age fallback.
167
+ * lane-service history to rank. Once its row has aged past one aging unit — the point where its
168
+ * own lane reaches the shared ceiling — its configured priority is retained as a small
169
+ * primary-key tie signal so a deep backlog cannot win solely by row age. A long-ago claim stamp
170
+ * contributes its full starvation age to the primary rank. A lane without a durable queue counter
171
+ * contributes zero and keeps the historical row-age fallback.
161
172
  * @param {object} args - Options.
162
173
  * @param {import("../configuration.js").default} args.configuration - Active configuration.
163
174
  * @param {import("../database/drivers/base.js").default} args.db - Connection.
@@ -171,7 +182,6 @@ export function buildLaneFairnessRankSql({configuration, db, nowMs, jobsTable, c
171
182
  if (unitMs === null) return null
172
183
 
173
184
  const queues = configuration.getBackgroundJobsConfig().queues || {}
174
- const highestPriority = Math.max(0, ...Object.values(queues).map((queueConfig) => Number.isFinite(queueConfig?.priority) ? Number(queueConfig.priority) : 0))
175
185
  const queueExpression = `COALESCE(${jobsTable}.${db.quoteColumn("queue")}, ${db.quote(DEFAULT_BACKGROUND_JOB_QUEUE)})`
176
186
  const cappedQueues = Object.entries(queues).filter(([, queueConfig]) => Number.isInteger(queueConfig?.maxConcurrent) && Number(queueConfig.maxConcurrent) > 0)
177
187
  // Only queues with a `maxConcurrent` get a durable lane counter. With none configured there is
@@ -184,8 +194,7 @@ export function buildLaneFairnessRankSql({configuration, db, nowMs, jobsTable, c
184
194
 
185
195
  const neverServedTie = `CASE ${queueExpression} ${cappedQueues.map(([queue, queueConfig]) => {
186
196
  const priority = Number.isFinite(queueConfig?.priority) ? Number(queueConfig.priority) : 0
187
- const agingCap = (highestPriority - priority + 1) * unitMs
188
- return `WHEN ${db.quote(queue)} THEN CASE WHEN ${jobsTable}.${db.quoteColumn("scheduled_at_ms")} <= (${Number(nowMs)} - ${agingCap}) THEN ${priority} ELSE 0 END`
197
+ return `WHEN ${db.quote(queue)} THEN CASE WHEN ${jobsTable}.${db.quoteColumn("scheduled_at_ms")} <= (${Number(nowMs)} - ${unitMs}) THEN ${priority} ELSE 0 END`
189
198
  }).join(" ")} ELSE 0 END`
190
199
 
191
200
  return `(CASE WHEN ${laneKeyCase} IS NULL THEN 0 WHEN ${lastClaimedLookup} IS NULL THEN (${neverServedTie}) ELSE (${Number(nowMs)} - ${lastClaimedLookup}) END)`
@@ -172,6 +172,60 @@ const RETENTION_INDEX_DEFINITIONS = [
172
172
  {columns: ["status", "failed_at_ms", "id"], name: "index_background_jobs_failed_retention"},
173
173
  {columns: ["status", "orphaned_at_ms", "id"], name: "index_background_jobs_orphaned_retention"}
174
174
  ]
175
+ // Monthly terminal-job archive tables (`background_jobs_YYYY_MM`) keep the
176
+ // retention history queryable for statistics instead of dropping it. The month
177
+ // pattern also identifies which tables the statistics surface may read.
178
+ const ARCHIVE_TABLE_PREFIX = "background_jobs_"
179
+ const ARCHIVE_TABLE_MONTH_PATTERN = /^background_jobs_(\d{4})_(\d{2})$/
180
+ const ARCHIVE_MONTH_KEY_PATTERN = /^(\d{4})-(\d{2})$/
181
+ // Terminal statuses the archival pass moves into the monthly tables. Each
182
+ // status is discovered through its own terminal timestamp column, which also
183
+ // decides the destination month table. `cancelled` has no terminal timestamp
184
+ // column, so its creation time decides the month and its retention window.
185
+ const ARCHIVE_TERMINAL_STATUSES = [
186
+ {column: "completed_at_ms", status: "completed", ttlKey: "completedTtlMs"},
187
+ {column: "created_at_ms", status: "cancelled", ttlKey: "completedTtlMs"},
188
+ {column: "failed_at_ms", status: "failed", ttlKey: "failedTtlMs"},
189
+ {column: "orphaned_at_ms", status: "orphaned", ttlKey: "failedTtlMs"}
190
+ ]
191
+ // Archived statuses whose rows count as failures in the statistics surface.
192
+ const ARCHIVE_FAILURE_STATUSES = ["failed", "orphaned"]
193
+
194
+ /**
195
+ * Maps a terminal timestamp to its archive month key (`YYYY-MM`, UTC so the
196
+ * month boundary is identical on every process regardless of local timezone).
197
+ * @param {number | null} timestampMs - Terminal timestamp.
198
+ * @returns {string} - Month key.
199
+ */
200
+ function archiveMonthKeyForTimestampMs(timestampMs) {
201
+ const date = new Date(timestampMs === null ? 0 : timestampMs)
202
+ const month = String(date.getUTCMonth() + 1).padStart(2, "0")
203
+
204
+ return `${date.getUTCFullYear()}-${month}`
205
+ }
206
+
207
+ /**
208
+ * Runs archive table name for month key.
209
+ * @param {string} monthKey - Month key in `YYYY-MM` form.
210
+ * @returns {string} - Archive table name (`background_jobs_YYYY_MM`).
211
+ */
212
+ function archiveTableNameForMonthKey(monthKey) {
213
+ return `${ARCHIVE_TABLE_PREFIX}${monthKey.replace("-", "_")}`
214
+ }
215
+
216
+ /**
217
+ * Nearest-rank percentile over archived job durations.
218
+ * @param {number[]} values - Duration samples in ms.
219
+ * @param {number} quantile - Quantile in (0, 1].
220
+ * @returns {number | null} - Percentile value, or null when there are no samples.
221
+ */
222
+ function archivePercentileMs(values, quantile) {
223
+ if (values.length === 0) return null
224
+
225
+ const sorted = [...values].sort((left, right) => left - right)
226
+
227
+ return sorted[Math.max(0, Math.ceil(quantile * sorted.length) - 1)]
228
+ }
175
229
  const CONCURRENCY_TABLE = "background_job_concurrency"
176
230
  const COUNTS_REVISION_TABLE = "background_job_count_revisions"
177
231
  const COUNTS_REVISION_KEY = "counts"
@@ -242,6 +296,11 @@ const schemaApplyChains = new Map()
242
296
  const transactionMutationChains = new Map()
243
297
 
244
298
  export default class BackgroundJobsStore extends BackgroundJobsAdapter {
299
+ /**
300
+ * Copy column sets cached per archive table name.
301
+ * @type {Map<string, string[]>} */
302
+ _archiveColumnCacheByTable = new Map()
303
+
245
304
  /**
246
305
  * Runs constructor.
247
306
  * @param {object} args - Options.
@@ -1317,9 +1376,10 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
1317
1376
  * queue ranking uses: `priority * queuePriorityAgingUnitMs + (now - scheduled_at_ms) DESC`, so a
1318
1377
  * priority unit buys exactly that many milliseconds of queue-jumping and no more.
1319
1378
  *
1320
- * The primary rank caps each lane's age term at the gap to the highest configured priority, so
1321
- * every lane whose backlog has aged past that cap converges on the identical ceiling and this
1322
- * tie-break is the only term left that still reflects how long a row has actually waited. Written
1379
+ * The primary rank bounds each lane's age term at one aging unit and multiplies it by the lane's
1380
+ * gap to the highest configured priority, so every lane whose backlog has aged past that unit
1381
+ * converges on the identical ceiling and this tie-break is the only term left that still reflects
1382
+ * how long a row has actually waited. Written
1323
1383
  * as an absolute `priority DESC` the lead never decays: a row demoted to
1324
1384
  * {@link RETRY_BACKGROUND_JOB_PRIORITY} (-1) cannot outrank any priority-0 row in its lane however
1325
1385
  * long it waits, because its own age stopped counting once it passed the cap. Measured on
@@ -2165,18 +2225,7 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
2165
2225
  let deleted = 0
2166
2226
 
2167
2227
  for (;;) {
2168
- const candidates = await this._withDb(async (db) =>
2169
- await db
2170
- .newQuery()
2171
- .from(JOBS_TABLE)
2172
- .select("id")
2173
- .where({status})
2174
- .where(`${db.quoteColumn(column)} <= ${db.quote(cutoff)}`)
2175
- .order({column, direction: "ASC"})
2176
- .order({column: "id", direction: "ASC"})
2177
- .limit(batchSize)
2178
- .results()
2179
- )
2228
+ const candidates = await this._selectTerminalCandidatePage({batchSize, column, cutoff, status})
2180
2229
 
2181
2230
  if (candidates.length === 0) break
2182
2231
 
@@ -2209,6 +2258,385 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
2209
2258
 
2210
2259
  return deleted
2211
2260
  }
2261
+ /**
2262
+ * Selects one bounded retention candidate page ordered exactly like every
2263
+ * other retention pass (terminal timestamp ascending, then id ascending) so
2264
+ * the delete-only prune and the monthly archive sweeps converge on the same
2265
+ * page sequence. Discovery runs on a plain connection, never inside the
2266
+ * singleton count-revision mutation lock.
2267
+ * @param {object} args - Options.
2268
+ * @param {number} args.batchSize - Max rows per page.
2269
+ * @param {string} args.column - Terminal timestamp column compared against the cutoff.
2270
+ * @param {number} args.cutoff - Select rows whose column value is `<= cutoff`.
2271
+ * @param {string} args.status - Terminal status to select.
2272
+ * @param {boolean} [args.withTerminalTimestamp] - Also select the terminal timestamp under a stable alias.
2273
+ * @returns {Promise<Array<Record<string, ReturnType<typeof JSON.parse>>>>} - Candidate rows.
2274
+ */
2275
+ async _selectTerminalCandidatePage({batchSize, column, cutoff, status, withTerminalTimestamp = false}) {
2276
+ return await this._withDb(async (db) => {
2277
+ const query = db
2278
+ .newQuery()
2279
+ .from(JOBS_TABLE)
2280
+ .select("id")
2281
+ .where({status})
2282
+ .where(`${db.quoteColumn(column)} <= ${db.quote(cutoff)}`)
2283
+ .order({column, direction: "ASC"})
2284
+ .order({column: "id", direction: "ASC"})
2285
+ .limit(batchSize)
2286
+
2287
+ if (withTerminalTimestamp) query.select(`${db.quoteColumn(column)} AS terminal_timestamp_ms`)
2288
+
2289
+ return await query.results()
2290
+ })
2291
+ }
2292
+
2293
+ /**
2294
+ * Archives terminal job rows past their retention window into monthly
2295
+ * statistics tables (`background_jobs_YYYY_MM`, one per terminal-timestamp
2296
+ * month) instead of deleting them, so the active `background_jobs` table
2297
+ * stays small while history stays queryable. Like `pruneTerminalJobs`,
2298
+ * candidate discovery runs on a plain connection outside the count-revision
2299
+ * lock; only the short move transaction (copy into the month table, then
2300
+ * delete the same revalidated ids) is serialized, and the whole pass is
2301
+ * bounded by `batchSize` rows per batch and a wall-clock `timeBudgetMs`. A
2302
+ * pass that runs out of budget stops early and the next scheduled run
2303
+ * resumes from the same deterministic page order, so no unbounded DELETE is
2304
+ * ever issued.
2305
+ * @param {object} [args] - Options.
2306
+ * @param {number | null} [args.completedTtlMs] - Archive `completed` and `cancelled` jobs whose window timestamp is older than this many ms. Falsy or `<= 0` disables.
2307
+ * @param {number | null} [args.failedTtlMs] - Archive terminal `failed`/`orphaned` jobs older than this many ms. Falsy or `<= 0` disables.
2308
+ * @param {number} [args.batchSize] - Max rows moved per batch. Default `5000`.
2309
+ * @param {number} [args.timeBudgetMs] - Wall-clock budget for the whole pass. Default `30000`.
2310
+ * @returns {Promise<import("./types.js").BackgroundJobsArchiveResult>} - Rows archived, per-month totals and whether the pass finished or stopped on budget.
2311
+ */
2312
+ async archiveTerminalJobs({completedTtlMs = null, failedTtlMs = null, batchSize = 5000, timeBudgetMs = 30000} = {}) {
2313
+ await this.ensureReady()
2314
+
2315
+ const startedAtMs = this.clock.now()
2316
+ const size = batchSize > 0 ? batchSize : 5000
2317
+ const budgetMs = timeBudgetMs > 0 ? timeBudgetMs : 30000
2318
+ const deadlineMs = startedAtMs + budgetMs
2319
+ /** @type {Record<string, number>} */
2320
+ const perMonth = {}
2321
+ let archived = 0
2322
+ let completedPass = true
2323
+
2324
+ for (const {column, status, ttlKey} of ARCHIVE_TERMINAL_STATUSES) {
2325
+ const ttlMs = ttlKey === "completedTtlMs" ? completedTtlMs : failedTtlMs
2326
+
2327
+ if (typeof ttlMs !== "number" || ttlMs <= 0) continue
2328
+
2329
+ const result = await this._archiveStatusBatches({
2330
+ archivedAtMs: this.clock.now(),
2331
+ batchSize: size,
2332
+ column,
2333
+ deadlineMs,
2334
+ perMonth,
2335
+ status,
2336
+ cutoff: this.clock.now() - ttlMs
2337
+ })
2338
+
2339
+ archived += result.archived
2340
+
2341
+ if (!result.completedPass) {
2342
+ completedPass = false
2343
+ break
2344
+ }
2345
+ }
2346
+
2347
+ return {archived, completedPass, perMonth}
2348
+ }
2349
+
2350
+ /**
2351
+ * Moves rows of one terminal status older than a cutoff into their monthly
2352
+ * archive tables, batch by batch, until a candidate page is short or the
2353
+ * wall-clock budget is spent. Pages are ordered exactly like the delete-only
2354
+ * retention pass (terminal timestamp ascending, then id ascending), so a
2355
+ * pass that stopped on budget resumes on the same page sequence.
2356
+ * @param {object} args - Options.
2357
+ * @param {number} args.archivedAtMs - Timestamp stamped into `archived_at_ms`.
2358
+ * @param {number} args.batchSize - Max rows per batch.
2359
+ * @param {string} args.column - Terminal timestamp column compared against the cutoff.
2360
+ * @param {number} args.cutoff - Move rows whose column value is `<= cutoff`.
2361
+ * @param {number} args.deadlineMs - Stop starting new batches at this clock time.
2362
+ * @param {Record<string, number>} args.perMonth - Accumulated per-month archive totals.
2363
+ * @param {string} args.status - Terminal status to archive.
2364
+ * @returns {Promise<{archived: number, completedPass: boolean}>} - Status totals.
2365
+ */
2366
+ async _archiveStatusBatches({archivedAtMs, batchSize, column, cutoff, deadlineMs, perMonth, status}) {
2367
+ let archived = 0
2368
+
2369
+ for (;;) {
2370
+ if (this.clock.now() >= deadlineMs) return {archived, completedPass: false}
2371
+
2372
+ const candidates = await this._selectTerminalCandidatePage({batchSize, column, cutoff, status, withTerminalTimestamp: true})
2373
+
2374
+ if (candidates.length === 0) break
2375
+
2376
+ /** @type {Record<string, string[]>} */
2377
+ const idsByMonth = {}
2378
+
2379
+ for (const candidate of candidates) {
2380
+ const typedRow = /** @type {Record<string, ReturnType<typeof JSON.parse>>} */ (candidate)
2381
+ const monthKey = archiveMonthKeyForTimestampMs(this._normalizeNumber(typedRow.terminal_timestamp_ms))
2382
+ const monthIds = idsByMonth[monthKey] || (idsByMonth[monthKey] = [])
2383
+
2384
+ monthIds.push(String(typedRow.id))
2385
+ }
2386
+
2387
+ const months = Object.keys(idsByMonth).sort()
2388
+
2389
+ // Table creation is schema DDL: it runs on its own advisory lock before
2390
+ // the move transaction, never inside the count-revision serialization.
2391
+ for (const monthKey of months) {
2392
+ await this._ensureArchiveTable(monthKey)
2393
+ }
2394
+
2395
+ const moved = await this._serializedCountMutation(async (db) => {
2396
+ let statusMoved = 0
2397
+
2398
+ for (const monthKey of months) {
2399
+ const monthIds = idsByMonth[monthKey]
2400
+ const archiveTableName = archiveTableNameForMonthKey(monthKey)
2401
+ const archiveTable = db.quoteTable(archiveTableName)
2402
+ const columnSql = (await this._archiveColumnNames(db, archiveTableName)).map((columnName) => db.quoteColumn(columnName)).join(", ")
2403
+ const jobsTable = db.quoteTable(JOBS_TABLE)
2404
+ const idSql = monthIds.map((id) => db.quote(id)).join(", ")
2405
+
2406
+ await db.query(this._archiveInsertSql({archiveTable, archivedAtMs, column, columnSql, cutoff, db, idSql, jobsTable, status}))
2407
+
2408
+ const removed = await db.affectedRows(
2409
+ `DELETE FROM ${jobsTable} WHERE ${db.quoteColumn("id")} IN (${idSql}) AND ${db.quoteColumn("status")} = ${db.quote(status)} AND ${db.quoteColumn(column)} <= ${db.quote(cutoff)}`
2410
+ )
2411
+
2412
+ statusMoved += removed
2413
+ perMonth[monthKey] = (perMonth[monthKey] || 0) + removed
2414
+ }
2415
+
2416
+ // `cancelled` rows are not part of the counted status buckets, so only
2417
+ // bucket-counted statuses publish a signed delta.
2418
+ if (statusMoved > 0 && COUNTED_JOB_STATUSES.includes(status)) {
2419
+ await this._recordCountDelta(db, {all: -statusMoved, [status]: -statusMoved})
2420
+ }
2421
+
2422
+ return statusMoved
2423
+ })
2424
+
2425
+ archived += moved
2426
+ if (candidates.length < batchSize) break
2427
+ }
2428
+
2429
+ return {archived, completedPass: true}
2430
+ }
2431
+
2432
+ /**
2433
+ * Builds the driver-appropriate conflict-skipping `INSERT ... SELECT` that
2434
+ * copies one candidate page into a monthly archive table. The archive
2435
+ * primary key is the job id, so replaying a batch over rows that were
2436
+ * already archived is a no-op instead of a duplicate or an error — that is
2437
+ * what makes an interrupted move resumable.
2438
+ * @param {object} args - Options.
2439
+ * @param {string} args.archiveTable - Quoted archive table name.
2440
+ * @param {number} args.archivedAtMs - Value for `archived_at_ms`.
2441
+ * @param {string} args.column - Terminal timestamp column revalidated by the copy.
2442
+ * @param {string} args.columnSql - Comma-separated quoted job column names.
2443
+ * @param {number} args.cutoff - Copy rows whose column value is `<= cutoff`.
2444
+ * @param {import("../database/drivers/base.js").default} args.db - Database connection.
2445
+ * @param {string} args.idSql - Comma-separated quoted candidate ids.
2446
+ * @param {string} args.jobsTable - Quoted jobs table name.
2447
+ * @param {string} args.status - Terminal status revalidated by the copy.
2448
+ * @returns {string} - The insert statement.
2449
+ */
2450
+ _archiveInsertSql({archiveTable, archivedAtMs, column, columnSql, cutoff, db, idSql, jobsTable, status}) {
2451
+ const insertColumns = `${columnSql}, ${db.quoteColumn("archived_at_ms")}`
2452
+ const revalidationSql = `${db.quoteColumn("id")} IN (${idSql}) AND ${db.quoteColumn("status")} = ${db.quote(status)} AND ${db.quoteColumn(column)} <= ${db.quote(cutoff)}`
2453
+ const selectSql = `SELECT ${columnSql}, ${db.quote(archivedAtMs)} FROM ${jobsTable} WHERE ${revalidationSql}`
2454
+
2455
+ if (db.getType() === "mysql") return `INSERT IGNORE INTO ${archiveTable} (${insertColumns}) ${selectSql}`
2456
+ if (db.getType() === "sqlite") return `INSERT OR IGNORE INTO ${archiveTable} (${insertColumns}) ${selectSql}`
2457
+ if (db.getType() === "pgsql") return `INSERT INTO ${archiveTable} (${insertColumns}) ${selectSql} ON CONFLICT (${db.quoteColumn("id")}) DO NOTHING`
2458
+
2459
+ // SQL Server has no INSERT IGNORE clause: the anti-join expresses the same
2460
+ // "skip rows already archived under this primary key" contract.
2461
+ return `INSERT INTO ${archiveTable} (${insertColumns}) ${selectSql} AND NOT EXISTS (SELECT 1 FROM ${archiveTable} AS archived_rows WHERE archived_rows.${db.quoteColumn("id")} = ${jobsTable}.${db.quoteColumn("id")})`
2462
+ }
2463
+
2464
+ /**
2465
+ * Reads the copy column set from the archive table itself (every column
2466
+ * except `archived_at_ms`), so an archive copy always matches the schema the
2467
+ * destination table actually has instead of a hand-maintained duplicate list.
2468
+ * Cached per table name: the columns of an existing archive table never
2469
+ * change under the archival service.
2470
+ * @param {import("../database/drivers/base.js").default} db - Database connection.
2471
+ * @param {string} archiveTableName - Archive table name.
2472
+ * @returns {Promise<string[]>} - Column names in table order.
2473
+ */
2474
+ async _archiveColumnNames(db, archiveTableName) {
2475
+ const cache = this._archiveColumnCacheByTable
2476
+ const cached = cache.get(archiveTableName)
2477
+
2478
+ if (cached) return cached
2479
+
2480
+ const table = await db.getTableByNameOrFail(archiveTableName)
2481
+ const columnNames = (await table.getColumns())
2482
+ .map((column) => column.getName())
2483
+ .filter((columnName) => columnName !== "archived_at_ms")
2484
+
2485
+ cache.set(archiveTableName, columnNames)
2486
+
2487
+ return columnNames
2488
+ }
2489
+
2490
+ /**
2491
+ * Idempotently creates one monthly archive table mirroring the current
2492
+ * `background_jobs` columns plus `archived_at_ms`, with the job id as
2493
+ * primary key so replays are no-ops. Serialized by a per-month advisory
2494
+ * lock because several processes may activate the same month concurrently.
2495
+ * @param {string} monthKey - Month key in `YYYY-MM` form.
2496
+ * @returns {Promise<string>} - The archive table name.
2497
+ */
2498
+ async _ensureArchiveTable(monthKey) {
2499
+ if (!ARCHIVE_MONTH_KEY_PATTERN.test(monthKey)) throw new Error(`Invalid background jobs archive month key: ${monthKey}`)
2500
+
2501
+ const tableName = archiveTableNameForMonthKey(monthKey)
2502
+
2503
+ return await this._withDb(async (db) => {
2504
+ if (await db.tableExists(tableName)) return tableName
2505
+
2506
+ const lockName = `${MIGRATION_SCOPE}:archive_table:${monthKey}`
2507
+ const acquired = await db.acquireAdvisoryLock(lockName)
2508
+
2509
+ if (!acquired) throw new Error(`Failed to acquire background jobs archive table lock for ${monthKey}`)
2510
+
2511
+ try {
2512
+ db.clearSchemaCache()
2513
+ if (await db.tableExists(tableName)) return tableName
2514
+
2515
+ const jobsTable = await db.getTableByNameOrFail(JOBS_TABLE)
2516
+ const tableData = new TableData(tableName, {ifNotExists: true})
2517
+
2518
+ for (const column of await jobsTable.getColumns()) {
2519
+ tableData.addColumn(column.getTableDataColumn())
2520
+ }
2521
+
2522
+ tableData.bigint("archived_at_ms", {null: false})
2523
+
2524
+ await db.createTable(tableData)
2525
+
2526
+ for (const sql of await db.createIndexSQLs({columns: ["job_name"], ifNotExists: db.getType() === "sqlite", tableName})) {
2527
+ await db.query(sql)
2528
+ }
2529
+
2530
+ db.clearSchemaCache()
2531
+
2532
+ return tableName
2533
+ } finally {
2534
+ await db.releaseAdvisoryLock(lockName)
2535
+ }
2536
+ })
2537
+ }
2538
+
2539
+ /**
2540
+ * Lists the monthly archive table month keys present in this database.
2541
+ * @returns {Promise<string[]>} - Month keys (`YYYY-MM`) in ascending order.
2542
+ */
2543
+ async listArchiveMonths() {
2544
+ await this.ensureReady()
2545
+
2546
+ return await this._withDb(async (db) => {
2547
+ const months = (await db.getTables())
2548
+ .map((table) => ARCHIVE_TABLE_MONTH_PATTERN.exec(table.getName()))
2549
+ .filter((match) => match !== null)
2550
+ .map((match) => `${match[1]}-${match[2]}`)
2551
+
2552
+ return months.sort()
2553
+ })
2554
+ }
2555
+
2556
+ /**
2557
+ * Reads archived job rows from one monthly archive table.
2558
+ * @param {string} monthKey - Month key (`YYYY-MM`).
2559
+ * @returns {Promise<import("./types.js").BackgroundJobRow[]>} - Archived rows in id order.
2560
+ */
2561
+ async listArchivedJobs(monthKey) {
2562
+ await this.ensureReady()
2563
+
2564
+ if (!ARCHIVE_MONTH_KEY_PATTERN.test(monthKey)) throw new Error(`Invalid background jobs archive month key: ${monthKey}`)
2565
+
2566
+ return await this._withDb(async (db) => {
2567
+ const rows = await db
2568
+ .newQuery()
2569
+ .from(archiveTableNameForMonthKey(monthKey))
2570
+ .order({column: "id", direction: "ASC"})
2571
+ .results()
2572
+
2573
+ return rows.map((row) => this._normalizeJobRow(row))
2574
+ })
2575
+ }
2576
+
2577
+ /**
2578
+ * Reads per-job statistics for archived terminal jobs. The query surface
2579
+ * reads only the monthly archive tables — never `background_jobs` — so
2580
+ * statistics stay available no matter how small the active table is kept.
2581
+ * @param {object} [args] - Options.
2582
+ * @param {string[] | null} [args.months] - Restrict to these `YYYY-MM` keys. Default: every archive month present.
2583
+ * @param {string | null} [args.jobName] - Restrict to one archived job name.
2584
+ * @returns {Promise<import("./types.js").BackgroundJobsArchiveStatistic[]>} - One row per month and job name, ordered by month then job name.
2585
+ */
2586
+ async jobStatistics({jobName = null, months = null} = {}) {
2587
+ await this.ensureReady()
2588
+
2589
+ const requestedMonths = Array.isArray(months) && months.length > 0 ? [...months].sort() : await this.listArchiveMonths()
2590
+ /** @type {Map<string, {count: number, durations: number[], failureCount: number, jobName: string, month: string}>} */
2591
+ const statistics = new Map()
2592
+
2593
+ for (const monthKey of requestedMonths) {
2594
+ if (!ARCHIVE_MONTH_KEY_PATTERN.test(monthKey)) throw new Error(`Invalid background jobs archive month key: ${monthKey}`)
2595
+
2596
+ const rows = await this._withDb(async (db) => {
2597
+ const archiveTable = db.quoteTable(archiveTableNameForMonthKey(monthKey))
2598
+ const durationSql = `(COALESCE(${db.quoteColumn("completed_at_ms")}, ${db.quoteColumn("failed_at_ms")}, ${db.quoteColumn("orphaned_at_ms")}) - ${db.quoteColumn("created_at_ms")})`
2599
+ const query = db
2600
+ .newQuery()
2601
+ .from(archiveTable)
2602
+ .select("job_name")
2603
+ .select("status")
2604
+ .select(`${durationSql} AS duration_ms`)
2605
+
2606
+ if (jobName) query.where({job_name: jobName})
2607
+
2608
+ return await query.results()
2609
+ })
2610
+
2611
+ for (const row of rows) {
2612
+ const typedRow = /** @type {Record<string, ReturnType<typeof JSON.parse>>} */ (row)
2613
+ const rowJobName = String(typedRow.job_name)
2614
+ const key = `${monthKey}\u0000${rowJobName}`
2615
+ const entry = statistics.get(key) || {count: 0, durations: [], failureCount: 0, jobName: rowJobName, month: monthKey}
2616
+
2617
+ entry.count += 1
2618
+
2619
+ if (ARCHIVE_FAILURE_STATUSES.includes(String(typedRow.status))) entry.failureCount += 1
2620
+
2621
+ const durationMs = this._normalizeNumber(typedRow.duration_ms)
2622
+
2623
+ if (durationMs !== null) entry.durations.push(durationMs)
2624
+
2625
+ statistics.set(key, entry)
2626
+ }
2627
+ }
2628
+
2629
+ return [...statistics.values()]
2630
+ .sort((left, right) => left.month < right.month ? -1 : (left.month > right.month ? 1 : (left.jobName < right.jobName ? -1 : 1)))
2631
+ .map(({count, durations, failureCount, jobName: statisticJobName, month}) => ({
2632
+ count,
2633
+ failureCount,
2634
+ jobName: statisticJobName,
2635
+ month,
2636
+ p50DurationMs: archivePercentileMs(durations, 0.5),
2637
+ p95DurationMs: archivePercentileMs(durations, 0.95)
2638
+ }))
2639
+ }
2212
2640
 
2213
2641
  /**
2214
2642
  * Runs clear all.
@@ -316,6 +316,21 @@
316
316
  * @typedef {{type: "replace-scheduled-error", error?: string}} BackgroundJobReplaceScheduledErrorMessage
317
317
  * @typedef {{type: "cancel-scheduled", scheduleKey: string}} BackgroundJobCancelScheduledMessage
318
318
  * @typedef {{type: "schedule-cancelled", jobId: string | null, outcome: BackgroundJobCancellationOutcome}} BackgroundJobScheduleCancelledMessage
319
+ * @typedef {object} BackgroundJobsArchiveResult
320
+ * @property {number} archived - Total rows moved into monthly archive tables.
321
+ * @property {boolean} completedPass - Whether the pass drained every expired row, or stopped early on its time budget and will resume on the next sweep.
322
+ * @property {Record<string, number>} perMonth - Archived rows keyed by month (`YYYY-MM`).
323
+ */
324
+ /**
325
+ * @typedef {object} BackgroundJobsArchiveStatistic
326
+ * @property {number} count - Archived job rows for this month and job name.
327
+ * @property {number} failureCount - Archived rows with status `failed` or `orphaned`.
328
+ * @property {string} jobName - Archived job name.
329
+ * @property {string} month - Archive month (`YYYY-MM`).
330
+ * @property {number | null} p50DurationMs - Median terminal duration (terminal timestamp minus creation), or null when no row has both timestamps.
331
+ * @property {number | null} p95DurationMs - 95th percentile terminal duration, or null when no row has both timestamps.
332
+ */
333
+ /**
319
334
  * @typedef {{type: "cancel-scheduled-error", error?: string}} BackgroundJobCancelScheduledErrorMessage
320
335
  * @typedef {{type: "get-scheduled-job", scheduleKey: string, includeLatestTerminal?: boolean}} BackgroundJobGetScheduledMessage
321
336
  * @typedef {{type: "scheduled-job", currentJob: BackgroundJobRow | null, latestTerminalJob: BackgroundJobRow | null}} BackgroundJobScheduledMessage
@@ -291,6 +291,21 @@
291
291
  * @property {number} [batchSize] - Rows deleted per batch. Default: `1000`.
292
292
  * @property {number} [sweepIntervalMs] - How often the retention sweep runs.
293
293
  * Default: `3600000` (1 hour).
294
+ * @property {BackgroundJobsRetentionArchiveConfiguration} [archive] - Monthly
295
+ * archive tables. When enabled (the default), the retention sweep moves
296
+ * terminal rows into `background_jobs_YYYY_MM` statistics tables instead of
297
+ * deleting them, so history stays queryable while the active table stays
298
+ * small. When disabled, the sweep deletes terminal rows as before.
299
+ */
300
+
301
+ /**
302
+ * @typedef {object} BackgroundJobsRetentionArchiveConfiguration
303
+ * @property {boolean} [enabled] - Whether terminal retention moves rows into
304
+ * monthly archive tables instead of deleting them. Default: `true`.
305
+ * @property {number} [batchSize] - Max rows moved per batch. Default: `5000`.
306
+ * @property {number} [timeBudgetMs] - Wall-clock budget for one archival pass;
307
+ * a pass that runs out of budget stops early and resumes on the next
308
+ * scheduled sweep from the same page order. Default: `30000`.
294
309
  */
295
310
  /**
296
311
  * Fully-resolved retention config as returned by `getBackgroundJobsConfig()`
@@ -300,6 +315,7 @@
300
315
  * @property {number | null} failedTtlMs - Resolved failed/orphaned TTL in ms (`null` disables).
301
316
  * @property {number} batchSize - Resolved delete batch size.
302
317
  * @property {number} sweepIntervalMs - Resolved sweep interval in ms.
318
+ * @property {{batchSize: number, enabled: boolean, timeBudgetMs: number}} archive - Resolved monthly-archive options.
303
319
  */
304
320
 
305
321
  /**