velocious 1.0.748 → 1.0.749

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. package/build/background-jobs/adapter.js +7 -0
  2. package/build/background-jobs/queue-priority-order.js +32 -23
  3. package/build/background-jobs/store.js +443 -15
  4. package/build/background-jobs/types.js +15 -0
  5. package/build/configuration-types.js +16 -0
  6. package/build/configuration.js +14 -1
  7. package/build/database/table-data/table-column.js +5 -1
  8. package/build/jobs/prune-terminal-background-jobs.js +15 -1
  9. package/build/src/background-jobs/adapter.d.ts +11 -0
  10. package/build/src/background-jobs/adapter.d.ts.map +1 -1
  11. package/build/src/background-jobs/adapter.js +7 -1
  12. package/build/src/background-jobs/queue-priority-order.d.ts +28 -14
  13. package/build/src/background-jobs/queue-priority-order.d.ts.map +1 -1
  14. package/build/src/background-jobs/queue-priority-order.js +33 -23
  15. package/build/src/background-jobs/store.d.ts +155 -3
  16. package/build/src/background-jobs/store.d.ts.map +1 -1
  17. package/build/src/background-jobs/store.js +386 -14
  18. package/build/src/background-jobs/types.d.ts +55 -0
  19. package/build/src/background-jobs/types.d.ts.map +1 -1
  20. package/build/src/background-jobs/types.js +16 -1
  21. package/build/src/configuration-types.d.ts +48 -0
  22. package/build/src/configuration-types.d.ts.map +1 -1
  23. package/build/src/configuration-types.js +16 -1
  24. package/build/src/configuration.d.ts.map +1 -1
  25. package/build/src/configuration.js +14 -2
  26. package/build/src/database/table-data/table-column.d.ts.map +1 -1
  27. package/build/src/database/table-data/table-column.js +6 -2
  28. package/build/src/jobs/prune-terminal-background-jobs.d.ts +4 -1
  29. package/build/src/jobs/prune-terminal-background-jobs.d.ts.map +1 -1
  30. package/build/src/jobs/prune-terminal-background-jobs.js +14 -2
  31. package/package.json +1 -1
  32. package/src/background-jobs/adapter.js +7 -0
  33. package/src/background-jobs/queue-priority-order.js +32 -23
  34. package/src/background-jobs/store.js +443 -15
  35. package/src/background-jobs/types.js +15 -0
  36. package/src/configuration-types.js +16 -0
  37. package/src/configuration.js +14 -1
  38. package/src/database/table-data/table-column.js +5 -1
  39. package/src/jobs/prune-terminal-background-jobs.js +15 -1
@@ -172,6 +172,60 @@ const RETENTION_INDEX_DEFINITIONS = [
172
172
  {columns: ["status", "failed_at_ms", "id"], name: "index_background_jobs_failed_retention"},
173
173
  {columns: ["status", "orphaned_at_ms", "id"], name: "index_background_jobs_orphaned_retention"}
174
174
  ]
175
+ // Monthly terminal-job archive tables (`background_jobs_YYYY_MM`) keep the
176
+ // retention history queryable for statistics instead of dropping it. The month
177
+ // pattern also identifies which tables the statistics surface may read.
178
+ const ARCHIVE_TABLE_PREFIX = "background_jobs_"
179
+ const ARCHIVE_TABLE_MONTH_PATTERN = /^background_jobs_(\d{4})_(\d{2})$/
180
+ const ARCHIVE_MONTH_KEY_PATTERN = /^(\d{4})-(\d{2})$/
181
+ // Terminal statuses the archival pass moves into the monthly tables. Each
182
+ // status is discovered through its own terminal timestamp column, which also
183
+ // decides the destination month table. `cancelled` has no terminal timestamp
184
+ // column, so its creation time decides the month and its retention window.
185
+ const ARCHIVE_TERMINAL_STATUSES = [
186
+ {column: "completed_at_ms", status: "completed", ttlKey: "completedTtlMs"},
187
+ {column: "created_at_ms", status: "cancelled", ttlKey: "completedTtlMs"},
188
+ {column: "failed_at_ms", status: "failed", ttlKey: "failedTtlMs"},
189
+ {column: "orphaned_at_ms", status: "orphaned", ttlKey: "failedTtlMs"}
190
+ ]
191
+ // Archived statuses whose rows count as failures in the statistics surface.
192
+ const ARCHIVE_FAILURE_STATUSES = ["failed", "orphaned"]
193
+
194
+ /**
195
+ * Maps a terminal timestamp to its archive month key (`YYYY-MM`, UTC so the
196
+ * month boundary is identical on every process regardless of local timezone).
197
+ * @param {number | null} timestampMs - Terminal timestamp.
198
+ * @returns {string} - Month key.
199
+ */
200
+ function archiveMonthKeyForTimestampMs(timestampMs) {
201
+ const date = new Date(timestampMs === null ? 0 : timestampMs)
202
+ const month = String(date.getUTCMonth() + 1).padStart(2, "0")
203
+
204
+ return `${date.getUTCFullYear()}-${month}`
205
+ }
206
+
207
+ /**
208
+ * Runs archive table name for month key.
209
+ * @param {string} monthKey - Month key in `YYYY-MM` form.
210
+ * @returns {string} - Archive table name (`background_jobs_YYYY_MM`).
211
+ */
212
+ function archiveTableNameForMonthKey(monthKey) {
213
+ return `${ARCHIVE_TABLE_PREFIX}${monthKey.replace("-", "_")}`
214
+ }
215
+
216
+ /**
217
+ * Nearest-rank percentile over archived job durations.
218
+ * @param {number[]} values - Duration samples in ms.
219
+ * @param {number} quantile - Quantile in (0, 1].
220
+ * @returns {number | null} - Percentile value, or null when there are no samples.
221
+ */
222
+ function archivePercentileMs(values, quantile) {
223
+ if (values.length === 0) return null
224
+
225
+ const sorted = [...values].sort((left, right) => left - right)
226
+
227
+ return sorted[Math.max(0, Math.ceil(quantile * sorted.length) - 1)]
228
+ }
175
229
  const CONCURRENCY_TABLE = "background_job_concurrency"
176
230
  const COUNTS_REVISION_TABLE = "background_job_count_revisions"
177
231
  const COUNTS_REVISION_KEY = "counts"
@@ -242,6 +296,11 @@ const schemaApplyChains = new Map()
242
296
  const transactionMutationChains = new Map()
243
297
 
244
298
  export default class BackgroundJobsStore extends BackgroundJobsAdapter {
299
+ /**
300
+ * Copy column sets cached per archive table name.
301
+ * @type {Map<string, string[]>} */
302
+ _archiveColumnCacheByTable = new Map()
303
+
245
304
  /**
246
305
  * Runs constructor.
247
306
  * @param {object} args - Options.
@@ -1317,9 +1376,10 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
1317
1376
  * queue ranking uses: `priority * queuePriorityAgingUnitMs + (now - scheduled_at_ms) DESC`, so a
1318
1377
  * priority unit buys exactly that many milliseconds of queue-jumping and no more.
1319
1378
  *
1320
- * The primary rank caps each lane's age term at the gap to the highest configured priority, so
1321
- * every lane whose backlog has aged past that cap converges on the identical ceiling and this
1322
- * tie-break is the only term left that still reflects how long a row has actually waited. Written
1379
+ * The primary rank bounds each lane's age term at one aging unit and multiplies it by the lane's
1380
+ * gap to the highest configured priority, so every lane whose backlog has aged past that unit
1381
+ * converges on the identical ceiling and this tie-break is the only term left that still reflects
1382
+ * how long a row has actually waited. Written
1323
1383
  * as an absolute `priority DESC` the lead never decays: a row demoted to
1324
1384
  * {@link RETRY_BACKGROUND_JOB_PRIORITY} (-1) cannot outrank any priority-0 row in its lane however
1325
1385
  * long it waits, because its own age stopped counting once it passed the cap. Measured on
@@ -2165,18 +2225,7 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
2165
2225
  let deleted = 0
2166
2226
 
2167
2227
  for (;;) {
2168
- const candidates = await this._withDb(async (db) =>
2169
- await db
2170
- .newQuery()
2171
- .from(JOBS_TABLE)
2172
- .select("id")
2173
- .where({status})
2174
- .where(`${db.quoteColumn(column)} <= ${db.quote(cutoff)}`)
2175
- .order({column, direction: "ASC"})
2176
- .order({column: "id", direction: "ASC"})
2177
- .limit(batchSize)
2178
- .results()
2179
- )
2228
+ const candidates = await this._selectTerminalCandidatePage({batchSize, column, cutoff, status})
2180
2229
 
2181
2230
  if (candidates.length === 0) break
2182
2231
 
@@ -2209,6 +2258,385 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
2209
2258
 
2210
2259
  return deleted
2211
2260
  }
2261
+ /**
2262
+ * Selects one bounded retention candidate page ordered exactly like every
2263
+ * other retention pass (terminal timestamp ascending, then id ascending) so
2264
+ * the delete-only prune and the monthly archive sweeps converge on the same
2265
+ * page sequence. Discovery runs on a plain connection, never inside the
2266
+ * singleton count-revision mutation lock.
2267
+ * @param {object} args - Options.
2268
+ * @param {number} args.batchSize - Max rows per page.
2269
+ * @param {string} args.column - Terminal timestamp column compared against the cutoff.
2270
+ * @param {number} args.cutoff - Select rows whose column value is `<= cutoff`.
2271
+ * @param {string} args.status - Terminal status to select.
2272
+ * @param {boolean} [args.withTerminalTimestamp] - Also select the terminal timestamp under a stable alias.
2273
+ * @returns {Promise<Array<Record<string, ReturnType<typeof JSON.parse>>>>} - Candidate rows.
2274
+ */
2275
+ async _selectTerminalCandidatePage({batchSize, column, cutoff, status, withTerminalTimestamp = false}) {
2276
+ return await this._withDb(async (db) => {
2277
+ const query = db
2278
+ .newQuery()
2279
+ .from(JOBS_TABLE)
2280
+ .select("id")
2281
+ .where({status})
2282
+ .where(`${db.quoteColumn(column)} <= ${db.quote(cutoff)}`)
2283
+ .order({column, direction: "ASC"})
2284
+ .order({column: "id", direction: "ASC"})
2285
+ .limit(batchSize)
2286
+
2287
+ if (withTerminalTimestamp) query.select(`${db.quoteColumn(column)} AS terminal_timestamp_ms`)
2288
+
2289
+ return await query.results()
2290
+ })
2291
+ }
2292
+
2293
+ /**
2294
+ * Archives terminal job rows past their retention window into monthly
2295
+ * statistics tables (`background_jobs_YYYY_MM`, one per terminal-timestamp
2296
+ * month) instead of deleting them, so the active `background_jobs` table
2297
+ * stays small while history stays queryable. Like `pruneTerminalJobs`,
2298
+ * candidate discovery runs on a plain connection outside the count-revision
2299
+ * lock; only the short move transaction (copy into the month table, then
2300
+ * delete the same revalidated ids) is serialized, and the whole pass is
2301
+ * bounded by `batchSize` rows per batch and a wall-clock `timeBudgetMs`. A
2302
+ * pass that runs out of budget stops early and the next scheduled run
2303
+ * resumes from the same deterministic page order, so no unbounded DELETE is
2304
+ * ever issued.
2305
+ * @param {object} [args] - Options.
2306
+ * @param {number | null} [args.completedTtlMs] - Archive `completed` and `cancelled` jobs whose window timestamp is older than this many ms. Falsy or `<= 0` disables.
2307
+ * @param {number | null} [args.failedTtlMs] - Archive terminal `failed`/`orphaned` jobs older than this many ms. Falsy or `<= 0` disables.
2308
+ * @param {number} [args.batchSize] - Max rows moved per batch. Default `5000`.
2309
+ * @param {number} [args.timeBudgetMs] - Wall-clock budget for the whole pass. Default `30000`.
2310
+ * @returns {Promise<import("./types.js").BackgroundJobsArchiveResult>} - Rows archived, per-month totals and whether the pass finished or stopped on budget.
2311
+ */
2312
+ async archiveTerminalJobs({completedTtlMs = null, failedTtlMs = null, batchSize = 5000, timeBudgetMs = 30000} = {}) {
2313
+ await this.ensureReady()
2314
+
2315
+ const startedAtMs = this.clock.now()
2316
+ const size = batchSize > 0 ? batchSize : 5000
2317
+ const budgetMs = timeBudgetMs > 0 ? timeBudgetMs : 30000
2318
+ const deadlineMs = startedAtMs + budgetMs
2319
+ /** @type {Record<string, number>} */
2320
+ const perMonth = {}
2321
+ let archived = 0
2322
+ let completedPass = true
2323
+
2324
+ for (const {column, status, ttlKey} of ARCHIVE_TERMINAL_STATUSES) {
2325
+ const ttlMs = ttlKey === "completedTtlMs" ? completedTtlMs : failedTtlMs
2326
+
2327
+ if (typeof ttlMs !== "number" || ttlMs <= 0) continue
2328
+
2329
+ const result = await this._archiveStatusBatches({
2330
+ archivedAtMs: this.clock.now(),
2331
+ batchSize: size,
2332
+ column,
2333
+ deadlineMs,
2334
+ perMonth,
2335
+ status,
2336
+ cutoff: this.clock.now() - ttlMs
2337
+ })
2338
+
2339
+ archived += result.archived
2340
+
2341
+ if (!result.completedPass) {
2342
+ completedPass = false
2343
+ break
2344
+ }
2345
+ }
2346
+
2347
+ return {archived, completedPass, perMonth}
2348
+ }
2349
+
2350
+ /**
2351
+ * Moves rows of one terminal status older than a cutoff into their monthly
2352
+ * archive tables, batch by batch, until a candidate page is short or the
2353
+ * wall-clock budget is spent. Pages are ordered exactly like the delete-only
2354
+ * retention pass (terminal timestamp ascending, then id ascending), so a
2355
+ * pass that stopped on budget resumes on the same page sequence.
2356
+ * @param {object} args - Options.
2357
+ * @param {number} args.archivedAtMs - Timestamp stamped into `archived_at_ms`.
2358
+ * @param {number} args.batchSize - Max rows per batch.
2359
+ * @param {string} args.column - Terminal timestamp column compared against the cutoff.
2360
+ * @param {number} args.cutoff - Move rows whose column value is `<= cutoff`.
2361
+ * @param {number} args.deadlineMs - Stop starting new batches at this clock time.
2362
+ * @param {Record<string, number>} args.perMonth - Accumulated per-month archive totals.
2363
+ * @param {string} args.status - Terminal status to archive.
2364
+ * @returns {Promise<{archived: number, completedPass: boolean}>} - Status totals.
2365
+ */
2366
+ async _archiveStatusBatches({archivedAtMs, batchSize, column, cutoff, deadlineMs, perMonth, status}) {
2367
+ let archived = 0
2368
+
2369
+ for (;;) {
2370
+ if (this.clock.now() >= deadlineMs) return {archived, completedPass: false}
2371
+
2372
+ const candidates = await this._selectTerminalCandidatePage({batchSize, column, cutoff, status, withTerminalTimestamp: true})
2373
+
2374
+ if (candidates.length === 0) break
2375
+
2376
+ /** @type {Record<string, string[]>} */
2377
+ const idsByMonth = {}
2378
+
2379
+ for (const candidate of candidates) {
2380
+ const typedRow = /** @type {Record<string, ReturnType<typeof JSON.parse>>} */ (candidate)
2381
+ const monthKey = archiveMonthKeyForTimestampMs(this._normalizeNumber(typedRow.terminal_timestamp_ms))
2382
+ const monthIds = idsByMonth[monthKey] || (idsByMonth[monthKey] = [])
2383
+
2384
+ monthIds.push(String(typedRow.id))
2385
+ }
2386
+
2387
+ const months = Object.keys(idsByMonth).sort()
2388
+
2389
+ // Table creation is schema DDL: it runs on its own advisory lock before
2390
+ // the move transaction, never inside the count-revision serialization.
2391
+ for (const monthKey of months) {
2392
+ await this._ensureArchiveTable(monthKey)
2393
+ }
2394
+
2395
+ const moved = await this._serializedCountMutation(async (db) => {
2396
+ let statusMoved = 0
2397
+
2398
+ for (const monthKey of months) {
2399
+ const monthIds = idsByMonth[monthKey]
2400
+ const archiveTableName = archiveTableNameForMonthKey(monthKey)
2401
+ const archiveTable = db.quoteTable(archiveTableName)
2402
+ const columnSql = (await this._archiveColumnNames(db, archiveTableName)).map((columnName) => db.quoteColumn(columnName)).join(", ")
2403
+ const jobsTable = db.quoteTable(JOBS_TABLE)
2404
+ const idSql = monthIds.map((id) => db.quote(id)).join(", ")
2405
+
2406
+ await db.query(this._archiveInsertSql({archiveTable, archivedAtMs, column, columnSql, cutoff, db, idSql, jobsTable, status}))
2407
+
2408
+ const removed = await db.affectedRows(
2409
+ `DELETE FROM ${jobsTable} WHERE ${db.quoteColumn("id")} IN (${idSql}) AND ${db.quoteColumn("status")} = ${db.quote(status)} AND ${db.quoteColumn(column)} <= ${db.quote(cutoff)}`
2410
+ )
2411
+
2412
+ statusMoved += removed
2413
+ perMonth[monthKey] = (perMonth[monthKey] || 0) + removed
2414
+ }
2415
+
2416
+ // `cancelled` rows are not part of the counted status buckets, so only
2417
+ // bucket-counted statuses publish a signed delta.
2418
+ if (statusMoved > 0 && COUNTED_JOB_STATUSES.includes(status)) {
2419
+ await this._recordCountDelta(db, {all: -statusMoved, [status]: -statusMoved})
2420
+ }
2421
+
2422
+ return statusMoved
2423
+ })
2424
+
2425
+ archived += moved
2426
+ if (candidates.length < batchSize) break
2427
+ }
2428
+
2429
+ return {archived, completedPass: true}
2430
+ }
2431
+
2432
+ /**
2433
+ * Builds the driver-appropriate conflict-skipping `INSERT ... SELECT` that
2434
+ * copies one candidate page into a monthly archive table. The archive
2435
+ * primary key is the job id, so replaying a batch over rows that were
2436
+ * already archived is a no-op instead of a duplicate or an error — that is
2437
+ * what makes an interrupted move resumable.
2438
+ * @param {object} args - Options.
2439
+ * @param {string} args.archiveTable - Quoted archive table name.
2440
+ * @param {number} args.archivedAtMs - Value for `archived_at_ms`.
2441
+ * @param {string} args.column - Terminal timestamp column revalidated by the copy.
2442
+ * @param {string} args.columnSql - Comma-separated quoted job column names.
2443
+ * @param {number} args.cutoff - Copy rows whose column value is `<= cutoff`.
2444
+ * @param {import("../database/drivers/base.js").default} args.db - Database connection.
2445
+ * @param {string} args.idSql - Comma-separated quoted candidate ids.
2446
+ * @param {string} args.jobsTable - Quoted jobs table name.
2447
+ * @param {string} args.status - Terminal status revalidated by the copy.
2448
+ * @returns {string} - The insert statement.
2449
+ */
2450
+ _archiveInsertSql({archiveTable, archivedAtMs, column, columnSql, cutoff, db, idSql, jobsTable, status}) {
2451
+ const insertColumns = `${columnSql}, ${db.quoteColumn("archived_at_ms")}`
2452
+ const revalidationSql = `${db.quoteColumn("id")} IN (${idSql}) AND ${db.quoteColumn("status")} = ${db.quote(status)} AND ${db.quoteColumn(column)} <= ${db.quote(cutoff)}`
2453
+ const selectSql = `SELECT ${columnSql}, ${db.quote(archivedAtMs)} FROM ${jobsTable} WHERE ${revalidationSql}`
2454
+
2455
+ if (db.getType() === "mysql") return `INSERT IGNORE INTO ${archiveTable} (${insertColumns}) ${selectSql}`
2456
+ if (db.getType() === "sqlite") return `INSERT OR IGNORE INTO ${archiveTable} (${insertColumns}) ${selectSql}`
2457
+ if (db.getType() === "pgsql") return `INSERT INTO ${archiveTable} (${insertColumns}) ${selectSql} ON CONFLICT (${db.quoteColumn("id")}) DO NOTHING`
2458
+
2459
+ // SQL Server has no INSERT IGNORE clause: the anti-join expresses the same
2460
+ // "skip rows already archived under this primary key" contract.
2461
+ return `INSERT INTO ${archiveTable} (${insertColumns}) ${selectSql} AND NOT EXISTS (SELECT 1 FROM ${archiveTable} AS archived_rows WHERE archived_rows.${db.quoteColumn("id")} = ${jobsTable}.${db.quoteColumn("id")})`
2462
+ }
2463
+
2464
+ /**
2465
+ * Reads the copy column set from the archive table itself (every column
2466
+ * except `archived_at_ms`), so an archive copy always matches the schema the
2467
+ * destination table actually has instead of a hand-maintained duplicate list.
2468
+ * Cached per table name: the columns of an existing archive table never
2469
+ * change under the archival service.
2470
+ * @param {import("../database/drivers/base.js").default} db - Database connection.
2471
+ * @param {string} archiveTableName - Archive table name.
2472
+ * @returns {Promise<string[]>} - Column names in table order.
2473
+ */
2474
+ async _archiveColumnNames(db, archiveTableName) {
2475
+ const cache = this._archiveColumnCacheByTable
2476
+ const cached = cache.get(archiveTableName)
2477
+
2478
+ if (cached) return cached
2479
+
2480
+ const table = await db.getTableByNameOrFail(archiveTableName)
2481
+ const columnNames = (await table.getColumns())
2482
+ .map((column) => column.getName())
2483
+ .filter((columnName) => columnName !== "archived_at_ms")
2484
+
2485
+ cache.set(archiveTableName, columnNames)
2486
+
2487
+ return columnNames
2488
+ }
2489
+
2490
+ /**
2491
+ * Idempotently creates one monthly archive table mirroring the current
2492
+ * `background_jobs` columns plus `archived_at_ms`, with the job id as
2493
+ * primary key so replays are no-ops. Serialized by a per-month advisory
2494
+ * lock because several processes may activate the same month concurrently.
2495
+ * @param {string} monthKey - Month key in `YYYY-MM` form.
2496
+ * @returns {Promise<string>} - The archive table name.
2497
+ */
2498
+ async _ensureArchiveTable(monthKey) {
2499
+ if (!ARCHIVE_MONTH_KEY_PATTERN.test(monthKey)) throw new Error(`Invalid background jobs archive month key: ${monthKey}`)
2500
+
2501
+ const tableName = archiveTableNameForMonthKey(monthKey)
2502
+
2503
+ return await this._withDb(async (db) => {
2504
+ if (await db.tableExists(tableName)) return tableName
2505
+
2506
+ const lockName = `${MIGRATION_SCOPE}:archive_table:${monthKey}`
2507
+ const acquired = await db.acquireAdvisoryLock(lockName)
2508
+
2509
+ if (!acquired) throw new Error(`Failed to acquire background jobs archive table lock for ${monthKey}`)
2510
+
2511
+ try {
2512
+ db.clearSchemaCache()
2513
+ if (await db.tableExists(tableName)) return tableName
2514
+
2515
+ const jobsTable = await db.getTableByNameOrFail(JOBS_TABLE)
2516
+ const tableData = new TableData(tableName, {ifNotExists: true})
2517
+
2518
+ for (const column of await jobsTable.getColumns()) {
2519
+ tableData.addColumn(column.getTableDataColumn())
2520
+ }
2521
+
2522
+ tableData.bigint("archived_at_ms", {null: false})
2523
+
2524
+ await db.createTable(tableData)
2525
+
2526
+ for (const sql of await db.createIndexSQLs({columns: ["job_name"], ifNotExists: db.getType() === "sqlite", tableName})) {
2527
+ await db.query(sql)
2528
+ }
2529
+
2530
+ db.clearSchemaCache()
2531
+
2532
+ return tableName
2533
+ } finally {
2534
+ await db.releaseAdvisoryLock(lockName)
2535
+ }
2536
+ })
2537
+ }
2538
+
2539
+ /**
2540
+ * Lists the monthly archive table month keys present in this database.
2541
+ * @returns {Promise<string[]>} - Month keys (`YYYY-MM`) in ascending order.
2542
+ */
2543
+ async listArchiveMonths() {
2544
+ await this.ensureReady()
2545
+
2546
+ return await this._withDb(async (db) => {
2547
+ const months = (await db.getTables())
2548
+ .map((table) => ARCHIVE_TABLE_MONTH_PATTERN.exec(table.getName()))
2549
+ .filter((match) => match !== null)
2550
+ .map((match) => `${match[1]}-${match[2]}`)
2551
+
2552
+ return months.sort()
2553
+ })
2554
+ }
2555
+
2556
+ /**
2557
+ * Reads archived job rows from one monthly archive table.
2558
+ * @param {string} monthKey - Month key (`YYYY-MM`).
2559
+ * @returns {Promise<import("./types.js").BackgroundJobRow[]>} - Archived rows in id order.
2560
+ */
2561
+ async listArchivedJobs(monthKey) {
2562
+ await this.ensureReady()
2563
+
2564
+ if (!ARCHIVE_MONTH_KEY_PATTERN.test(monthKey)) throw new Error(`Invalid background jobs archive month key: ${monthKey}`)
2565
+
2566
+ return await this._withDb(async (db) => {
2567
+ const rows = await db
2568
+ .newQuery()
2569
+ .from(archiveTableNameForMonthKey(monthKey))
2570
+ .order({column: "id", direction: "ASC"})
2571
+ .results()
2572
+
2573
+ return rows.map((row) => this._normalizeJobRow(row))
2574
+ })
2575
+ }
2576
+
2577
+ /**
2578
+ * Reads per-job statistics for archived terminal jobs. The query surface
2579
+ * reads only the monthly archive tables — never `background_jobs` — so
2580
+ * statistics stay available no matter how small the active table is kept.
2581
+ * @param {object} [args] - Options.
2582
+ * @param {string[] | null} [args.months] - Restrict to these `YYYY-MM` keys. Default: every archive month present.
2583
+ * @param {string | null} [args.jobName] - Restrict to one archived job name.
2584
+ * @returns {Promise<import("./types.js").BackgroundJobsArchiveStatistic[]>} - One row per month and job name, ordered by month then job name.
2585
+ */
2586
+ async jobStatistics({jobName = null, months = null} = {}) {
2587
+ await this.ensureReady()
2588
+
2589
+ const requestedMonths = Array.isArray(months) && months.length > 0 ? [...months].sort() : await this.listArchiveMonths()
2590
+ /** @type {Map<string, {count: number, durations: number[], failureCount: number, jobName: string, month: string}>} */
2591
+ const statistics = new Map()
2592
+
2593
+ for (const monthKey of requestedMonths) {
2594
+ if (!ARCHIVE_MONTH_KEY_PATTERN.test(monthKey)) throw new Error(`Invalid background jobs archive month key: ${monthKey}`)
2595
+
2596
+ const rows = await this._withDb(async (db) => {
2597
+ const archiveTable = db.quoteTable(archiveTableNameForMonthKey(monthKey))
2598
+ const durationSql = `(COALESCE(${db.quoteColumn("completed_at_ms")}, ${db.quoteColumn("failed_at_ms")}, ${db.quoteColumn("orphaned_at_ms")}) - ${db.quoteColumn("created_at_ms")})`
2599
+ const query = db
2600
+ .newQuery()
2601
+ .from(archiveTable)
2602
+ .select("job_name")
2603
+ .select("status")
2604
+ .select(`${durationSql} AS duration_ms`)
2605
+
2606
+ if (jobName) query.where({job_name: jobName})
2607
+
2608
+ return await query.results()
2609
+ })
2610
+
2611
+ for (const row of rows) {
2612
+ const typedRow = /** @type {Record<string, ReturnType<typeof JSON.parse>>} */ (row)
2613
+ const rowJobName = String(typedRow.job_name)
2614
+ const key = `${monthKey}\u0000${rowJobName}`
2615
+ const entry = statistics.get(key) || {count: 0, durations: [], failureCount: 0, jobName: rowJobName, month: monthKey}
2616
+
2617
+ entry.count += 1
2618
+
2619
+ if (ARCHIVE_FAILURE_STATUSES.includes(String(typedRow.status))) entry.failureCount += 1
2620
+
2621
+ const durationMs = this._normalizeNumber(typedRow.duration_ms)
2622
+
2623
+ if (durationMs !== null) entry.durations.push(durationMs)
2624
+
2625
+ statistics.set(key, entry)
2626
+ }
2627
+ }
2628
+
2629
+ return [...statistics.values()]
2630
+ .sort((left, right) => left.month < right.month ? -1 : (left.month > right.month ? 1 : (left.jobName < right.jobName ? -1 : 1)))
2631
+ .map(({count, durations, failureCount, jobName: statisticJobName, month}) => ({
2632
+ count,
2633
+ failureCount,
2634
+ jobName: statisticJobName,
2635
+ month,
2636
+ p50DurationMs: archivePercentileMs(durations, 0.5),
2637
+ p95DurationMs: archivePercentileMs(durations, 0.95)
2638
+ }))
2639
+ }
2212
2640
 
2213
2641
  /**
2214
2642
  * Runs clear all.
@@ -316,6 +316,21 @@
316
316
  * @typedef {{type: "replace-scheduled-error", error?: string}} BackgroundJobReplaceScheduledErrorMessage
317
317
  * @typedef {{type: "cancel-scheduled", scheduleKey: string}} BackgroundJobCancelScheduledMessage
318
318
  * @typedef {{type: "schedule-cancelled", jobId: string | null, outcome: BackgroundJobCancellationOutcome}} BackgroundJobScheduleCancelledMessage
319
+ * @typedef {object} BackgroundJobsArchiveResult
320
+ * @property {number} archived - Total rows moved into monthly archive tables.
321
+ * @property {boolean} completedPass - Whether the pass drained every expired row, or stopped early on its time budget and will resume on the next sweep.
322
+ * @property {Record<string, number>} perMonth - Archived rows keyed by month (`YYYY-MM`).
323
+ */
324
+ /**
325
+ * @typedef {object} BackgroundJobsArchiveStatistic
326
+ * @property {number} count - Archived job rows for this month and job name.
327
+ * @property {number} failureCount - Archived rows with status `failed` or `orphaned`.
328
+ * @property {string} jobName - Archived job name.
329
+ * @property {string} month - Archive month (`YYYY-MM`).
330
+ * @property {number | null} p50DurationMs - Median terminal duration (terminal timestamp minus creation), or null when no row has both timestamps.
331
+ * @property {number | null} p95DurationMs - 95th percentile terminal duration, or null when no row has both timestamps.
332
+ */
333
+ /**
319
334
  * @typedef {{type: "cancel-scheduled-error", error?: string}} BackgroundJobCancelScheduledErrorMessage
320
335
  * @typedef {{type: "get-scheduled-job", scheduleKey: string, includeLatestTerminal?: boolean}} BackgroundJobGetScheduledMessage
321
336
  * @typedef {{type: "scheduled-job", currentJob: BackgroundJobRow | null, latestTerminalJob: BackgroundJobRow | null}} BackgroundJobScheduledMessage
@@ -291,6 +291,21 @@
291
291
  * @property {number} [batchSize] - Rows deleted per batch. Default: `1000`.
292
292
  * @property {number} [sweepIntervalMs] - How often the retention sweep runs.
293
293
  * Default: `3600000` (1 hour).
294
+ * @property {BackgroundJobsRetentionArchiveConfiguration} [archive] - Monthly
295
+ * archive tables. When enabled (the default), the retention sweep moves
296
+ * terminal rows into `background_jobs_YYYY_MM` statistics tables instead of
297
+ * deleting them, so history stays queryable while the active table stays
298
+ * small. When disabled, the sweep deletes terminal rows as before.
299
+ */
300
+
301
+ /**
302
+ * @typedef {object} BackgroundJobsRetentionArchiveConfiguration
303
+ * @property {boolean} [enabled] - Whether terminal retention moves rows into
304
+ * monthly archive tables instead of deleting them. Default: `true`.
305
+ * @property {number} [batchSize] - Max rows moved per batch. Default: `5000`.
306
+ * @property {number} [timeBudgetMs] - Wall-clock budget for one archival pass;
307
+ * a pass that runs out of budget stops early and resumes on the next
308
+ * scheduled sweep from the same page order. Default: `30000`.
294
309
  */
295
310
  /**
296
311
  * Fully-resolved retention config as returned by `getBackgroundJobsConfig()`
@@ -300,6 +315,7 @@
300
315
  * @property {number | null} failedTtlMs - Resolved failed/orphaned TTL in ms (`null` disables).
301
316
  * @property {number} batchSize - Resolved delete batch size.
302
317
  * @property {number} sweepIntervalMs - Resolved sweep interval in ms.
318
+ * @property {{batchSize: number, enabled: boolean, timeBudgetMs: number}} archive - Resolved monthly-archive options.
303
319
  */
304
320
 
305
321
  /**
@@ -1999,7 +1999,20 @@ export default class VelociousConfiguration {
1999
1999
  : 1000,
2000
2000
  sweepIntervalMs: typeof configuredRetention.sweepIntervalMs === "number" && configuredRetention.sweepIntervalMs > 0
2001
2001
  ? configuredRetention.sweepIntervalMs
2002
- : 60 * 60 * 1000
2002
+ : 60 * 60 * 1000,
2003
+ archive: (() => {
2004
+ const configuredArchive = configuredRetention.archive && typeof configuredRetention.archive === "object" ? configuredRetention.archive : {}
2005
+
2006
+ return {
2007
+ enabled: configuredArchive.enabled !== false,
2008
+ batchSize: typeof configuredArchive.batchSize === "number" && configuredArchive.batchSize > 0
2009
+ ? configuredArchive.batchSize
2010
+ : 5000,
2011
+ timeBudgetMs: typeof configuredArchive.timeBudgetMs === "number" && configuredArchive.timeBudgetMs > 0
2012
+ ? configuredArchive.timeBudgetMs
2013
+ : 30000
2014
+ }
2015
+ })()
2003
2016
  }
2004
2017
 
2005
2018
  const jobClasses = this.getBackgroundJobClasses()
@@ -352,7 +352,11 @@ export default class TableColumn {
352
352
  } else if (databaseType == "mysql" && type == "DATETIME") {
353
353
  sql += "(3)"
354
354
  } else if (type && maxlength !== undefined && maxlength !== null) {
355
- sql += `(${maxlength})`
355
+ // MSSQL metadata reports CHARACTER_MAXIMUM_LENGTH = -1 for max-length
356
+ // columns. That sentinel is not a length: emitting `NVARCHAR(-1)` is a
357
+ // syntax error, and a bare `NVARCHAR` would silently mean NVARCHAR(1).
358
+ // Emit the MAX form so copied columns retain their text contract.
359
+ sql += databaseType == "mssql" && maxlength == -1 ? "(MAX)" : `(${maxlength})`
356
360
  }
357
361
 
358
362
  if (this.getAutoIncrement() && driver.shouldSetAutoIncrementWhenPrimaryKey()) {
@@ -52,7 +52,10 @@ export default class PruneTerminalBackgroundJobsJob extends VelociousJob {
52
52
  }
53
53
 
54
54
  /**
55
- * Prunes terminal job rows past their retention window.
55
+ * Prunes terminal job rows past their retention window. With the monthly
56
+ * archive enabled (the default), the rows are moved into
57
+ * `background_jobs_YYYY_MM` statistics tables instead of deleted, so history
58
+ * stays queryable; with the archive disabled, they are deleted outright.
56
59
  * @returns {Promise<void>}
57
60
  */
58
61
  async perform() {
@@ -60,6 +63,17 @@ export default class PruneTerminalBackgroundJobsJob extends VelociousJob {
60
63
  const config = configuration.getBackgroundJobsConfig()
61
64
  const adapter = await configuration.acquireReadyBackgroundJobsAdapter()
62
65
 
66
+ if (config.retention.archive.enabled) {
67
+ await adapter.archiveTerminalJobs({
68
+ completedTtlMs: config.retention.completedTtlMs,
69
+ failedTtlMs: config.retention.failedTtlMs,
70
+ batchSize: config.retention.archive.batchSize,
71
+ timeBudgetMs: config.retention.archive.timeBudgetMs
72
+ })
73
+
74
+ return
75
+ }
76
+
63
77
  await adapter.pruneTerminalJobs({
64
78
  completedTtlMs: config.retention.completedTtlMs,
65
79
  failedTtlMs: config.retention.failedTtlMs,