velocious 1.0.747 → 1.0.749

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. package/README.md +1 -1
  2. package/build/background-jobs/adapter.js +7 -0
  3. package/build/background-jobs/pooled-runner-child.js +41 -4
  4. package/build/background-jobs/pooled-runner-shutdown.js +92 -0
  5. package/build/background-jobs/queue-priority-order.js +32 -23
  6. package/build/background-jobs/store.js +443 -15
  7. package/build/background-jobs/types.js +31 -0
  8. package/build/background-jobs/worker.js +56 -3
  9. package/build/configuration-types.js +16 -0
  10. package/build/configuration.js +14 -1
  11. package/build/database/table-data/table-column.js +5 -1
  12. package/build/frontend-model-controller.js +41 -0
  13. package/build/frontend-models/query.js +42 -0
  14. package/build/jobs/prune-terminal-background-jobs.js +15 -1
  15. package/build/src/background-jobs/adapter.d.ts +11 -0
  16. package/build/src/background-jobs/adapter.d.ts.map +1 -1
  17. package/build/src/background-jobs/adapter.js +7 -1
  18. package/build/src/background-jobs/pooled-runner-child.js +39 -5
  19. package/build/src/background-jobs/pooled-runner-shutdown.d.ts +22 -0
  20. package/build/src/background-jobs/pooled-runner-shutdown.d.ts.map +1 -1
  21. package/build/src/background-jobs/pooled-runner-shutdown.js +87 -1
  22. package/build/src/background-jobs/queue-priority-order.d.ts +28 -14
  23. package/build/src/background-jobs/queue-priority-order.d.ts.map +1 -1
  24. package/build/src/background-jobs/queue-priority-order.js +33 -23
  25. package/build/src/background-jobs/store.d.ts +155 -3
  26. package/build/src/background-jobs/store.d.ts.map +1 -1
  27. package/build/src/background-jobs/store.js +386 -14
  28. package/build/src/background-jobs/types.d.ts +102 -0
  29. package/build/src/background-jobs/types.d.ts.map +1 -1
  30. package/build/src/background-jobs/types.js +32 -1
  31. package/build/src/background-jobs/worker.d.ts +16 -0
  32. package/build/src/background-jobs/worker.d.ts.map +1 -1
  33. package/build/src/background-jobs/worker.js +56 -4
  34. package/build/src/configuration-types.d.ts +48 -0
  35. package/build/src/configuration-types.d.ts.map +1 -1
  36. package/build/src/configuration-types.js +16 -1
  37. package/build/src/configuration.d.ts.map +1 -1
  38. package/build/src/configuration.js +14 -2
  39. package/build/src/database/table-data/table-column.d.ts.map +1 -1
  40. package/build/src/database/table-data/table-column.js +6 -2
  41. package/build/src/frontend-model-controller.d.ts +4 -0
  42. package/build/src/frontend-model-controller.d.ts.map +1 -1
  43. package/build/src/frontend-model-controller.js +34 -1
  44. package/build/src/frontend-models/query.d.ts +4 -0
  45. package/build/src/frontend-models/query.d.ts.map +1 -1
  46. package/build/src/frontend-models/query.js +38 -1
  47. package/build/src/jobs/prune-terminal-background-jobs.d.ts +4 -1
  48. package/build/src/jobs/prune-terminal-background-jobs.d.ts.map +1 -1
  49. package/build/src/jobs/prune-terminal-background-jobs.js +14 -2
  50. package/build/src/sync/sync-client.d.ts.map +1 -1
  51. package/build/src/sync/sync-client.js +2 -1
  52. package/build/src/sync/sync-offline-session.d.ts.map +1 -1
  53. package/build/src/sync/sync-offline-session.js +9 -3
  54. package/build/sync/sync-client.js +1 -0
  55. package/build/sync/sync-offline-session.js +9 -1
  56. package/package.json +1 -1
  57. package/src/background-jobs/adapter.js +7 -0
  58. package/src/background-jobs/pooled-runner-child.js +41 -4
  59. package/src/background-jobs/pooled-runner-shutdown.js +92 -0
  60. package/src/background-jobs/queue-priority-order.js +32 -23
  61. package/src/background-jobs/store.js +443 -15
  62. package/src/background-jobs/types.js +31 -0
  63. package/src/background-jobs/worker.js +56 -3
  64. package/src/configuration-types.js +16 -0
  65. package/src/configuration.js +14 -1
  66. package/src/database/table-data/table-column.js +5 -1
  67. package/src/frontend-model-controller.js +41 -0
  68. package/src/frontend-models/query.js +42 -0
  69. package/src/jobs/prune-terminal-background-jobs.js +15 -1
  70. package/src/sync/sync-client.js +1 -0
  71. package/src/sync/sync-offline-session.js +9 -1
@@ -172,6 +172,60 @@ const RETENTION_INDEX_DEFINITIONS = [
172
172
  {columns: ["status", "failed_at_ms", "id"], name: "index_background_jobs_failed_retention"},
173
173
  {columns: ["status", "orphaned_at_ms", "id"], name: "index_background_jobs_orphaned_retention"}
174
174
  ]
175
+ // Monthly terminal-job archive tables (`background_jobs_YYYY_MM`) keep the
176
+ // retention history queryable for statistics instead of dropping it. The month
177
+ // pattern also identifies which tables the statistics surface may read.
178
+ const ARCHIVE_TABLE_PREFIX = "background_jobs_"
179
+ const ARCHIVE_TABLE_MONTH_PATTERN = /^background_jobs_(\d{4})_(\d{2})$/
180
+ const ARCHIVE_MONTH_KEY_PATTERN = /^(\d{4})-(\d{2})$/
181
+ // Terminal statuses the archival pass moves into the monthly tables. Each
182
+ // status is discovered through its own terminal timestamp column, which also
183
+ // decides the destination month table. `cancelled` has no terminal timestamp
184
+ // column, so its creation time decides the month and its retention window.
185
+ const ARCHIVE_TERMINAL_STATUSES = [
186
+ {column: "completed_at_ms", status: "completed", ttlKey: "completedTtlMs"},
187
+ {column: "created_at_ms", status: "cancelled", ttlKey: "completedTtlMs"},
188
+ {column: "failed_at_ms", status: "failed", ttlKey: "failedTtlMs"},
189
+ {column: "orphaned_at_ms", status: "orphaned", ttlKey: "failedTtlMs"}
190
+ ]
191
+ // Archived statuses whose rows count as failures in the statistics surface.
192
+ const ARCHIVE_FAILURE_STATUSES = ["failed", "orphaned"]
193
+
194
+ /**
195
+ * Maps a terminal timestamp to its archive month key (`YYYY-MM`, UTC so the
196
+ * month boundary is identical on every process regardless of local timezone).
197
+ * @param {number | null} timestampMs - Terminal timestamp.
198
+ * @returns {string} - Month key.
199
+ */
200
+ function archiveMonthKeyForTimestampMs(timestampMs) {
201
+ const date = new Date(timestampMs === null ? 0 : timestampMs)
202
+ const month = String(date.getUTCMonth() + 1).padStart(2, "0")
203
+
204
+ return `${date.getUTCFullYear()}-${month}`
205
+ }
206
+
207
+ /**
208
+ * Runs archive table name for month key.
209
+ * @param {string} monthKey - Month key in `YYYY-MM` form.
210
+ * @returns {string} - Archive table name (`background_jobs_YYYY_MM`).
211
+ */
212
+ function archiveTableNameForMonthKey(monthKey) {
213
+ return `${ARCHIVE_TABLE_PREFIX}${monthKey.replace("-", "_")}`
214
+ }
215
+
216
+ /**
217
+ * Nearest-rank percentile over archived job durations.
218
+ * @param {number[]} values - Duration samples in ms.
219
+ * @param {number} quantile - Quantile in (0, 1].
220
+ * @returns {number | null} - Percentile value, or null when there are no samples.
221
+ */
222
+ function archivePercentileMs(values, quantile) {
223
+ if (values.length === 0) return null
224
+
225
+ const sorted = [...values].sort((left, right) => left - right)
226
+
227
+ return sorted[Math.max(0, Math.ceil(quantile * sorted.length) - 1)]
228
+ }
175
229
  const CONCURRENCY_TABLE = "background_job_concurrency"
176
230
  const COUNTS_REVISION_TABLE = "background_job_count_revisions"
177
231
  const COUNTS_REVISION_KEY = "counts"
@@ -242,6 +296,11 @@ const schemaApplyChains = new Map()
242
296
  const transactionMutationChains = new Map()
243
297
 
244
298
  export default class BackgroundJobsStore extends BackgroundJobsAdapter {
299
+ /**
300
+ * Copy column sets cached per archive table name.
301
+ * @type {Map<string, string[]>} */
302
+ _archiveColumnCacheByTable = new Map()
303
+
245
304
  /**
246
305
  * Runs constructor.
247
306
  * @param {object} args - Options.
@@ -1317,9 +1376,10 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
1317
1376
  * queue ranking uses: `priority * queuePriorityAgingUnitMs + (now - scheduled_at_ms) DESC`, so a
1318
1377
  * priority unit buys exactly that many milliseconds of queue-jumping and no more.
1319
1378
  *
1320
- * The primary rank caps each lane's age term at the gap to the highest configured priority, so
1321
- * every lane whose backlog has aged past that cap converges on the identical ceiling and this
1322
- * tie-break is the only term left that still reflects how long a row has actually waited. Written
1379
+ * The primary rank bounds each lane's age term at one aging unit and multiplies it by the lane's
1380
+ * gap to the highest configured priority, so every lane whose backlog has aged past that unit
1381
+ * converges on the identical ceiling and this tie-break is the only term left that still reflects
1382
+ * how long a row has actually waited. Written
1323
1383
  * as an absolute `priority DESC` the lead never decays: a row demoted to
1324
1384
  * {@link RETRY_BACKGROUND_JOB_PRIORITY} (-1) cannot outrank any priority-0 row in its lane however
1325
1385
  * long it waits, because its own age stopped counting once it passed the cap. Measured on
@@ -2165,18 +2225,7 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
2165
2225
  let deleted = 0
2166
2226
 
2167
2227
  for (;;) {
2168
- const candidates = await this._withDb(async (db) =>
2169
- await db
2170
- .newQuery()
2171
- .from(JOBS_TABLE)
2172
- .select("id")
2173
- .where({status})
2174
- .where(`${db.quoteColumn(column)} <= ${db.quote(cutoff)}`)
2175
- .order({column, direction: "ASC"})
2176
- .order({column: "id", direction: "ASC"})
2177
- .limit(batchSize)
2178
- .results()
2179
- )
2228
+ const candidates = await this._selectTerminalCandidatePage({batchSize, column, cutoff, status})
2180
2229
 
2181
2230
  if (candidates.length === 0) break
2182
2231
 
@@ -2209,6 +2258,385 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
2209
2258
 
2210
2259
  return deleted
2211
2260
  }
2261
+ /**
2262
+ * Selects one bounded retention candidate page ordered exactly like every
2263
+ * other retention pass (terminal timestamp ascending, then id ascending) so
2264
+ * the delete-only prune and the monthly archive sweeps converge on the same
2265
+ * page sequence. Discovery runs on a plain connection, never inside the
2266
+ * singleton count-revision mutation lock.
2267
+ * @param {object} args - Options.
2268
+ * @param {number} args.batchSize - Max rows per page.
2269
+ * @param {string} args.column - Terminal timestamp column compared against the cutoff.
2270
+ * @param {number} args.cutoff - Select rows whose column value is `<= cutoff`.
2271
+ * @param {string} args.status - Terminal status to select.
2272
+ * @param {boolean} [args.withTerminalTimestamp] - Also select the terminal timestamp under a stable alias.
2273
+ * @returns {Promise<Array<Record<string, ReturnType<typeof JSON.parse>>>>} - Candidate rows.
2274
+ */
2275
+ async _selectTerminalCandidatePage({batchSize, column, cutoff, status, withTerminalTimestamp = false}) {
2276
+ return await this._withDb(async (db) => {
2277
+ const query = db
2278
+ .newQuery()
2279
+ .from(JOBS_TABLE)
2280
+ .select("id")
2281
+ .where({status})
2282
+ .where(`${db.quoteColumn(column)} <= ${db.quote(cutoff)}`)
2283
+ .order({column, direction: "ASC"})
2284
+ .order({column: "id", direction: "ASC"})
2285
+ .limit(batchSize)
2286
+
2287
+ if (withTerminalTimestamp) query.select(`${db.quoteColumn(column)} AS terminal_timestamp_ms`)
2288
+
2289
+ return await query.results()
2290
+ })
2291
+ }
2292
+
2293
+ /**
2294
+ * Archives terminal job rows past their retention window into monthly
2295
+ * statistics tables (`background_jobs_YYYY_MM`, one per terminal-timestamp
2296
+ * month) instead of deleting them, so the active `background_jobs` table
2297
+ * stays small while history stays queryable. Like `pruneTerminalJobs`,
2298
+ * candidate discovery runs on a plain connection outside the count-revision
2299
+ * lock; only the short move transaction (copy into the month table, then
2300
+ * delete the same revalidated ids) is serialized, and the whole pass is
2301
+ * bounded by `batchSize` rows per batch and a wall-clock `timeBudgetMs`. A
2302
+ * pass that runs out of budget stops early and the next scheduled run
2303
+ * resumes from the same deterministic page order, so no unbounded DELETE is
2304
+ * ever issued.
2305
+ * @param {object} [args] - Options.
2306
+ * @param {number | null} [args.completedTtlMs] - Archive `completed` and `cancelled` jobs whose window timestamp is older than this many ms. Falsy or `<= 0` disables.
2307
+ * @param {number | null} [args.failedTtlMs] - Archive terminal `failed`/`orphaned` jobs older than this many ms. Falsy or `<= 0` disables.
2308
+ * @param {number} [args.batchSize] - Max rows moved per batch. Default `5000`.
2309
+ * @param {number} [args.timeBudgetMs] - Wall-clock budget for the whole pass. Default `30000`.
2310
+ * @returns {Promise<import("./types.js").BackgroundJobsArchiveResult>} - Rows archived, per-month totals and whether the pass finished or stopped on budget.
2311
+ */
2312
+ async archiveTerminalJobs({completedTtlMs = null, failedTtlMs = null, batchSize = 5000, timeBudgetMs = 30000} = {}) {
2313
+ await this.ensureReady()
2314
+
2315
+ const startedAtMs = this.clock.now()
2316
+ const size = batchSize > 0 ? batchSize : 5000
2317
+ const budgetMs = timeBudgetMs > 0 ? timeBudgetMs : 30000
2318
+ const deadlineMs = startedAtMs + budgetMs
2319
+ /** @type {Record<string, number>} */
2320
+ const perMonth = {}
2321
+ let archived = 0
2322
+ let completedPass = true
2323
+
2324
+ for (const {column, status, ttlKey} of ARCHIVE_TERMINAL_STATUSES) {
2325
+ const ttlMs = ttlKey === "completedTtlMs" ? completedTtlMs : failedTtlMs
2326
+
2327
+ if (typeof ttlMs !== "number" || ttlMs <= 0) continue
2328
+
2329
+ const result = await this._archiveStatusBatches({
2330
+ archivedAtMs: this.clock.now(),
2331
+ batchSize: size,
2332
+ column,
2333
+ deadlineMs,
2334
+ perMonth,
2335
+ status,
2336
+ cutoff: this.clock.now() - ttlMs
2337
+ })
2338
+
2339
+ archived += result.archived
2340
+
2341
+ if (!result.completedPass) {
2342
+ completedPass = false
2343
+ break
2344
+ }
2345
+ }
2346
+
2347
+ return {archived, completedPass, perMonth}
2348
+ }
2349
+
2350
+ /**
2351
+ * Moves rows of one terminal status older than a cutoff into their monthly
2352
+ * archive tables, batch by batch, until a candidate page is short or the
2353
+ * wall-clock budget is spent. Pages are ordered exactly like the delete-only
2354
+ * retention pass (terminal timestamp ascending, then id ascending), so a
2355
+ * pass that stopped on budget resumes on the same page sequence.
2356
+ * @param {object} args - Options.
2357
+ * @param {number} args.archivedAtMs - Timestamp stamped into `archived_at_ms`.
2358
+ * @param {number} args.batchSize - Max rows per batch.
2359
+ * @param {string} args.column - Terminal timestamp column compared against the cutoff.
2360
+ * @param {number} args.cutoff - Move rows whose column value is `<= cutoff`.
2361
+ * @param {number} args.deadlineMs - Stop starting new batches at this clock time.
2362
+ * @param {Record<string, number>} args.perMonth - Accumulated per-month archive totals.
2363
+ * @param {string} args.status - Terminal status to archive.
2364
+ * @returns {Promise<{archived: number, completedPass: boolean}>} - Status totals.
2365
+ */
2366
+ async _archiveStatusBatches({archivedAtMs, batchSize, column, cutoff, deadlineMs, perMonth, status}) {
2367
+ let archived = 0
2368
+
2369
+ for (;;) {
2370
+ if (this.clock.now() >= deadlineMs) return {archived, completedPass: false}
2371
+
2372
+ const candidates = await this._selectTerminalCandidatePage({batchSize, column, cutoff, status, withTerminalTimestamp: true})
2373
+
2374
+ if (candidates.length === 0) break
2375
+
2376
+ /** @type {Record<string, string[]>} */
2377
+ const idsByMonth = {}
2378
+
2379
+ for (const candidate of candidates) {
2380
+ const typedRow = /** @type {Record<string, ReturnType<typeof JSON.parse>>} */ (candidate)
2381
+ const monthKey = archiveMonthKeyForTimestampMs(this._normalizeNumber(typedRow.terminal_timestamp_ms))
2382
+ const monthIds = idsByMonth[monthKey] || (idsByMonth[monthKey] = [])
2383
+
2384
+ monthIds.push(String(typedRow.id))
2385
+ }
2386
+
2387
+ const months = Object.keys(idsByMonth).sort()
2388
+
2389
+ // Table creation is schema DDL: it runs on its own advisory lock before
2390
+ // the move transaction, never inside the count-revision serialization.
2391
+ for (const monthKey of months) {
2392
+ await this._ensureArchiveTable(monthKey)
2393
+ }
2394
+
2395
+ const moved = await this._serializedCountMutation(async (db) => {
2396
+ let statusMoved = 0
2397
+
2398
+ for (const monthKey of months) {
2399
+ const monthIds = idsByMonth[monthKey]
2400
+ const archiveTableName = archiveTableNameForMonthKey(monthKey)
2401
+ const archiveTable = db.quoteTable(archiveTableName)
2402
+ const columnSql = (await this._archiveColumnNames(db, archiveTableName)).map((columnName) => db.quoteColumn(columnName)).join(", ")
2403
+ const jobsTable = db.quoteTable(JOBS_TABLE)
2404
+ const idSql = monthIds.map((id) => db.quote(id)).join(", ")
2405
+
2406
+ await db.query(this._archiveInsertSql({archiveTable, archivedAtMs, column, columnSql, cutoff, db, idSql, jobsTable, status}))
2407
+
2408
+ const removed = await db.affectedRows(
2409
+ `DELETE FROM ${jobsTable} WHERE ${db.quoteColumn("id")} IN (${idSql}) AND ${db.quoteColumn("status")} = ${db.quote(status)} AND ${db.quoteColumn(column)} <= ${db.quote(cutoff)}`
2410
+ )
2411
+
2412
+ statusMoved += removed
2413
+ perMonth[monthKey] = (perMonth[monthKey] || 0) + removed
2414
+ }
2415
+
2416
+ // `cancelled` rows are not part of the counted status buckets, so only
2417
+ // bucket-counted statuses publish a signed delta.
2418
+ if (statusMoved > 0 && COUNTED_JOB_STATUSES.includes(status)) {
2419
+ await this._recordCountDelta(db, {all: -statusMoved, [status]: -statusMoved})
2420
+ }
2421
+
2422
+ return statusMoved
2423
+ })
2424
+
2425
+ archived += moved
2426
+ if (candidates.length < batchSize) break
2427
+ }
2428
+
2429
+ return {archived, completedPass: true}
2430
+ }
2431
+
2432
+ /**
2433
+ * Builds the driver-appropriate conflict-skipping `INSERT ... SELECT` that
2434
+ * copies one candidate page into a monthly archive table. The archive
2435
+ * primary key is the job id, so replaying a batch over rows that were
2436
+ * already archived is a no-op instead of a duplicate or an error — that is
2437
+ * what makes an interrupted move resumable.
2438
+ * @param {object} args - Options.
2439
+ * @param {string} args.archiveTable - Quoted archive table name.
2440
+ * @param {number} args.archivedAtMs - Value for `archived_at_ms`.
2441
+ * @param {string} args.column - Terminal timestamp column revalidated by the copy.
2442
+ * @param {string} args.columnSql - Comma-separated quoted job column names.
2443
+ * @param {number} args.cutoff - Copy rows whose column value is `<= cutoff`.
2444
+ * @param {import("../database/drivers/base.js").default} args.db - Database connection.
2445
+ * @param {string} args.idSql - Comma-separated quoted candidate ids.
2446
+ * @param {string} args.jobsTable - Quoted jobs table name.
2447
+ * @param {string} args.status - Terminal status revalidated by the copy.
2448
+ * @returns {string} - The insert statement.
2449
+ */
2450
+ _archiveInsertSql({archiveTable, archivedAtMs, column, columnSql, cutoff, db, idSql, jobsTable, status}) {
2451
+ const insertColumns = `${columnSql}, ${db.quoteColumn("archived_at_ms")}`
2452
+ const revalidationSql = `${db.quoteColumn("id")} IN (${idSql}) AND ${db.quoteColumn("status")} = ${db.quote(status)} AND ${db.quoteColumn(column)} <= ${db.quote(cutoff)}`
2453
+ const selectSql = `SELECT ${columnSql}, ${db.quote(archivedAtMs)} FROM ${jobsTable} WHERE ${revalidationSql}`
2454
+
2455
+ if (db.getType() === "mysql") return `INSERT IGNORE INTO ${archiveTable} (${insertColumns}) ${selectSql}`
2456
+ if (db.getType() === "sqlite") return `INSERT OR IGNORE INTO ${archiveTable} (${insertColumns}) ${selectSql}`
2457
+ if (db.getType() === "pgsql") return `INSERT INTO ${archiveTable} (${insertColumns}) ${selectSql} ON CONFLICT (${db.quoteColumn("id")}) DO NOTHING`
2458
+
2459
+ // SQL Server has no INSERT IGNORE clause: the anti-join expresses the same
2460
+ // "skip rows already archived under this primary key" contract.
2461
+ return `INSERT INTO ${archiveTable} (${insertColumns}) ${selectSql} AND NOT EXISTS (SELECT 1 FROM ${archiveTable} AS archived_rows WHERE archived_rows.${db.quoteColumn("id")} = ${jobsTable}.${db.quoteColumn("id")})`
2462
+ }
2463
+
2464
+ /**
2465
+ * Reads the copy column set from the archive table itself (every column
2466
+ * except `archived_at_ms`), so an archive copy always matches the schema the
2467
+ * destination table actually has instead of a hand-maintained duplicate list.
2468
+ * Cached per table name: the columns of an existing archive table never
2469
+ * change under the archival service.
2470
+ * @param {import("../database/drivers/base.js").default} db - Database connection.
2471
+ * @param {string} archiveTableName - Archive table name.
2472
+ * @returns {Promise<string[]>} - Column names in table order.
2473
+ */
2474
+ async _archiveColumnNames(db, archiveTableName) {
2475
+ const cache = this._archiveColumnCacheByTable
2476
+ const cached = cache.get(archiveTableName)
2477
+
2478
+ if (cached) return cached
2479
+
2480
+ const table = await db.getTableByNameOrFail(archiveTableName)
2481
+ const columnNames = (await table.getColumns())
2482
+ .map((column) => column.getName())
2483
+ .filter((columnName) => columnName !== "archived_at_ms")
2484
+
2485
+ cache.set(archiveTableName, columnNames)
2486
+
2487
+ return columnNames
2488
+ }
2489
+
2490
+ /**
2491
+ * Idempotently creates one monthly archive table mirroring the current
2492
+ * `background_jobs` columns plus `archived_at_ms`, with the job id as
2493
+ * primary key so replays are no-ops. Serialized by a per-month advisory
2494
+ * lock because several processes may activate the same month concurrently.
2495
+ * @param {string} monthKey - Month key in `YYYY-MM` form.
2496
+ * @returns {Promise<string>} - The archive table name.
2497
+ */
2498
+ async _ensureArchiveTable(monthKey) {
2499
+ if (!ARCHIVE_MONTH_KEY_PATTERN.test(monthKey)) throw new Error(`Invalid background jobs archive month key: ${monthKey}`)
2500
+
2501
+ const tableName = archiveTableNameForMonthKey(monthKey)
2502
+
2503
+ return await this._withDb(async (db) => {
2504
+ if (await db.tableExists(tableName)) return tableName
2505
+
2506
+ const lockName = `${MIGRATION_SCOPE}:archive_table:${monthKey}`
2507
+ const acquired = await db.acquireAdvisoryLock(lockName)
2508
+
2509
+ if (!acquired) throw new Error(`Failed to acquire background jobs archive table lock for ${monthKey}`)
2510
+
2511
+ try {
2512
+ db.clearSchemaCache()
2513
+ if (await db.tableExists(tableName)) return tableName
2514
+
2515
+ const jobsTable = await db.getTableByNameOrFail(JOBS_TABLE)
2516
+ const tableData = new TableData(tableName, {ifNotExists: true})
2517
+
2518
+ for (const column of await jobsTable.getColumns()) {
2519
+ tableData.addColumn(column.getTableDataColumn())
2520
+ }
2521
+
2522
+ tableData.bigint("archived_at_ms", {null: false})
2523
+
2524
+ await db.createTable(tableData)
2525
+
2526
+ for (const sql of await db.createIndexSQLs({columns: ["job_name"], ifNotExists: db.getType() === "sqlite", tableName})) {
2527
+ await db.query(sql)
2528
+ }
2529
+
2530
+ db.clearSchemaCache()
2531
+
2532
+ return tableName
2533
+ } finally {
2534
+ await db.releaseAdvisoryLock(lockName)
2535
+ }
2536
+ })
2537
+ }
2538
+
2539
+ /**
2540
+ * Lists the monthly archive table month keys present in this database.
2541
+ * @returns {Promise<string[]>} - Month keys (`YYYY-MM`) in ascending order.
2542
+ */
2543
+ async listArchiveMonths() {
2544
+ await this.ensureReady()
2545
+
2546
+ return await this._withDb(async (db) => {
2547
+ const months = (await db.getTables())
2548
+ .map((table) => ARCHIVE_TABLE_MONTH_PATTERN.exec(table.getName()))
2549
+ .filter((match) => match !== null)
2550
+ .map((match) => `${match[1]}-${match[2]}`)
2551
+
2552
+ return months.sort()
2553
+ })
2554
+ }
2555
+
2556
+ /**
2557
+ * Reads archived job rows from one monthly archive table.
2558
+ * @param {string} monthKey - Month key (`YYYY-MM`).
2559
+ * @returns {Promise<import("./types.js").BackgroundJobRow[]>} - Archived rows in id order.
2560
+ */
2561
+ async listArchivedJobs(monthKey) {
2562
+ await this.ensureReady()
2563
+
2564
+ if (!ARCHIVE_MONTH_KEY_PATTERN.test(monthKey)) throw new Error(`Invalid background jobs archive month key: ${monthKey}`)
2565
+
2566
+ return await this._withDb(async (db) => {
2567
+ const rows = await db
2568
+ .newQuery()
2569
+ .from(archiveTableNameForMonthKey(monthKey))
2570
+ .order({column: "id", direction: "ASC"})
2571
+ .results()
2572
+
2573
+ return rows.map((row) => this._normalizeJobRow(row))
2574
+ })
2575
+ }
2576
+
2577
+ /**
2578
+ * Reads per-job statistics for archived terminal jobs. The query surface
2579
+ * reads only the monthly archive tables — never `background_jobs` — so
2580
+ * statistics stay available no matter how small the active table is kept.
2581
+ * @param {object} [args] - Options.
2582
+ * @param {string[] | null} [args.months] - Restrict to these `YYYY-MM` keys. Default: every archive month present.
2583
+ * @param {string | null} [args.jobName] - Restrict to one archived job name.
2584
+ * @returns {Promise<import("./types.js").BackgroundJobsArchiveStatistic[]>} - One row per month and job name, ordered by month then job name.
2585
+ */
2586
+ async jobStatistics({jobName = null, months = null} = {}) {
2587
+ await this.ensureReady()
2588
+
2589
+ const requestedMonths = Array.isArray(months) && months.length > 0 ? [...months].sort() : await this.listArchiveMonths()
2590
+ /** @type {Map<string, {count: number, durations: number[], failureCount: number, jobName: string, month: string}>} */
2591
+ const statistics = new Map()
2592
+
2593
+ for (const monthKey of requestedMonths) {
2594
+ if (!ARCHIVE_MONTH_KEY_PATTERN.test(monthKey)) throw new Error(`Invalid background jobs archive month key: ${monthKey}`)
2595
+
2596
+ const rows = await this._withDb(async (db) => {
2597
+ const archiveTable = db.quoteTable(archiveTableNameForMonthKey(monthKey))
2598
+ const durationSql = `(COALESCE(${db.quoteColumn("completed_at_ms")}, ${db.quoteColumn("failed_at_ms")}, ${db.quoteColumn("orphaned_at_ms")}) - ${db.quoteColumn("created_at_ms")})`
2599
+ const query = db
2600
+ .newQuery()
2601
+ .from(archiveTable)
2602
+ .select("job_name")
2603
+ .select("status")
2604
+ .select(`${durationSql} AS duration_ms`)
2605
+
2606
+ if (jobName) query.where({job_name: jobName})
2607
+
2608
+ return await query.results()
2609
+ })
2610
+
2611
+ for (const row of rows) {
2612
+ const typedRow = /** @type {Record<string, ReturnType<typeof JSON.parse>>} */ (row)
2613
+ const rowJobName = String(typedRow.job_name)
2614
+ const key = `${monthKey}\u0000${rowJobName}`
2615
+ const entry = statistics.get(key) || {count: 0, durations: [], failureCount: 0, jobName: rowJobName, month: monthKey}
2616
+
2617
+ entry.count += 1
2618
+
2619
+ if (ARCHIVE_FAILURE_STATUSES.includes(String(typedRow.status))) entry.failureCount += 1
2620
+
2621
+ const durationMs = this._normalizeNumber(typedRow.duration_ms)
2622
+
2623
+ if (durationMs !== null) entry.durations.push(durationMs)
2624
+
2625
+ statistics.set(key, entry)
2626
+ }
2627
+ }
2628
+
2629
+ return [...statistics.values()]
2630
+ .sort((left, right) => left.month < right.month ? -1 : (left.month > right.month ? 1 : (left.jobName < right.jobName ? -1 : 1)))
2631
+ .map(({count, durations, failureCount, jobName: statisticJobName, month}) => ({
2632
+ count,
2633
+ failureCount,
2634
+ jobName: statisticJobName,
2635
+ month,
2636
+ p50DurationMs: archivePercentileMs(durations, 0.5),
2637
+ p95DurationMs: archivePercentileMs(durations, 0.95)
2638
+ }))
2639
+ }
2212
2640
 
2213
2641
  /**
2214
2642
  * Runs clear all.
@@ -12,6 +12,7 @@
12
12
  /** @typedef {"exit" | "process-error" | "ipc-send" | "child-silence"} PooledRunnerFailureOrigin */
13
13
  /** @typedef {"starting" | "running" | "retiring"} PooledRunnerLifecycleState */
14
14
  /** @typedef {"parent_retire_drained" | "job_timeout" | "worker_stop" | "child_silence" | "signal_sigterm" | "signal_sigint" | "signal_sigkill" | "signal_other" | "ipc_disconnect" | "process_error" | "unexpected_exit"} PooledChildShutdownReason */
15
+ /** @typedef {"uncaughtException" | "unhandledRejection"} PooledChildProcessFaultType */
15
16
  /**
16
17
  * @deprecated Use PooledChildShutdownReason for exact shutdown provenance.
17
18
  * @typedef {"unexpected" | "job-timeout" | "worker-shutdown-timeout"} PooledRunnerTerminationReason
@@ -45,6 +46,7 @@
45
46
  * @property {number} inflightJobIdsTruncatedCount - In-flight ids omitted from the bounded snapshot.
46
47
  * @property {boolean | null} oomKilled - False when the observed exit rules OOM out; null when an unexpected SIGKILL cannot be distinguished from an OOM kill without supervisor/kernel evidence.
47
48
  * @property {PooledRunnerFailureOrigin} origin - Worker observation that initiated failure handling.
49
+ * @property {PooledChildProcessFault | null} processFault - Process-level fault the child reported over IPC before exiting, or null when it reported none.
48
50
  * @property {number} runnerAgeMs - Child age when failure handling started.
49
51
  * @property {number} runnerCreatedAtMs - Child creation timestamp.
50
52
  * @property {boolean} runnerDetached - Whether the runner owned a detached process group.
@@ -62,10 +64,24 @@
62
64
  * @property {BackgroundJobsWorkerLifecycleState} workerLifecycle - Parent worker lifecycle immediately before recovery.
63
65
  * @property {number} workerPid - Parent worker process id.
64
66
  */
67
+ /**
68
+ * One process-level fault observed by a pooled runner child and transmitted to the
69
+ * worker before the child exits. The child's stdio is ignored by the worker fork, so
70
+ * without this record the only surviving evidence of an `uncaughtException` or an
71
+ * `unhandledRejection` is the exit code and signal, and every such exit is attributed
72
+ * to whatever job happened to be in flight.
73
+ * @typedef {object} PooledChildProcessFault
74
+ * @property {string | null} message - Fault message, bounded.
75
+ * @property {string | null} name - Error name when the fault value carried one.
76
+ * @property {string | null} stack - Bounded stack when the fault value was an Error.
77
+ * @property {PooledChildProcessFaultType} type - Process event that observed the fault.
78
+ * @property {string | null} valuePreview - Bounded preview of a rejected value that was not an Error.
79
+ */
65
80
  /**
66
81
  * Best-effort observation sent before a pooled child closes its resources.
67
82
  * @typedef {object} PooledChildShutdownObservation
68
83
  * @property {string} childInstanceId - Stable pooled-child identity.
84
+ * @property {PooledChildProcessFault | null} fault - Process-level fault that caused this shutdown, or null for a requested/signalled/IPC shutdown.
69
85
  * @property {string[]} inflightJobIds - Bounded in-flight durable job ids.
70
86
  * @property {number} inflightJobIdsTruncatedCount - In-flight ids omitted from the bounded snapshot.
71
87
  * @property {PooledChildShutdownReason} reason - Shutdown reason observed by the child.
@@ -300,6 +316,21 @@
300
316
  * @typedef {{type: "replace-scheduled-error", error?: string}} BackgroundJobReplaceScheduledErrorMessage
301
317
  * @typedef {{type: "cancel-scheduled", scheduleKey: string}} BackgroundJobCancelScheduledMessage
302
318
  * @typedef {{type: "schedule-cancelled", jobId: string | null, outcome: BackgroundJobCancellationOutcome}} BackgroundJobScheduleCancelledMessage
319
+ * @typedef {object} BackgroundJobsArchiveResult
320
+ * @property {number} archived - Total rows moved into monthly archive tables.
321
+ * @property {boolean} completedPass - Whether the pass drained every expired row, or stopped early on its time budget and will resume on the next sweep.
322
+ * @property {Record<string, number>} perMonth - Archived rows keyed by month (`YYYY-MM`).
323
+ */
324
+ /**
325
+ * @typedef {object} BackgroundJobsArchiveStatistic
326
+ * @property {number} count - Archived job rows for this month and job name.
327
+ * @property {number} failureCount - Archived rows with status `failed` or `orphaned`.
328
+ * @property {string} jobName - Archived job name.
329
+ * @property {string} month - Archive month (`YYYY-MM`).
330
+ * @property {number | null} p50DurationMs - Median terminal duration (terminal timestamp minus creation), or null when no row has both timestamps.
331
+ * @property {number | null} p95DurationMs - 95th percentile terminal duration, or null when no row has both timestamps.
332
+ */
333
+ /**
303
334
  * @typedef {{type: "cancel-scheduled-error", error?: string}} BackgroundJobCancelScheduledErrorMessage
304
335
  * @typedef {{type: "get-scheduled-job", scheduleKey: string, includeLatestTerminal?: boolean}} BackgroundJobGetScheduledMessage
305
336
  * @typedef {{type: "scheduled-job", currentJob: BackgroundJobRow | null, latestTerminalJob: BackgroundJobRow | null}} BackgroundJobScheduledMessage
@@ -15,7 +15,7 @@ import { runWithBackgroundJobPayload } from "./execution-context.js"
15
15
  import { createGenerationWorkerId } from "./generation-identity.js"
16
16
  import { isProcessAlive } from "./owner-liveness.js"
17
17
  import BackgroundJobsGenerationHandshakeTimeoutError, { DEFAULT_GENERATION_HANDSHAKE_TIMEOUT_MS, validateGenerationHandshakeTimeoutMs } from "./generation-handshake-timeout-error.js"
18
- import { POOLED_RUNNER_INFLIGHT_JOB_ID_LIMIT, boundedPooledRunnerInflightJobIds, isPooledChildShutdownReason, isPooledChildShutdownSignal } from "./pooled-runner-shutdown.js"
18
+ import { POOLED_RUNNER_INFLIGHT_JOB_ID_LIMIT, boundedPooledRunnerInflightJobIds, isPooledChildProcessFault, isPooledChildShutdownReason, isPooledChildShutdownSignal } from "./pooled-runner-shutdown.js"
19
19
 
20
20
  /**
21
21
  * Per-forked-child bookkeeping, shared by the wall-clock backstop, the runner's
@@ -160,10 +160,13 @@ function isForkedChildJobReport(message) {
160
160
  */
161
161
  function isChildShutdownObservationMessage(message) {
162
162
  if (!message || typeof message !== "object") return false
163
- const record = /** @type {{childInstanceId?: ReturnType<typeof JSON.parse>, inflightJobIds?: ReturnType<typeof JSON.parse>, inflightJobIdsTruncatedCount?: ReturnType<typeof JSON.parse>, reason?: ReturnType<typeof JSON.parse>, shutdownObservedAtMs?: ReturnType<typeof JSON.parse>, shutdownRequestedAtMs?: ReturnType<typeof JSON.parse>, signal?: ReturnType<typeof JSON.parse>, type?: ReturnType<typeof JSON.parse>}} */ (message)
163
+ const record = /** @type {{childInstanceId?: ReturnType<typeof JSON.parse>, fault?: ReturnType<typeof JSON.parse>, inflightJobIds?: ReturnType<typeof JSON.parse>, inflightJobIdsTruncatedCount?: ReturnType<typeof JSON.parse>, reason?: ReturnType<typeof JSON.parse>, shutdownObservedAtMs?: ReturnType<typeof JSON.parse>, shutdownRequestedAtMs?: ReturnType<typeof JSON.parse>, signal?: ReturnType<typeof JSON.parse>, type?: ReturnType<typeof JSON.parse>}} */ (message)
164
164
 
165
165
  return record.type === "shutdown-observation"
166
166
  && typeof record.childInstanceId === "string"
167
+ // A child predating the fault contract sends no fault field at all; its shutdown
168
+ // provenance must still be accepted.
169
+ && (record.fault === undefined || record.fault === null || isPooledChildProcessFault(record.fault))
167
170
  && Array.isArray(record.inflightJobIds)
168
171
  && record.inflightJobIds.length <= POOLED_RUNNER_INFLIGHT_JOB_ID_LIMIT
169
172
  && record.inflightJobIds.every((jobId) => typeof jobId === "string")
@@ -176,6 +179,23 @@ function isChildShutdownObservationMessage(message) {
176
179
  && isPooledChildShutdownSignal(record.signal)
177
180
  }
178
181
 
182
+ /**
183
+ * Names the failure reported for every job lost with a pooled child. When the child
184
+ * transmitted its process-level fault, that fault is part of the report: the exit
185
+ * line alone is what makes a real framework fault read as infrastructure loss and
186
+ * re-triggers a whole shard with zero reported failures. The exit line is kept so the
187
+ * observed code and signal remain readable in the same record.
188
+ * @param {object} args - Failure details.
189
+ * @param {Error} args.error - Worker-side failure observed for the child.
190
+ * @param {import("./types.js").PooledChildProcessFault | null} args.fault - Fault the child transmitted before exiting.
191
+ * @returns {Error} - Error to report for the lost jobs.
192
+ */
193
+ function pooledChildFailureError({error, fault}) {
194
+ if (!fault) return error
195
+
196
+ return new Error(`${error.message} — ${fault.type}: ${fault.message ?? fault.valuePreview ?? "unspecified fault"}`)
197
+ }
198
+
179
199
  /**
180
200
  * Checks whether an IPC value is a pooled child's bounded memory observation.
181
201
  * The pooled child's stdio is ignored by the worker fork, so this observation
@@ -2102,14 +2122,16 @@ export default class BackgroundJobsWorker {
2102
2122
  const runnerFailure = state
2103
2123
  ? this._pooledRunnerFailure({child, exitCode, origin, signal, state})
2104
2124
  : undefined
2125
+ if (runnerFailure?.processFault) this._logPooledChildProcessFault({child, runnerFailure})
2105
2126
  if (state) state.inflight.clear()
2106
2127
  this.pooledChildStates.delete(child)
2107
2128
 
2129
+ const failureError = pooledChildFailureError({error, fault: runnerFailure?.processFault ?? null})
2108
2130
  const failureReports = entries.map(async (entry) => {
2109
2131
  await this._reportJobResult({
2110
2132
  jobId: entry.payload.id,
2111
2133
  status: "failed",
2112
- error,
2134
+ error: failureError,
2113
2135
  handoffId: entry.payload.handoffId,
2114
2136
  handedOffAtMs: entry.payload.handedOffAtMs,
2115
2137
  runnerFailure,
@@ -2211,6 +2233,7 @@ export default class BackgroundJobsWorker {
2211
2233
  ...boundedInflight,
2212
2234
  oomKilled: signal === "SIGKILL" && shutdownReason === "signal_sigkill" ? null : false,
2213
2235
  origin,
2236
+ processFault: observation?.fault ?? null,
2214
2237
  runnerAgeMs: Math.max(0, Date.now() - state.createdAtMs),
2215
2238
  runnerCreatedAtMs: state.createdAtMs,
2216
2239
  runnerDetached: false,
@@ -2230,6 +2253,36 @@ export default class BackgroundJobsWorker {
2230
2253
  })
2231
2254
  }
2232
2255
 
2256
+ /**
2257
+ * Logs one compact JSON line naming the process-level fault a pooled child
2258
+ * transmitted over IPC before it exited — the same stderr surface the memory
2259
+ * observations use, which reaches the production log while the child's own stdio
2260
+ * is ignored. A child that dies of an unhandled rejection otherwise looks exactly
2261
+ * like an infrastructure exit, so the shard is retried with zero reported failures
2262
+ * and the fault is never attributed.
2263
+ * @param {object} args - Event details.
2264
+ * @param {import("node:child_process").ChildProcess} args.child - Pooled child.
2265
+ * @param {import("./types.js").PooledRunnerFailure} args.runnerFailure - Shared failure provenance carrying the fault.
2266
+ * @returns {void}
2267
+ */
2268
+ _logPooledChildProcessFault({child, runnerFailure}) {
2269
+ const fault = runnerFailure.processFault
2270
+ if (!fault) return
2271
+
2272
+ console.error(JSON.stringify({
2273
+ event: "pooled-child-process-fault",
2274
+ childInstanceId: runnerFailure.childInstanceId,
2275
+ childPid: runnerFailure.runnerPid ?? child.pid ?? null,
2276
+ exitCode: runnerFailure.exitCode,
2277
+ faultMessage: fault.message,
2278
+ faultName: fault.name,
2279
+ faultType: fault.type,
2280
+ jobs: runnerFailure.inflightJobIds.length,
2281
+ shutdownReason: runnerFailure.shutdownReason,
2282
+ valuePreview: fault.valuePreview
2283
+ }))
2284
+ }
2285
+
2233
2286
  /**
2234
2287
  * Runs run job inline.
2235
2288
  * @param {import("./types.js").BackgroundJobPayload} payload - Payload.