velocious 1.0.758 → 1.0.760

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. package/README.md +1 -1
  2. package/build/background-jobs/worker.js +55 -5
  3. package/build/database/advisory-lock-errors.js +3 -2
  4. package/build/database/advisory-lock-lock-server-backend.js +19 -7
  5. package/build/database/advisory-lock-runner.js +49 -19
  6. package/build/database/drivers/base-table.js +13 -0
  7. package/build/database/drivers/mssql/table.js +18 -8
  8. package/build/database/drivers/mysql/table.js +14 -4
  9. package/build/database/drivers/pgsql/table.js +18 -8
  10. package/build/database/drivers/sqlite/table.js +16 -6
  11. package/build/src/background-jobs/worker.d.ts +25 -2
  12. package/build/src/background-jobs/worker.d.ts.map +1 -1
  13. package/build/src/background-jobs/worker.js +54 -6
  14. package/build/src/database/advisory-lock-errors.d.ts +4 -2
  15. package/build/src/database/advisory-lock-errors.d.ts.map +1 -1
  16. package/build/src/database/advisory-lock-errors.js +4 -3
  17. package/build/src/database/advisory-lock-lock-server-backend.d.ts +19 -10
  18. package/build/src/database/advisory-lock-lock-server-backend.d.ts.map +1 -1
  19. package/build/src/database/advisory-lock-lock-server-backend.js +19 -8
  20. package/build/src/database/advisory-lock-runner.d.ts +3 -0
  21. package/build/src/database/advisory-lock-runner.d.ts.map +1 -1
  22. package/build/src/database/advisory-lock-runner.js +46 -20
  23. package/build/src/database/drivers/base-table.d.ts +8 -0
  24. package/build/src/database/drivers/base-table.d.ts.map +1 -1
  25. package/build/src/database/drivers/base-table.js +12 -1
  26. package/build/src/database/drivers/mssql/table.d.ts +7 -2
  27. package/build/src/database/drivers/mssql/table.d.ts.map +1 -1
  28. package/build/src/database/drivers/mssql/table.js +15 -7
  29. package/build/src/database/drivers/mysql/table.d.ts +7 -2
  30. package/build/src/database/drivers/mysql/table.d.ts.map +1 -1
  31. package/build/src/database/drivers/mysql/table.js +13 -5
  32. package/build/src/database/drivers/pgsql/table.d.ts +7 -2
  33. package/build/src/database/drivers/pgsql/table.d.ts.map +1 -1
  34. package/build/src/database/drivers/pgsql/table.js +15 -7
  35. package/build/src/database/drivers/sqlite/table.d.ts +6 -0
  36. package/build/src/database/drivers/sqlite/table.d.ts.map +1 -1
  37. package/build/src/database/drivers/sqlite/table.js +15 -7
  38. package/build/src/tenants/tenant-aggregator.d.ts +21 -3
  39. package/build/src/tenants/tenant-aggregator.d.ts.map +1 -1
  40. package/build/src/tenants/tenant-aggregator.js +50 -4
  41. package/build/src/tenants/tenant.d.ts +4 -2
  42. package/build/src/tenants/tenant.d.ts.map +1 -1
  43. package/build/src/tenants/tenant.js +5 -3
  44. package/build/tenants/tenant-aggregator.js +56 -3
  45. package/build/tenants/tenant.js +4 -2
  46. package/package.json +1 -1
  47. package/src/background-jobs/worker.js +55 -5
  48. package/src/database/advisory-lock-errors.js +3 -2
  49. package/src/database/advisory-lock-lock-server-backend.js +19 -7
  50. package/src/database/advisory-lock-runner.js +49 -19
  51. package/src/database/drivers/base-table.js +13 -0
  52. package/src/database/drivers/mssql/table.js +18 -8
  53. package/src/database/drivers/mysql/table.js +14 -4
  54. package/src/database/drivers/pgsql/table.js +18 -8
  55. package/src/database/drivers/sqlite/table.js +16 -6
  56. package/src/tenants/tenant-aggregator.js +56 -3
  57. package/src/tenants/tenant.js +4 -2
package/README.md CHANGED
@@ -3078,7 +3078,7 @@ Tenant lifecycle commands print start and final counts, report each completed te
3078
3078
 
3079
3079
  `afterMigrateTenant` hooks run inside the active default and tenant database connection scope for the tenant being migrated.
3080
3080
 
3081
- At runtime, the apartment-style `Tenant` façade (`velocious/build/src/tenants/tenant.js`) is the single entry point: `Tenant.with(tenant, callback)` / `Tenant.current()` to switch into and read a Node async context, `Tenant.handle(tenant)` to deeply capture immutable application and physical database identity for overlapping browser/native work, `Tenant.each({identifier, callback, parallel?, filter?})` to run a callback within every provider-listed tenant, and `Tenant.drop({identifier, tenant})` (plus the `db:tenants:drop` CLI command) to drop a tenant's database through the provider's `dropDatabase` hook. SQLite handles additionally expose framework-owned `open`, `flush`, `close`, `delete`, `inspect`, and `withPin` lifecycle methods; `frontendTenantSqlite.maxOpenHandles` bounds resident identities and clean, idle, unpinned handles are evicted least-recently-used. `Tenant.handle(...).databaseOperation(...)` and `.transaction(...)` use bounded pool-owned checkouts and pin model/query/write/association/preload/audit/attachment/raw work to the captured database even if a later UI project switch changes ambient tenant state. The handle also builds tenant-bound live-query sources, exposes an opaque physical identity for filtered record-change subscriptions, and binds project `SyncClient` instances. `Model.usingTenant(tenant)` uses the same safe core and adds eager helpers plus general `databaseOperation`/`transaction` model callbacks; eager records preserve legacy ambient Node follow-up semantics, while browser/native follow-up database work belongs inside the callback APIs. Inactive identifiers, mixed physical tenants, unsupported/cyclic descriptors, expired operations, unscoped tenant event subscriptions, and stale/cross-tenant sync state fail closed. `Tenant.with` and `Tenant.each` retain their connection-establishing and model-initializing ambient behavior for Node request/job flows. `Tenant.aggregateAcross({identifier, aggregates, keyColumns, subquery, tenants?, filter?})` runs one aggregate over the same table across many tenant databases and returns the merged result — grouping tenants by server and using a single cross-database `UNION ALL` where the driver supports two-part `` `database`.`table` `` references (MySQL/MariaDB) or one query per tenant otherwise (PostgreSQL/SQLite/MSSQL).
3081
+ At runtime, the apartment-style `Tenant` façade (`velocious/build/src/tenants/tenant.js`) is the single entry point: `Tenant.with(tenant, callback)` / `Tenant.current()` to switch into and read a Node async context, `Tenant.handle(tenant)` to deeply capture immutable application and physical database identity for overlapping browser/native work, `Tenant.each({identifier, callback, parallel?, filter?})` to run a callback within every provider-listed tenant, and `Tenant.drop({identifier, tenant})` (plus the `db:tenants:drop` CLI command) to drop a tenant's database through the provider's `dropDatabase` hook. SQLite handles additionally expose framework-owned `open`, `flush`, `close`, `delete`, `inspect`, and `withPin` lifecycle methods; `frontendTenantSqlite.maxOpenHandles` bounds resident identities and clean, idle, unpinned handles are evicted least-recently-used. `Tenant.handle(...).databaseOperation(...)` and `.transaction(...)` use bounded pool-owned checkouts and pin model/query/write/association/preload/audit/attachment/raw work to the captured database even if a later UI project switch changes ambient tenant state. The handle also builds tenant-bound live-query sources, exposes an opaque physical identity for filtered record-change subscriptions, and binds project `SyncClient` instances. `Model.usingTenant(tenant)` uses the same safe core and adds eager helpers plus general `databaseOperation`/`transaction` model callbacks; eager records preserve legacy ambient Node follow-up semantics, while browser/native follow-up database work belongs inside the callback APIs. Inactive identifiers, mixed physical tenants, unsupported/cyclic descriptors, expired operations, unscoped tenant event subscriptions, and stale/cross-tenant sync state fail closed. `Tenant.with` and `Tenant.each` retain their connection-establishing and model-initializing ambient behavior for Node request/job flows. `Tenant.aggregateAcross({identifier, aggregates, keyColumns, subquery, tenants?, filter?, rowIdentity?})` runs one aggregate over the same table across many tenant databases and returns the merged result — grouping tenants by server and using a single cross-database `UNION ALL` where the driver supports two-part `` `database`.`table` `` references (MySQL/MariaDB) or one query per tenant otherwise (PostgreSQL/SQLite/MSSQL). `aggregates: {}` with `rowIdentity: true` skips the aggregation shape for callers whose branches already return the rows: every branch is unioned as written, with no derived-table wrapper, no outer grouping and no per-key merge, so each tenant's rows come back as read ([row-identity rows](docs/tenant-databases.md#row-identity-rows-instead-of-an-aggregate)).
3082
3082
 
3083
3083
  `SchemaCloner` adds a missing auto-increment column and its separate source unique index in one schema alteration, including on MySQL/MariaDB where an auto-increment column must be keyed when it is created.
3084
3084
 
@@ -2,6 +2,7 @@
2
2
 
3
3
  import net from "net"
4
4
  import { fork, spawn } from "node:child_process"
5
+ import timeout, {TimeoutError} from "awaitery/build/timeout.js"
5
6
  import JsonSocket from "./json-socket.js"
6
7
  import BackgroundJobRegistry from "./job-registry.js"
7
8
  import configurationResolver from "../configuration-resolver.js"
@@ -99,6 +100,20 @@ const HEARTBEAT_INTERVAL_MS = 15000
99
100
  const CHILD_ACCEPTANCE_REPORT_MAX_DURATION_MS = 10000
100
101
  /** TCP keepalive so a half-open connection to the main surfaces as a close. */
101
102
  const SOCKET_KEEPALIVE_MS = 10000
103
+ /**
104
+ * Bound on one owned shutdown close (`configuration.shutdown()`, the Beacon
105
+ * disconnect, the database close). Each of them is somebody else's code — the
106
+ * application's initializers, the Beacon client, the driver pool — and the
107
+ * worker's retirement contract is that the generation closes itself, so none of
108
+ * them may be waited on without end. Production held background-jobs-worker pid
109
+ * 1363854 (release 20261122033406) alive 5 h past its retirement with its Beacon
110
+ * socket still open and its Socketduct durable log sink still open, because the
111
+ * application teardown before `disconnectBeacon()` never settled and nothing
112
+ * bounded it, so the CLI never reached its exit. 30 s leaves room for this
113
+ * deployment's 4 s Socketduct stream-drain bound plus a durable-log flush, and a
114
+ * close that runs out is reported as the named step that ran out.
115
+ */
116
+ const WORKER_SHUTDOWN_CLOSE_TIMEOUT_MS = 30000
102
117
  /**
103
118
  * Execution modes.
104
119
  * @type {import("./types.js").BackgroundJobExecutionMode[]} */
@@ -262,6 +277,7 @@ export default class BackgroundJobsWorker {
262
277
  * @param {number} [args.reconnectDelayMs] - Delay before reconnecting an established worker connection (default: 1000).
263
278
  * @param {number} [args.jobTimeoutMs] - Override the wall-clock timeout for forked and pooled jobs from `configuration.getBackgroundJobsConfig()`. `0` disables it.
264
279
  * @param {number | null} [args.workerShutdownTimeoutMs] - Override the wall-clock cap on one whole shutdown drain from `configuration.getBackgroundJobsConfig().workerShutdownTimeoutMs`. `null` drains without a bound.
280
+ * @param {number} [args.shutdownCloseTimeoutMs] - Override the bound on one owned shutdown close (application teardown, Beacon disconnect, database close) in ms (default 30000).
265
281
  * @param {number} [args.pooledRunnerSilenceTimeoutMs] - Override the per-child IPC-silence bound for pooled children with in-flight jobs from `configuration.getBackgroundJobsConfig()` (default 120000ms). `0` disables the watchdog.
266
282
  * @param {boolean} [args.closeDatabaseConnectionsOnStop] - Whether stop owns closing the configuration's database pools (default true).
267
283
  * @param {() => void | Promise<void>} [args.onStopped] - Lifecycle hook invoked after the worker finishes stopping.
@@ -270,7 +286,7 @@ export default class BackgroundJobsWorker {
270
286
  * @param {(observation: import("./types.js").PooledChildMemoryObservation) => void | Promise<void>} [args.onPooledRunnerMemoryObservation] - Explicit pooled-child memory observation hook. Every validated observation (periodic while a child has in-flight jobs, plus on demand) is forwarded here, in addition to the worker's compact stderr log line, so an application can route memory diagnostics (e.g. to a bug reporter) without parsing logs.
271
287
  * @param {(payload: import("./types.js").BackgroundJobPayload) => void} [args.onJobAccepted] - Explicit job-acceptance observation hook. Fires the moment the worker accepts a dispatched job payload, before execution mode routing, so an observer can wait for the pickup event instead of polling in-flight sets.
272
288
  */
273
- constructor({configuration, host, port, generationId, workerInstanceId, maxConcurrentForkedJobs, maxConcurrentInlineJobs, pooledRunnerCount, pooledRunnerConcurrency, pooledRunnerMaxJobs, pooledRunnerMaxRssBytes, pooledRunnerMaxLifetimeMs, runnerMaxOldSpaceSizeMb, runnerMallocArenaMax, forkedChildSigkillGraceMs, heartbeatIntervalMs, generationHandshakeTimeoutMs = DEFAULT_GENERATION_HANDSHAKE_TIMEOUT_MS, reconnectDelayMs = 1000, jobTimeoutMs, workerShutdownTimeoutMs, pooledRunnerSilenceTimeoutMs, closeDatabaseConnectionsOnStop = true, onStopped, onGenerationAccepted, onRetireMessage, onPooledRunnerMemoryObservation, onJobAccepted} = {}) {
289
+ constructor({configuration, host, port, generationId, workerInstanceId, maxConcurrentForkedJobs, maxConcurrentInlineJobs, pooledRunnerCount, pooledRunnerConcurrency, pooledRunnerMaxJobs, pooledRunnerMaxRssBytes, pooledRunnerMaxLifetimeMs, runnerMaxOldSpaceSizeMb, runnerMallocArenaMax, forkedChildSigkillGraceMs, heartbeatIntervalMs, generationHandshakeTimeoutMs = DEFAULT_GENERATION_HANDSHAKE_TIMEOUT_MS, reconnectDelayMs = 1000, jobTimeoutMs, workerShutdownTimeoutMs, shutdownCloseTimeoutMs, pooledRunnerSilenceTimeoutMs, closeDatabaseConnectionsOnStop = true, onStopped, onGenerationAccepted, onRetireMessage, onPooledRunnerMemoryObservation, onJobAccepted} = {}) {
274
290
  /**
275
291
  * Narrows the runtime value to the documented type.
276
292
  * @type {Promise<import("../configuration.js").default>} */
@@ -362,6 +378,12 @@ export default class BackgroundJobsWorker {
362
378
  : (typeof workerShutdownTimeoutMs === "number" && Number.isInteger(workerShutdownTimeoutMs) && workerShutdownTimeoutMs > 0 ? workerShutdownTimeoutMs : null)
363
379
  /** @type {number | null} */
364
380
  this.workerShutdownTimeoutMs = this.workerShutdownTimeoutMsOverride ?? null
381
+ /**
382
+ * Bound on one owned shutdown close, in ms. Constructor override wins so a
383
+ * spec can exercise the bound without waiting for the production one.
384
+ * @type {number}
385
+ */
386
+ this.shutdownCloseTimeoutMs = positiveInteger(shutdownCloseTimeoutMs) || WORKER_SHUTDOWN_CLOSE_TIMEOUT_MS
365
387
  this.shouldStop = false
366
388
  this.isRetiring = false
367
389
  /** @type {Promise<void> | undefined} */
@@ -723,7 +745,8 @@ export default class BackgroundJobsWorker {
723
745
  }
724
746
 
725
747
  /**
726
- * Closes application resources before framework resources when this worker owns them.
748
+ * Closes application resources before framework resources when this worker owns them,
749
+ * with every close bounded so the shutdown always reaches its end.
727
750
  * @returns {Promise<void>} - Resolves after every owned close succeeds.
728
751
  */
729
752
  async _closeConfiguration() {
@@ -735,16 +758,43 @@ export default class BackgroundJobsWorker {
735
758
  message: "Background jobs worker application and framework shutdown failed",
736
759
  steps: [
737
760
  ...(this.closeDatabaseConnectionsOnStop
738
- ? [async () => await configuration.shutdown()]
761
+ ? [async () => await this._boundedShutdownClose("application shutdown", () => configuration.shutdown())]
739
762
  : []),
740
- async () => await configuration.disconnectBeacon(),
763
+ async () => await this._boundedShutdownClose("Beacon disconnect", () => configuration.disconnectBeacon()),
741
764
  ...(this.closeDatabaseConnectionsOnStop
742
- ? [async () => await configuration.closeDatabaseConnections()]
765
+ ? [async () => await this._boundedShutdownClose("database connection close", () => configuration.closeDatabaseConnections())]
743
766
  : [])
744
767
  ]
745
768
  })
746
769
  }
747
770
 
771
+ /**
772
+ * Runs one owned shutdown close inside `shutdownCloseTimeoutMs` and names the step
773
+ * when it runs out.
774
+ *
775
+ * The close is not cancelled — it is left running and simply no longer waited on.
776
+ * That is the whole point: the next close (and then the CLI's exit) can happen at
777
+ * all. A close that never settling is a defect in whatever owns it, and it is
778
+ * reported here as the named step that ran out rather than as a worker that quietly
779
+ * never exits.
780
+ * @param {string} label - The step, named in the failure this reports.
781
+ * @param {(() => Promise<void>)} close - The close to run.
782
+ * @returns {Promise<void>} - Resolves when the close settles; rejects with the close's own error, or a named timeout.
783
+ */
784
+ async _boundedShutdownClose(label, close) {
785
+ const timeoutMs = this.shutdownCloseTimeoutMs
786
+
787
+ try {
788
+ await timeout({timeout: timeoutMs}, close)
789
+ } catch (error) {
790
+ if (error instanceof TimeoutError) {
791
+ throw new Error(`Background jobs worker ${label} did not finish within ${timeoutMs}ms`, {cause: error})
792
+ }
793
+
794
+ throw error
795
+ }
796
+ }
797
+
748
798
  /**
749
799
  * Waits for a set of in-flight job promises to settle, optionally bounded by
750
800
  * `timeoutMs`. When a bounded window runs out with entries still held, the drain
@@ -31,15 +31,16 @@ export class AdvisoryLockLostError extends Error {
31
31
  /**
32
32
  * Runs constructor.
33
33
  * @param {string} message - Error message.
34
- * @param {{cause?: Error, currentEpoch: number | null, epoch: number | null, lockName: string}} args - Fencing details.
34
+ * @param {{cause?: Error, currentEpoch: number | null, epoch: number | null, lockName: string, reason: "fenced" | "expired" | "transport"}} args - Loss details.
35
35
  */
36
- constructor(message, {cause, currentEpoch, epoch, lockName}) {
36
+ constructor(message, {cause, currentEpoch, epoch, lockName, reason}) {
37
37
  super(message)
38
38
  this.name = "AdvisoryLockLostError"
39
39
  this.lockName = lockName
40
40
  this.epoch = epoch
41
41
  this.currentEpoch = currentEpoch
42
42
  this.cause = cause
43
+ this.reason = reason
43
44
  }
44
45
  }
45
46
 
@@ -11,6 +11,8 @@ import {AdvisoryLockServiceUnavailableError} from "./advisory-lock-errors.js"
11
11
  * @property {() => Promise<void>} connect - Opens the transport.
12
12
  * @property {(type: string, args: {onMessage: (body: Record<string, unknown>) => void, params: Record<string, unknown>, timeoutMs: number}) => LockServerConnectionLike} openConnection - Opens the 1:1 connection.
13
13
  * @property {() => Promise<void> | void} [close] - Closes the transport.
14
+ * @typedef {"fenced" | "expired" | "transport"} LockLossReason
15
+ * @typedef {{currentEpoch: number | null, epoch: number | null, reason: LockLossReason}} LockLoss
14
16
  * @typedef {{epoch: number, name: string, timer: ReturnType<typeof setTimeout> | null, leaseExpiresAt: number, renewing: boolean}} LockServerHold
15
17
  * @typedef {{status: "acquired", epoch: number, leaseExpiresAt: number} | {status: "already-held", epoch: number, leaseExpiresAt: number} | {status: "busy"} | {status: "timeout"}} LockServerAcquireResult
16
18
  */
@@ -76,7 +78,7 @@ export default class AdvisoryLockLockServerBackend {
76
78
  this.helloWaiter = null
77
79
  /** @type {Map<string, LockServerHold>} */
78
80
  this.holds = new Map()
79
- /** @type {Map<string, {currentEpoch: number | null, epoch: number | null}>} */
81
+ /** @type {Map<string, LockLoss>} */
80
82
  this.fencedHolds = new Map()
81
83
  /** @type {Map<string, {reject: (error: Error) => void, resolve: (message: Record<string, unknown>) => void, timer: ReturnType<typeof setTimeout> | null}>} */
82
84
  this.pending = new Map()
@@ -289,7 +291,7 @@ export default class AdvisoryLockLockServerBackend {
289
291
  * Whether this hold was fenced or lost during its critical section.
290
292
  * @param {string} name - Lock name.
291
293
  * @param {number | null} epoch - Epoch the hold was granted.
292
- * @returns {{currentEpoch: number | null, epoch: number | null} | null} - Fencing details when lost.
294
+ * @returns {LockLoss | null} - Fencing details when lost.
293
295
  */
294
296
  takeLoss(name, epoch) {
295
297
  const loss = this.fencedHolds.get(name)
@@ -302,7 +304,7 @@ export default class AdvisoryLockLockServerBackend {
302
304
 
303
305
  const hold = this.holds.get(name)
304
306
 
305
- if (hold && hold.epoch !== epoch) return {currentEpoch: hold.epoch, epoch}
307
+ if (hold && hold.epoch !== epoch) return {currentEpoch: hold.epoch, epoch, reason: "fenced"}
306
308
 
307
309
  return null
308
310
  }
@@ -382,7 +384,7 @@ export default class AdvisoryLockLockServerBackend {
382
384
 
383
385
  this.holds.delete(name)
384
386
  if (hold.timer) clearTimeout(hold.timer)
385
- this.fencedHolds.set(name, {currentEpoch: Number(body.epoch ?? 0), epoch: hold.epoch})
387
+ this.fencedHolds.set(name, {currentEpoch: Number(body.epoch ?? 0), epoch: hold.epoch, reason: "fenced"})
386
388
 
387
389
  return
388
390
  }
@@ -516,7 +518,7 @@ export default class AdvisoryLockLockServerBackend {
516
518
  if (status === "lost" || status === "not-found") {
517
519
  this.holds.delete(name)
518
520
  if (hold.timer) clearTimeout(hold.timer)
519
- this.fencedHolds.set(name, {currentEpoch: Number(response.currentEpoch ?? 0), epoch})
521
+ this.fencedHolds.set(name, {currentEpoch: Number(response.currentEpoch ?? 0), epoch, reason: "fenced"})
520
522
  }
521
523
  } catch (error) {
522
524
  if (!(error instanceof AdvisoryLockServiceUnavailableError)) throw error
@@ -535,7 +537,7 @@ export default class AdvisoryLockLockServerBackend {
535
537
  this.markUnavailable(error)
536
538
  this.holds.delete(name)
537
539
  if (hold.timer) clearTimeout(hold.timer)
538
- this.fencedHolds.set(name, {currentEpoch: null, epoch})
540
+ this.fencedHolds.set(name, {currentEpoch: null, epoch, reason: this.connection ? "expired" : "transport"})
539
541
  this.reportError(error)
540
542
  } finally {
541
543
  hold.renewing = false
@@ -553,6 +555,16 @@ export default class AdvisoryLockLockServerBackend {
553
555
  this.status.unavailableUntil = Date.now() + this.reprobeCooldownMs
554
556
  }
555
557
 
558
+ /**
559
+ * How long this backend still refuses to re-probe, in milliseconds. `0` means
560
+ * a probe is allowed on the next call, so there is no bounded reconnect event
561
+ * left to wait for.
562
+ * @returns {number} - Remaining re-probe cooldown, or 0 when one is not pending.
563
+ */
564
+ reprobeCooldownRemainingMs() {
565
+ return Math.max(0, this.status.unavailableUntil - Date.now())
566
+ }
567
+
556
568
  /**
557
569
  * Drops transport state and marks every outstanding hold lost.
558
570
  * @returns {void}
@@ -560,7 +572,7 @@ export default class AdvisoryLockLockServerBackend {
560
572
  teardown() {
561
573
  for (const [name, hold] of this.holds) {
562
574
  if (hold.timer) clearTimeout(hold.timer)
563
- this.fencedHolds.set(name, {currentEpoch: null, epoch: hold.epoch})
575
+ this.fencedHolds.set(name, {currentEpoch: null, epoch: hold.epoch, reason: "transport"})
564
576
  }
565
577
 
566
578
  this.holds.clear()
@@ -367,6 +367,9 @@ export default class AdvisoryLockRunner {
367
367
  * A name whose minimum tier is `lock-server` is never served by a weaker
368
368
  * tier: an absent or unreachable server raises
369
369
  * `AdvisoryLockBackendUnavailableError` instead of reporting a fallback.
370
+ * A mandated name still waits for the backend's own bounded re-probe window
371
+ * whenever this acquisition's budget covers it, so a lock-server restart is
372
+ * not mistaken for an unavailable backend.
370
373
  *
371
374
  * `null` means "this tier did not serve the acquisition — continue with the
372
375
  * next eligible tier". A returned value means the lock-server tier answered,
@@ -400,29 +403,55 @@ export default class AdvisoryLockRunner {
400
403
  return null
401
404
  }
402
405
 
406
+ // A mandated tier may only be abandoned once the backend has no bounded
407
+ // reconnect event left inside this acquisition's own budget. The backend
408
+ // publishes that window (`reprobeCooldownRemainingMs`), so the wait is
409
+ // bounded by the backend's own re-probe rather than by a retry policy.
410
+ const acquireBudgetMs = this.acquireTimeoutMs(args)
411
+ const transportWaitBudgetMs = mode === "wait" && acquireBudgetMs !== null ? acquireBudgetMs : null
403
412
  /** @type {boolean} */
404
413
  let acquired
414
+ /** @type {number} */
415
+ let waitedForTransportMs = 0
405
416
 
406
- try {
407
- if (mode === "wait") {
408
- acquired = await backend.acquire(name, {timeoutMs: this.acquireTimeoutMs(args)})
409
- } else {
410
- acquired = await backend.tryAcquire(name)
411
- }
412
- } catch (error) {
413
- if (!(error instanceof AdvisoryLockServiceUnavailableError)) throw error
417
+ for (;;) {
418
+ try {
419
+ if (mode === "wait") {
420
+ const remainingBudgetMs = acquireBudgetMs === null ? null : Math.max(0, acquireBudgetMs - waitedForTransportMs)
414
421
 
415
- if (policy.minTier === "lock-server") {
416
- throw advisoryLockBackendUnavailable({
417
- lockName: name,
418
- minTier: policy.minTier,
419
- reasons: [`lock-server unavailable: ${error.message}`]
420
- })
421
- }
422
+ acquired = await backend.acquire(name, {timeoutMs: remainingBudgetMs})
423
+ } else {
424
+ acquired = await backend.tryAcquire(name)
425
+ }
422
426
 
423
- this.reportAdvisoryLockFallback({reason: error.message, servedBy: "driver-native", lockName: name})
427
+ break
428
+ } catch (error) {
429
+ if (!(error instanceof AdvisoryLockServiceUnavailableError)) throw error
424
430
 
425
- return null
431
+ if (policy.minTier !== "lock-server") {
432
+ this.reportAdvisoryLockFallback({reason: error.message, servedBy: "driver-native", lockName: name})
433
+
434
+ return null
435
+ }
436
+
437
+ const cooldownMs = typeof backend.reprobeCooldownRemainingMs === "function" ? backend.reprobeCooldownRemainingMs() : 0
438
+ // `or-fail`/`defer` declare no wait budget, so their bound is the
439
+ // backend's own single re-probe cycle: one reconnect attempt, never a queue.
440
+ const waitBudgetMs = transportWaitBudgetMs ?? cooldownMs
441
+
442
+ if (cooldownMs <= 0 || waitedForTransportMs + cooldownMs > waitBudgetMs) {
443
+ throw advisoryLockBackendUnavailable({
444
+ lockName: name,
445
+ minTier: policy.minTier,
446
+ reasons: [`lock-server unavailable: ${error.message}`]
447
+ })
448
+ }
449
+
450
+ await new Promise((resolve) => {
451
+ setTimeout(resolve, cooldownMs)
452
+ })
453
+ waitedForTransportMs += cooldownMs
454
+ }
426
455
  }
427
456
 
428
457
  if (!acquired) {
@@ -485,12 +514,13 @@ export default class AdvisoryLockRunner {
485
514
 
486
515
  if (loss) {
487
516
  throw new AdvisoryLockLostError(
488
- `Lost the lease for advisory lock ${JSON.stringify(name)} while the critical section was running`,
517
+ `Lost the lease for advisory lock ${JSON.stringify(name)} while the critical section was running (reason=${loss.reason}; epoch=${String(loss.epoch)}; currentEpoch=${String(loss.currentEpoch)})`,
489
518
  {
490
519
  cause: callbackError instanceof Error ? callbackError : undefined,
491
520
  currentEpoch: loss.currentEpoch,
492
521
  epoch: lockBackend.epoch ?? null,
493
- lockName: name
522
+ lockName: name,
523
+ reason: loss.reason
494
524
  }
495
525
  )
496
526
  }
@@ -9,6 +9,19 @@ export default class VelociousDatabaseDriversBaseTable {
9
9
  * @type {import("./base.js").default | undefined} */
10
10
  driver = undefined
11
11
 
12
+ /**
13
+ * Builds this table's column objects from raw introspection rows that were already read,
14
+ * so a batched catalog read and a per-table read produce the same columns.
15
+ * @abstract
16
+ * @param {import("./base.js").QueryResultType} columnRows - Raw column rows for this table.
17
+ * @returns {Array<import("./base-column.js").default>} - Columns built from the rows.
18
+ */
19
+ columnsFromRows(columnRows) {
20
+ void columnRows
21
+
22
+ throw new Error("columnsFromRows not implemented")
23
+ }
24
+
12
25
  /**
13
26
  * Runs get column by name.
14
27
  * @param {string} columnName - Column name.
@@ -63,6 +63,23 @@ export default class VelociousDatabaseDriversMssqlTable extends BaseTable {
63
63
  this.driver = driver
64
64
  }
65
65
 
66
+ /**
67
+ * Builds this table's column objects from raw `INFORMATION_SCHEMA.COLUMNS`-shaped rows.
68
+ * @param {import("../base.js").QueryResultType} columnRows - Raw column rows for this table.
69
+ * @returns {Array<import("../base-column.js").default>} - Columns built from the rows.
70
+ */
71
+ columnsFromRows(columnRows) {
72
+ const columns = []
73
+
74
+ for (const data of columnRows) {
75
+ const column = new Column(this, data)
76
+
77
+ columns.push(column)
78
+ }
79
+
80
+ return columns
81
+ }
82
+
66
83
  async getColumns() {
67
84
  const columnRows = await this.getDriver()._cachedTableSchemaMetadata(this.getName(), "columns", async () => {
68
85
  const result = await this.driver.query(`
@@ -88,15 +105,8 @@ export default class VelociousDatabaseDriversMssqlTable extends BaseTable {
88
105
  `)
89
106
  return result
90
107
  })
91
- const columns = []
92
-
93
- for (const data of columnRows) {
94
- const column = new Column(this, data)
95
108
 
96
- columns.push(column)
97
- }
98
-
99
- return columns
109
+ return this.columnsFromRows(columnRows)
100
110
  }
101
111
 
102
112
  async getForeignKeys() {
@@ -17,10 +17,12 @@ export default class VelociousDatabaseDriversMysqlTable extends BaseTable {
17
17
  this.driver = driver
18
18
  }
19
19
 
20
- async getColumns() {
21
- const columnRows = await this.getDriver()._cachedTableSchemaMetadata(this.getName(), "columns", async () => {
22
- return await this.driver.query(`SHOW FULL COLUMNS FROM \`${this.getName()}\``)
23
- })
20
+ /**
21
+ * Builds this table's column objects from raw `SHOW FULL COLUMNS`-shaped rows.
22
+ * @param {import("../base.js").QueryResultType} columnRows - Raw column rows for this table.
23
+ * @returns {Array<import("../base-column.js").default>} - Columns built from the rows.
24
+ */
25
+ columnsFromRows(columnRows) {
24
26
  const columns = []
25
27
 
26
28
  for (const data of columnRows) {
@@ -32,6 +34,14 @@ export default class VelociousDatabaseDriversMysqlTable extends BaseTable {
32
34
  return columns
33
35
  }
34
36
 
37
+ async getColumns() {
38
+ const columnRows = await this.getDriver()._cachedTableSchemaMetadata(this.getName(), "columns", async () => {
39
+ return await this.driver.query(`SHOW FULL COLUMNS FROM \`${this.getName()}\``)
40
+ })
41
+
42
+ return this.columnsFromRows(columnRows)
43
+ }
44
+
35
45
  async getForeignKeys() {
36
46
  const foreignKeyRows = await this.getDriver()._cachedTableSchemaMetadata(this.getName(), "foreignKeys", async () => {
37
47
  const sql = `
@@ -62,6 +62,23 @@ export default class VelociousDatabaseDriversPgsqlTable extends BaseTable {
62
62
  this.driver = driver
63
63
  }
64
64
 
65
+ /**
66
+ * Builds this table's column objects from raw `information_schema.columns`-shaped rows.
67
+ * @param {import("../base.js").QueryResultType} columnRows - Raw column rows for this table.
68
+ * @returns {Array<import("../base-column.js").default>} - Columns built from the rows.
69
+ */
70
+ columnsFromRows(columnRows) {
71
+ const columns = []
72
+
73
+ for (const data of columnRows) {
74
+ const column = new Column(this, data)
75
+
76
+ columns.push(column)
77
+ }
78
+
79
+ return columns
80
+ }
81
+
65
82
  async getColumns() {
66
83
  const columnRows = await this.getDriver()._cachedTableSchemaMetadata(this.getName(), "columns", async () => {
67
84
  const result = await this.driver.query(`
@@ -91,15 +108,8 @@ export default class VelociousDatabaseDriversPgsqlTable extends BaseTable {
91
108
  `)
92
109
  return result
93
110
  })
94
- const columns = []
95
-
96
- for (const data of columnRows) {
97
- const column = new Column(this, data)
98
111
 
99
- columns.push(column)
100
- }
101
-
102
- return columns
112
+ return this.columnsFromRows(columnRows)
103
113
  }
104
114
 
105
115
  async getForeignKeys() {
@@ -19,13 +19,11 @@ export default class VelociousDatabaseDriversSqliteTable extends BaseTable {
19
19
  }
20
20
 
21
21
  /**
22
- * Runs get columns.
23
- * @returns {Promise<Array<import("../base-column.js").default>>} - Resolves with the columns.
22
+ * Builds this table's column objects from raw `PRAGMA table_info` rows.
23
+ * @param {import("../base.js").QueryResultType} columnRows - Raw column rows for this table.
24
+ * @returns {Array<import("../base-column.js").default>} - Columns built from the rows.
24
25
  */
25
- async getColumns() {
26
- const columnRows = await this.getDriver()._cachedTableSchemaMetadata(this.getName(), "columns", async () => {
27
- return await this.driver.query(`PRAGMA table_info('${this.getName()}')`)
28
- })
26
+ columnsFromRows(columnRows) {
29
27
  const columns = []
30
28
 
31
29
  for (const columnData of columnRows) {
@@ -37,6 +35,18 @@ export default class VelociousDatabaseDriversSqliteTable extends BaseTable {
37
35
  return columns
38
36
  }
39
37
 
38
+ /**
39
+ * Runs get columns.
40
+ * @returns {Promise<Array<import("../base-column.js").default>>} - Resolves with the columns.
41
+ */
42
+ async getColumns() {
43
+ const columnRows = await this.getDriver()._cachedTableSchemaMetadata(this.getName(), "columns", async () => {
44
+ return await this.driver.query(`PRAGMA table_info('${this.getName()}')`)
45
+ })
46
+
47
+ return this.columnsFromRows(columnRows)
48
+ }
49
+
40
50
  async getForeignKeys() {
41
51
  const foreignKeysData = await this.getDriver()._cachedTableSchemaMetadata(this.getName(), "foreignKeys", async () => {
42
52
  return await this.driver.query(`SELECT * FROM pragma_foreign_key_list(${this.driver.quote(this.getName())})`)
@@ -234,6 +234,12 @@ export default class BackgroundJobsWorker {
234
234
  workerShutdownTimeoutMsOverride: number | null | undefined;
235
235
  /** @type {number | null} */
236
236
  workerShutdownTimeoutMs: number | null;
237
+ /**
238
+ * Bound on one owned shutdown close, in ms. Constructor override wins so a
239
+ * spec can exercise the bound without waiting for the production one.
240
+ * @type {number}
241
+ */
242
+ shutdownCloseTimeoutMs: number;
237
243
  shouldStop: boolean;
238
244
  isRetiring: boolean;
239
245
  /** @type {Promise<void> | undefined} */
@@ -348,6 +354,7 @@ export default class BackgroundJobsWorker {
348
354
  * @param {number} [args.reconnectDelayMs] - Delay before reconnecting an established worker connection (default: 1000).
349
355
  * @param {number} [args.jobTimeoutMs] - Override the wall-clock timeout for forked and pooled jobs from `configuration.getBackgroundJobsConfig()`. `0` disables it.
350
356
  * @param {number | null} [args.workerShutdownTimeoutMs] - Override the wall-clock cap on one whole shutdown drain from `configuration.getBackgroundJobsConfig().workerShutdownTimeoutMs`. `null` drains without a bound.
357
+ * @param {number} [args.shutdownCloseTimeoutMs] - Override the bound on one owned shutdown close (application teardown, Beacon disconnect, database close) in ms (default 30000).
351
358
  * @param {number} [args.pooledRunnerSilenceTimeoutMs] - Override the per-child IPC-silence bound for pooled children with in-flight jobs from `configuration.getBackgroundJobsConfig()` (default 120000ms). `0` disables the watchdog.
352
359
  * @param {boolean} [args.closeDatabaseConnectionsOnStop] - Whether stop owns closing the configuration's database pools (default true).
353
360
  * @param {() => void | Promise<void>} [args.onStopped] - Lifecycle hook invoked after the worker finishes stopping.
@@ -356,7 +363,7 @@ export default class BackgroundJobsWorker {
356
363
  * @param {(observation: import("./types.js").PooledChildMemoryObservation) => void | Promise<void>} [args.onPooledRunnerMemoryObservation] - Explicit pooled-child memory observation hook. Every validated observation (periodic while a child has in-flight jobs, plus on demand) is forwarded here, in addition to the worker's compact stderr log line, so an application can route memory diagnostics (e.g. to a bug reporter) without parsing logs.
357
364
  * @param {(payload: import("./types.js").BackgroundJobPayload) => void} [args.onJobAccepted] - Explicit job-acceptance observation hook. Fires the moment the worker accepts a dispatched job payload, before execution mode routing, so an observer can wait for the pickup event instead of polling in-flight sets.
358
365
  */
359
- constructor({ configuration, host, port, generationId, workerInstanceId, maxConcurrentForkedJobs, maxConcurrentInlineJobs, pooledRunnerCount, pooledRunnerConcurrency, pooledRunnerMaxJobs, pooledRunnerMaxRssBytes, pooledRunnerMaxLifetimeMs, runnerMaxOldSpaceSizeMb, runnerMallocArenaMax, forkedChildSigkillGraceMs, heartbeatIntervalMs, generationHandshakeTimeoutMs, reconnectDelayMs, jobTimeoutMs, workerShutdownTimeoutMs, pooledRunnerSilenceTimeoutMs, closeDatabaseConnectionsOnStop, onStopped, onGenerationAccepted, onRetireMessage, onPooledRunnerMemoryObservation, onJobAccepted }?: {
366
+ constructor({ configuration, host, port, generationId, workerInstanceId, maxConcurrentForkedJobs, maxConcurrentInlineJobs, pooledRunnerCount, pooledRunnerConcurrency, pooledRunnerMaxJobs, pooledRunnerMaxRssBytes, pooledRunnerMaxLifetimeMs, runnerMaxOldSpaceSizeMb, runnerMallocArenaMax, forkedChildSigkillGraceMs, heartbeatIntervalMs, generationHandshakeTimeoutMs, reconnectDelayMs, jobTimeoutMs, workerShutdownTimeoutMs, shutdownCloseTimeoutMs, pooledRunnerSilenceTimeoutMs, closeDatabaseConnectionsOnStop, onStopped, onGenerationAccepted, onRetireMessage, onPooledRunnerMemoryObservation, onJobAccepted }?: {
360
367
  configuration?: import("../configuration.js").default;
361
368
  host?: string;
362
369
  port?: number;
@@ -377,6 +384,7 @@ export default class BackgroundJobsWorker {
377
384
  reconnectDelayMs?: number;
378
385
  jobTimeoutMs?: number;
379
386
  workerShutdownTimeoutMs?: number | null;
387
+ shutdownCloseTimeoutMs?: number;
380
388
  pooledRunnerSilenceTimeoutMs?: number;
381
389
  closeDatabaseConnectionsOnStop?: boolean;
382
390
  onStopped?: () => void | Promise<void>;
@@ -451,10 +459,25 @@ export default class BackgroundJobsWorker {
451
459
  */
452
460
  _remainingDrainMs(deadlineMs: number | undefined): number | undefined;
453
461
  /**
454
- * Closes application resources before framework resources when this worker owns them.
462
+ * Closes application resources before framework resources when this worker owns them,
463
+ * with every close bounded so the shutdown always reaches its end.
455
464
  * @returns {Promise<void>} - Resolves after every owned close succeeds.
456
465
  */
457
466
  _closeConfiguration(): Promise<void>;
467
+ /**
468
+ * Runs one owned shutdown close inside `shutdownCloseTimeoutMs` and names the step
469
+ * when it runs out.
470
+ *
471
+ * The close is not cancelled — it is left running and simply no longer waited on.
472
+ * That is the whole point: the next close (and then the CLI's exit) can happen at
473
+ * all. A close that never settling is a defect in whatever owns it, and it is
474
+ * reported here as the named step that ran out rather than as a worker that quietly
475
+ * never exits.
476
+ * @param {string} label - The step, named in the failure this reports.
477
+ * @param {(() => Promise<void>)} close - The close to run.
478
+ * @returns {Promise<void>} - Resolves when the close settles; rejects with the close's own error, or a named timeout.
479
+ */
480
+ _boundedShutdownClose(label: string, close: (() => Promise<void>)): Promise<void>;
458
481
  /**
459
482
  * Waits for a set of in-flight job promises to settle, optionally bounded by
460
483
  * `timeoutMs`. When a bounded window runs out with entries still held, the drain