velocious 1.0.758 → 1.0.760
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/build/background-jobs/worker.js +55 -5
- package/build/database/advisory-lock-errors.js +3 -2
- package/build/database/advisory-lock-lock-server-backend.js +19 -7
- package/build/database/advisory-lock-runner.js +49 -19
- package/build/database/drivers/base-table.js +13 -0
- package/build/database/drivers/mssql/table.js +18 -8
- package/build/database/drivers/mysql/table.js +14 -4
- package/build/database/drivers/pgsql/table.js +18 -8
- package/build/database/drivers/sqlite/table.js +16 -6
- package/build/src/background-jobs/worker.d.ts +25 -2
- package/build/src/background-jobs/worker.d.ts.map +1 -1
- package/build/src/background-jobs/worker.js +54 -6
- package/build/src/database/advisory-lock-errors.d.ts +4 -2
- package/build/src/database/advisory-lock-errors.d.ts.map +1 -1
- package/build/src/database/advisory-lock-errors.js +4 -3
- package/build/src/database/advisory-lock-lock-server-backend.d.ts +19 -10
- package/build/src/database/advisory-lock-lock-server-backend.d.ts.map +1 -1
- package/build/src/database/advisory-lock-lock-server-backend.js +19 -8
- package/build/src/database/advisory-lock-runner.d.ts +3 -0
- package/build/src/database/advisory-lock-runner.d.ts.map +1 -1
- package/build/src/database/advisory-lock-runner.js +46 -20
- package/build/src/database/drivers/base-table.d.ts +8 -0
- package/build/src/database/drivers/base-table.d.ts.map +1 -1
- package/build/src/database/drivers/base-table.js +12 -1
- package/build/src/database/drivers/mssql/table.d.ts +7 -2
- package/build/src/database/drivers/mssql/table.d.ts.map +1 -1
- package/build/src/database/drivers/mssql/table.js +15 -7
- package/build/src/database/drivers/mysql/table.d.ts +7 -2
- package/build/src/database/drivers/mysql/table.d.ts.map +1 -1
- package/build/src/database/drivers/mysql/table.js +13 -5
- package/build/src/database/drivers/pgsql/table.d.ts +7 -2
- package/build/src/database/drivers/pgsql/table.d.ts.map +1 -1
- package/build/src/database/drivers/pgsql/table.js +15 -7
- package/build/src/database/drivers/sqlite/table.d.ts +6 -0
- package/build/src/database/drivers/sqlite/table.d.ts.map +1 -1
- package/build/src/database/drivers/sqlite/table.js +15 -7
- package/build/src/tenants/tenant-aggregator.d.ts +21 -3
- package/build/src/tenants/tenant-aggregator.d.ts.map +1 -1
- package/build/src/tenants/tenant-aggregator.js +50 -4
- package/build/src/tenants/tenant.d.ts +4 -2
- package/build/src/tenants/tenant.d.ts.map +1 -1
- package/build/src/tenants/tenant.js +5 -3
- package/build/tenants/tenant-aggregator.js +56 -3
- package/build/tenants/tenant.js +4 -2
- package/package.json +1 -1
- package/src/background-jobs/worker.js +55 -5
- package/src/database/advisory-lock-errors.js +3 -2
- package/src/database/advisory-lock-lock-server-backend.js +19 -7
- package/src/database/advisory-lock-runner.js +49 -19
- package/src/database/drivers/base-table.js +13 -0
- package/src/database/drivers/mssql/table.js +18 -8
- package/src/database/drivers/mysql/table.js +14 -4
- package/src/database/drivers/pgsql/table.js +18 -8
- package/src/database/drivers/sqlite/table.js +16 -6
- package/src/tenants/tenant-aggregator.js +56 -3
- package/src/tenants/tenant.js +4 -2
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
import net from "net"
|
|
4
4
|
import { fork, spawn } from "node:child_process"
|
|
5
|
+
import timeout, {TimeoutError} from "awaitery/build/timeout.js"
|
|
5
6
|
import JsonSocket from "./json-socket.js"
|
|
6
7
|
import BackgroundJobRegistry from "./job-registry.js"
|
|
7
8
|
import configurationResolver from "../configuration-resolver.js"
|
|
@@ -99,6 +100,20 @@ const HEARTBEAT_INTERVAL_MS = 15000
|
|
|
99
100
|
const CHILD_ACCEPTANCE_REPORT_MAX_DURATION_MS = 10000
|
|
100
101
|
/** TCP keepalive so a half-open connection to the main surfaces as a close. */
|
|
101
102
|
const SOCKET_KEEPALIVE_MS = 10000
|
|
103
|
+
/**
|
|
104
|
+
* Bound on one owned shutdown close (`configuration.shutdown()`, the Beacon
|
|
105
|
+
* disconnect, the database close). Each of them is somebody else's code — the
|
|
106
|
+
* application's initializers, the Beacon client, the driver pool — and the
|
|
107
|
+
* worker's retirement contract is that the generation closes itself, so none of
|
|
108
|
+
* them may be waited on without end. Production held background-jobs-worker pid
|
|
109
|
+
* 1363854 (release 20261122033406) alive 5 h past its retirement with its Beacon
|
|
110
|
+
* socket still open and its Socketduct durable log sink still open, because the
|
|
111
|
+
* application teardown before `disconnectBeacon()` never settled and nothing
|
|
112
|
+
* bounded it, so the CLI never reached its exit. 30 s leaves room for this
|
|
113
|
+
* deployment's 4 s Socketduct stream-drain bound plus a durable-log flush, and a
|
|
114
|
+
* close that runs out is reported as the named step that ran out.
|
|
115
|
+
*/
|
|
116
|
+
const WORKER_SHUTDOWN_CLOSE_TIMEOUT_MS = 30000
|
|
102
117
|
/**
|
|
103
118
|
* Execution modes.
|
|
104
119
|
* @type {import("./types.js").BackgroundJobExecutionMode[]} */
|
|
@@ -262,6 +277,7 @@ export default class BackgroundJobsWorker {
|
|
|
262
277
|
* @param {number} [args.reconnectDelayMs] - Delay before reconnecting an established worker connection (default: 1000).
|
|
263
278
|
* @param {number} [args.jobTimeoutMs] - Override the wall-clock timeout for forked and pooled jobs from `configuration.getBackgroundJobsConfig()`. `0` disables it.
|
|
264
279
|
* @param {number | null} [args.workerShutdownTimeoutMs] - Override the wall-clock cap on one whole shutdown drain from `configuration.getBackgroundJobsConfig().workerShutdownTimeoutMs`. `null` drains without a bound.
|
|
280
|
+
* @param {number} [args.shutdownCloseTimeoutMs] - Override the bound on one owned shutdown close (application teardown, Beacon disconnect, database close) in ms (default 30000).
|
|
265
281
|
* @param {number} [args.pooledRunnerSilenceTimeoutMs] - Override the per-child IPC-silence bound for pooled children with in-flight jobs from `configuration.getBackgroundJobsConfig()` (default 120000ms). `0` disables the watchdog.
|
|
266
282
|
* @param {boolean} [args.closeDatabaseConnectionsOnStop] - Whether stop owns closing the configuration's database pools (default true).
|
|
267
283
|
* @param {() => void | Promise<void>} [args.onStopped] - Lifecycle hook invoked after the worker finishes stopping.
|
|
@@ -270,7 +286,7 @@ export default class BackgroundJobsWorker {
|
|
|
270
286
|
* @param {(observation: import("./types.js").PooledChildMemoryObservation) => void | Promise<void>} [args.onPooledRunnerMemoryObservation] - Explicit pooled-child memory observation hook. Every validated observation (periodic while a child has in-flight jobs, plus on demand) is forwarded here, in addition to the worker's compact stderr log line, so an application can route memory diagnostics (e.g. to a bug reporter) without parsing logs.
|
|
271
287
|
* @param {(payload: import("./types.js").BackgroundJobPayload) => void} [args.onJobAccepted] - Explicit job-acceptance observation hook. Fires the moment the worker accepts a dispatched job payload, before execution mode routing, so an observer can wait for the pickup event instead of polling in-flight sets.
|
|
272
288
|
*/
|
|
273
|
-
constructor({configuration, host, port, generationId, workerInstanceId, maxConcurrentForkedJobs, maxConcurrentInlineJobs, pooledRunnerCount, pooledRunnerConcurrency, pooledRunnerMaxJobs, pooledRunnerMaxRssBytes, pooledRunnerMaxLifetimeMs, runnerMaxOldSpaceSizeMb, runnerMallocArenaMax, forkedChildSigkillGraceMs, heartbeatIntervalMs, generationHandshakeTimeoutMs = DEFAULT_GENERATION_HANDSHAKE_TIMEOUT_MS, reconnectDelayMs = 1000, jobTimeoutMs, workerShutdownTimeoutMs, pooledRunnerSilenceTimeoutMs, closeDatabaseConnectionsOnStop = true, onStopped, onGenerationAccepted, onRetireMessage, onPooledRunnerMemoryObservation, onJobAccepted} = {}) {
|
|
289
|
+
constructor({configuration, host, port, generationId, workerInstanceId, maxConcurrentForkedJobs, maxConcurrentInlineJobs, pooledRunnerCount, pooledRunnerConcurrency, pooledRunnerMaxJobs, pooledRunnerMaxRssBytes, pooledRunnerMaxLifetimeMs, runnerMaxOldSpaceSizeMb, runnerMallocArenaMax, forkedChildSigkillGraceMs, heartbeatIntervalMs, generationHandshakeTimeoutMs = DEFAULT_GENERATION_HANDSHAKE_TIMEOUT_MS, reconnectDelayMs = 1000, jobTimeoutMs, workerShutdownTimeoutMs, shutdownCloseTimeoutMs, pooledRunnerSilenceTimeoutMs, closeDatabaseConnectionsOnStop = true, onStopped, onGenerationAccepted, onRetireMessage, onPooledRunnerMemoryObservation, onJobAccepted} = {}) {
|
|
274
290
|
/**
|
|
275
291
|
* Narrows the runtime value to the documented type.
|
|
276
292
|
* @type {Promise<import("../configuration.js").default>} */
|
|
@@ -362,6 +378,12 @@ export default class BackgroundJobsWorker {
|
|
|
362
378
|
: (typeof workerShutdownTimeoutMs === "number" && Number.isInteger(workerShutdownTimeoutMs) && workerShutdownTimeoutMs > 0 ? workerShutdownTimeoutMs : null)
|
|
363
379
|
/** @type {number | null} */
|
|
364
380
|
this.workerShutdownTimeoutMs = this.workerShutdownTimeoutMsOverride ?? null
|
|
381
|
+
/**
|
|
382
|
+
* Bound on one owned shutdown close, in ms. Constructor override wins so a
|
|
383
|
+
* spec can exercise the bound without waiting for the production one.
|
|
384
|
+
* @type {number}
|
|
385
|
+
*/
|
|
386
|
+
this.shutdownCloseTimeoutMs = positiveInteger(shutdownCloseTimeoutMs) || WORKER_SHUTDOWN_CLOSE_TIMEOUT_MS
|
|
365
387
|
this.shouldStop = false
|
|
366
388
|
this.isRetiring = false
|
|
367
389
|
/** @type {Promise<void> | undefined} */
|
|
@@ -723,7 +745,8 @@ export default class BackgroundJobsWorker {
|
|
|
723
745
|
}
|
|
724
746
|
|
|
725
747
|
/**
|
|
726
|
-
* Closes application resources before framework resources when this worker owns them
|
|
748
|
+
* Closes application resources before framework resources when this worker owns them,
|
|
749
|
+
* with every close bounded so the shutdown always reaches its end.
|
|
727
750
|
* @returns {Promise<void>} - Resolves after every owned close succeeds.
|
|
728
751
|
*/
|
|
729
752
|
async _closeConfiguration() {
|
|
@@ -735,16 +758,43 @@ export default class BackgroundJobsWorker {
|
|
|
735
758
|
message: "Background jobs worker application and framework shutdown failed",
|
|
736
759
|
steps: [
|
|
737
760
|
...(this.closeDatabaseConnectionsOnStop
|
|
738
|
-
? [async () => await configuration.shutdown()]
|
|
761
|
+
? [async () => await this._boundedShutdownClose("application shutdown", () => configuration.shutdown())]
|
|
739
762
|
: []),
|
|
740
|
-
async () => await configuration.disconnectBeacon(),
|
|
763
|
+
async () => await this._boundedShutdownClose("Beacon disconnect", () => configuration.disconnectBeacon()),
|
|
741
764
|
...(this.closeDatabaseConnectionsOnStop
|
|
742
|
-
? [async () => await configuration.closeDatabaseConnections()]
|
|
765
|
+
? [async () => await this._boundedShutdownClose("database connection close", () => configuration.closeDatabaseConnections())]
|
|
743
766
|
: [])
|
|
744
767
|
]
|
|
745
768
|
})
|
|
746
769
|
}
|
|
747
770
|
|
|
771
|
+
/**
|
|
772
|
+
* Runs one owned shutdown close inside `shutdownCloseTimeoutMs` and names the step
|
|
773
|
+
* when it runs out.
|
|
774
|
+
*
|
|
775
|
+
* The close is not cancelled — it is left running and simply no longer waited on.
|
|
776
|
+
* That is the whole point: the next close (and then the CLI's exit) can happen at
|
|
777
|
+
* all. A close that never settling is a defect in whatever owns it, and it is
|
|
778
|
+
* reported here as the named step that ran out rather than as a worker that quietly
|
|
779
|
+
* never exits.
|
|
780
|
+
* @param {string} label - The step, named in the failure this reports.
|
|
781
|
+
* @param {(() => Promise<void>)} close - The close to run.
|
|
782
|
+
* @returns {Promise<void>} - Resolves when the close settles; rejects with the close's own error, or a named timeout.
|
|
783
|
+
*/
|
|
784
|
+
async _boundedShutdownClose(label, close) {
|
|
785
|
+
const timeoutMs = this.shutdownCloseTimeoutMs
|
|
786
|
+
|
|
787
|
+
try {
|
|
788
|
+
await timeout({timeout: timeoutMs}, close)
|
|
789
|
+
} catch (error) {
|
|
790
|
+
if (error instanceof TimeoutError) {
|
|
791
|
+
throw new Error(`Background jobs worker ${label} did not finish within ${timeoutMs}ms`, {cause: error})
|
|
792
|
+
}
|
|
793
|
+
|
|
794
|
+
throw error
|
|
795
|
+
}
|
|
796
|
+
}
|
|
797
|
+
|
|
748
798
|
/**
|
|
749
799
|
* Waits for a set of in-flight job promises to settle, optionally bounded by
|
|
750
800
|
* `timeoutMs`. When a bounded window runs out with entries still held, the drain
|
|
@@ -31,15 +31,16 @@ export class AdvisoryLockLostError extends Error {
|
|
|
31
31
|
/**
|
|
32
32
|
* Runs constructor.
|
|
33
33
|
* @param {string} message - Error message.
|
|
34
|
-
* @param {{cause?: Error, currentEpoch: number | null, epoch: number | null, lockName: string}} args -
|
|
34
|
+
* @param {{cause?: Error, currentEpoch: number | null, epoch: number | null, lockName: string, reason: "fenced" | "expired" | "transport"}} args - Loss details.
|
|
35
35
|
*/
|
|
36
|
-
constructor(message, {cause, currentEpoch, epoch, lockName}) {
|
|
36
|
+
constructor(message, {cause, currentEpoch, epoch, lockName, reason}) {
|
|
37
37
|
super(message)
|
|
38
38
|
this.name = "AdvisoryLockLostError"
|
|
39
39
|
this.lockName = lockName
|
|
40
40
|
this.epoch = epoch
|
|
41
41
|
this.currentEpoch = currentEpoch
|
|
42
42
|
this.cause = cause
|
|
43
|
+
this.reason = reason
|
|
43
44
|
}
|
|
44
45
|
}
|
|
45
46
|
|
|
@@ -11,6 +11,8 @@ import {AdvisoryLockServiceUnavailableError} from "./advisory-lock-errors.js"
|
|
|
11
11
|
* @property {() => Promise<void>} connect - Opens the transport.
|
|
12
12
|
* @property {(type: string, args: {onMessage: (body: Record<string, unknown>) => void, params: Record<string, unknown>, timeoutMs: number}) => LockServerConnectionLike} openConnection - Opens the 1:1 connection.
|
|
13
13
|
* @property {() => Promise<void> | void} [close] - Closes the transport.
|
|
14
|
+
* @typedef {"fenced" | "expired" | "transport"} LockLossReason
|
|
15
|
+
* @typedef {{currentEpoch: number | null, epoch: number | null, reason: LockLossReason}} LockLoss
|
|
14
16
|
* @typedef {{epoch: number, name: string, timer: ReturnType<typeof setTimeout> | null, leaseExpiresAt: number, renewing: boolean}} LockServerHold
|
|
15
17
|
* @typedef {{status: "acquired", epoch: number, leaseExpiresAt: number} | {status: "already-held", epoch: number, leaseExpiresAt: number} | {status: "busy"} | {status: "timeout"}} LockServerAcquireResult
|
|
16
18
|
*/
|
|
@@ -76,7 +78,7 @@ export default class AdvisoryLockLockServerBackend {
|
|
|
76
78
|
this.helloWaiter = null
|
|
77
79
|
/** @type {Map<string, LockServerHold>} */
|
|
78
80
|
this.holds = new Map()
|
|
79
|
-
/** @type {Map<string,
|
|
81
|
+
/** @type {Map<string, LockLoss>} */
|
|
80
82
|
this.fencedHolds = new Map()
|
|
81
83
|
/** @type {Map<string, {reject: (error: Error) => void, resolve: (message: Record<string, unknown>) => void, timer: ReturnType<typeof setTimeout> | null}>} */
|
|
82
84
|
this.pending = new Map()
|
|
@@ -289,7 +291,7 @@ export default class AdvisoryLockLockServerBackend {
|
|
|
289
291
|
* Whether this hold was fenced or lost during its critical section.
|
|
290
292
|
* @param {string} name - Lock name.
|
|
291
293
|
* @param {number | null} epoch - Epoch the hold was granted.
|
|
292
|
-
* @returns {
|
|
294
|
+
* @returns {LockLoss | null} - Fencing details when lost.
|
|
293
295
|
*/
|
|
294
296
|
takeLoss(name, epoch) {
|
|
295
297
|
const loss = this.fencedHolds.get(name)
|
|
@@ -302,7 +304,7 @@ export default class AdvisoryLockLockServerBackend {
|
|
|
302
304
|
|
|
303
305
|
const hold = this.holds.get(name)
|
|
304
306
|
|
|
305
|
-
if (hold && hold.epoch !== epoch) return {currentEpoch: hold.epoch, epoch}
|
|
307
|
+
if (hold && hold.epoch !== epoch) return {currentEpoch: hold.epoch, epoch, reason: "fenced"}
|
|
306
308
|
|
|
307
309
|
return null
|
|
308
310
|
}
|
|
@@ -382,7 +384,7 @@ export default class AdvisoryLockLockServerBackend {
|
|
|
382
384
|
|
|
383
385
|
this.holds.delete(name)
|
|
384
386
|
if (hold.timer) clearTimeout(hold.timer)
|
|
385
|
-
this.fencedHolds.set(name, {currentEpoch: Number(body.epoch ?? 0), epoch: hold.epoch})
|
|
387
|
+
this.fencedHolds.set(name, {currentEpoch: Number(body.epoch ?? 0), epoch: hold.epoch, reason: "fenced"})
|
|
386
388
|
|
|
387
389
|
return
|
|
388
390
|
}
|
|
@@ -516,7 +518,7 @@ export default class AdvisoryLockLockServerBackend {
|
|
|
516
518
|
if (status === "lost" || status === "not-found") {
|
|
517
519
|
this.holds.delete(name)
|
|
518
520
|
if (hold.timer) clearTimeout(hold.timer)
|
|
519
|
-
this.fencedHolds.set(name, {currentEpoch: Number(response.currentEpoch ?? 0), epoch})
|
|
521
|
+
this.fencedHolds.set(name, {currentEpoch: Number(response.currentEpoch ?? 0), epoch, reason: "fenced"})
|
|
520
522
|
}
|
|
521
523
|
} catch (error) {
|
|
522
524
|
if (!(error instanceof AdvisoryLockServiceUnavailableError)) throw error
|
|
@@ -535,7 +537,7 @@ export default class AdvisoryLockLockServerBackend {
|
|
|
535
537
|
this.markUnavailable(error)
|
|
536
538
|
this.holds.delete(name)
|
|
537
539
|
if (hold.timer) clearTimeout(hold.timer)
|
|
538
|
-
this.fencedHolds.set(name, {currentEpoch: null, epoch})
|
|
540
|
+
this.fencedHolds.set(name, {currentEpoch: null, epoch, reason: this.connection ? "expired" : "transport"})
|
|
539
541
|
this.reportError(error)
|
|
540
542
|
} finally {
|
|
541
543
|
hold.renewing = false
|
|
@@ -553,6 +555,16 @@ export default class AdvisoryLockLockServerBackend {
|
|
|
553
555
|
this.status.unavailableUntil = Date.now() + this.reprobeCooldownMs
|
|
554
556
|
}
|
|
555
557
|
|
|
558
|
+
/**
|
|
559
|
+
* How long this backend still refuses to re-probe, in milliseconds. `0` means
|
|
560
|
+
* a probe is allowed on the next call, so there is no bounded reconnect event
|
|
561
|
+
* left to wait for.
|
|
562
|
+
* @returns {number} - Remaining re-probe cooldown, or 0 when one is not pending.
|
|
563
|
+
*/
|
|
564
|
+
reprobeCooldownRemainingMs() {
|
|
565
|
+
return Math.max(0, this.status.unavailableUntil - Date.now())
|
|
566
|
+
}
|
|
567
|
+
|
|
556
568
|
/**
|
|
557
569
|
* Drops transport state and marks every outstanding hold lost.
|
|
558
570
|
* @returns {void}
|
|
@@ -560,7 +572,7 @@ export default class AdvisoryLockLockServerBackend {
|
|
|
560
572
|
teardown() {
|
|
561
573
|
for (const [name, hold] of this.holds) {
|
|
562
574
|
if (hold.timer) clearTimeout(hold.timer)
|
|
563
|
-
this.fencedHolds.set(name, {currentEpoch: null, epoch: hold.epoch})
|
|
575
|
+
this.fencedHolds.set(name, {currentEpoch: null, epoch: hold.epoch, reason: "transport"})
|
|
564
576
|
}
|
|
565
577
|
|
|
566
578
|
this.holds.clear()
|
|
@@ -367,6 +367,9 @@ export default class AdvisoryLockRunner {
|
|
|
367
367
|
* A name whose minimum tier is `lock-server` is never served by a weaker
|
|
368
368
|
* tier: an absent or unreachable server raises
|
|
369
369
|
* `AdvisoryLockBackendUnavailableError` instead of reporting a fallback.
|
|
370
|
+
* A mandated name still waits for the backend's own bounded re-probe window
|
|
371
|
+
* whenever this acquisition's budget covers it, so a lock-server restart is
|
|
372
|
+
* not mistaken for an unavailable backend.
|
|
370
373
|
*
|
|
371
374
|
* `null` means "this tier did not serve the acquisition — continue with the
|
|
372
375
|
* next eligible tier". A returned value means the lock-server tier answered,
|
|
@@ -400,29 +403,55 @@ export default class AdvisoryLockRunner {
|
|
|
400
403
|
return null
|
|
401
404
|
}
|
|
402
405
|
|
|
406
|
+
// A mandated tier may only be abandoned once the backend has no bounded
|
|
407
|
+
// reconnect event left inside this acquisition's own budget. The backend
|
|
408
|
+
// publishes that window (`reprobeCooldownRemainingMs`), so the wait is
|
|
409
|
+
// bounded by the backend's own re-probe rather than by a retry policy.
|
|
410
|
+
const acquireBudgetMs = this.acquireTimeoutMs(args)
|
|
411
|
+
const transportWaitBudgetMs = mode === "wait" && acquireBudgetMs !== null ? acquireBudgetMs : null
|
|
403
412
|
/** @type {boolean} */
|
|
404
413
|
let acquired
|
|
414
|
+
/** @type {number} */
|
|
415
|
+
let waitedForTransportMs = 0
|
|
405
416
|
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
acquired = await backend.tryAcquire(name)
|
|
411
|
-
}
|
|
412
|
-
} catch (error) {
|
|
413
|
-
if (!(error instanceof AdvisoryLockServiceUnavailableError)) throw error
|
|
417
|
+
for (;;) {
|
|
418
|
+
try {
|
|
419
|
+
if (mode === "wait") {
|
|
420
|
+
const remainingBudgetMs = acquireBudgetMs === null ? null : Math.max(0, acquireBudgetMs - waitedForTransportMs)
|
|
414
421
|
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
reasons: [`lock-server unavailable: ${error.message}`]
|
|
420
|
-
})
|
|
421
|
-
}
|
|
422
|
+
acquired = await backend.acquire(name, {timeoutMs: remainingBudgetMs})
|
|
423
|
+
} else {
|
|
424
|
+
acquired = await backend.tryAcquire(name)
|
|
425
|
+
}
|
|
422
426
|
|
|
423
|
-
|
|
427
|
+
break
|
|
428
|
+
} catch (error) {
|
|
429
|
+
if (!(error instanceof AdvisoryLockServiceUnavailableError)) throw error
|
|
424
430
|
|
|
425
|
-
|
|
431
|
+
if (policy.minTier !== "lock-server") {
|
|
432
|
+
this.reportAdvisoryLockFallback({reason: error.message, servedBy: "driver-native", lockName: name})
|
|
433
|
+
|
|
434
|
+
return null
|
|
435
|
+
}
|
|
436
|
+
|
|
437
|
+
const cooldownMs = typeof backend.reprobeCooldownRemainingMs === "function" ? backend.reprobeCooldownRemainingMs() : 0
|
|
438
|
+
// `or-fail`/`defer` declare no wait budget, so their bound is the
|
|
439
|
+
// backend's own single re-probe cycle: one reconnect attempt, never a queue.
|
|
440
|
+
const waitBudgetMs = transportWaitBudgetMs ?? cooldownMs
|
|
441
|
+
|
|
442
|
+
if (cooldownMs <= 0 || waitedForTransportMs + cooldownMs > waitBudgetMs) {
|
|
443
|
+
throw advisoryLockBackendUnavailable({
|
|
444
|
+
lockName: name,
|
|
445
|
+
minTier: policy.minTier,
|
|
446
|
+
reasons: [`lock-server unavailable: ${error.message}`]
|
|
447
|
+
})
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
await new Promise((resolve) => {
|
|
451
|
+
setTimeout(resolve, cooldownMs)
|
|
452
|
+
})
|
|
453
|
+
waitedForTransportMs += cooldownMs
|
|
454
|
+
}
|
|
426
455
|
}
|
|
427
456
|
|
|
428
457
|
if (!acquired) {
|
|
@@ -485,12 +514,13 @@ export default class AdvisoryLockRunner {
|
|
|
485
514
|
|
|
486
515
|
if (loss) {
|
|
487
516
|
throw new AdvisoryLockLostError(
|
|
488
|
-
`Lost the lease for advisory lock ${JSON.stringify(name)} while the critical section was running`,
|
|
517
|
+
`Lost the lease for advisory lock ${JSON.stringify(name)} while the critical section was running (reason=${loss.reason}; epoch=${String(loss.epoch)}; currentEpoch=${String(loss.currentEpoch)})`,
|
|
489
518
|
{
|
|
490
519
|
cause: callbackError instanceof Error ? callbackError : undefined,
|
|
491
520
|
currentEpoch: loss.currentEpoch,
|
|
492
521
|
epoch: lockBackend.epoch ?? null,
|
|
493
|
-
lockName: name
|
|
522
|
+
lockName: name,
|
|
523
|
+
reason: loss.reason
|
|
494
524
|
}
|
|
495
525
|
)
|
|
496
526
|
}
|
|
@@ -9,6 +9,19 @@ export default class VelociousDatabaseDriversBaseTable {
|
|
|
9
9
|
* @type {import("./base.js").default | undefined} */
|
|
10
10
|
driver = undefined
|
|
11
11
|
|
|
12
|
+
/**
|
|
13
|
+
* Builds this table's column objects from raw introspection rows that were already read,
|
|
14
|
+
* so a batched catalog read and a per-table read produce the same columns.
|
|
15
|
+
* @abstract
|
|
16
|
+
* @param {import("./base.js").QueryResultType} columnRows - Raw column rows for this table.
|
|
17
|
+
* @returns {Array<import("./base-column.js").default>} - Columns built from the rows.
|
|
18
|
+
*/
|
|
19
|
+
columnsFromRows(columnRows) {
|
|
20
|
+
void columnRows
|
|
21
|
+
|
|
22
|
+
throw new Error("columnsFromRows not implemented")
|
|
23
|
+
}
|
|
24
|
+
|
|
12
25
|
/**
|
|
13
26
|
* Runs get column by name.
|
|
14
27
|
* @param {string} columnName - Column name.
|
|
@@ -63,6 +63,23 @@ export default class VelociousDatabaseDriversMssqlTable extends BaseTable {
|
|
|
63
63
|
this.driver = driver
|
|
64
64
|
}
|
|
65
65
|
|
|
66
|
+
/**
|
|
67
|
+
* Builds this table's column objects from raw `INFORMATION_SCHEMA.COLUMNS`-shaped rows.
|
|
68
|
+
* @param {import("../base.js").QueryResultType} columnRows - Raw column rows for this table.
|
|
69
|
+
* @returns {Array<import("../base-column.js").default>} - Columns built from the rows.
|
|
70
|
+
*/
|
|
71
|
+
columnsFromRows(columnRows) {
|
|
72
|
+
const columns = []
|
|
73
|
+
|
|
74
|
+
for (const data of columnRows) {
|
|
75
|
+
const column = new Column(this, data)
|
|
76
|
+
|
|
77
|
+
columns.push(column)
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
return columns
|
|
81
|
+
}
|
|
82
|
+
|
|
66
83
|
async getColumns() {
|
|
67
84
|
const columnRows = await this.getDriver()._cachedTableSchemaMetadata(this.getName(), "columns", async () => {
|
|
68
85
|
const result = await this.driver.query(`
|
|
@@ -88,15 +105,8 @@ export default class VelociousDatabaseDriversMssqlTable extends BaseTable {
|
|
|
88
105
|
`)
|
|
89
106
|
return result
|
|
90
107
|
})
|
|
91
|
-
const columns = []
|
|
92
|
-
|
|
93
|
-
for (const data of columnRows) {
|
|
94
|
-
const column = new Column(this, data)
|
|
95
108
|
|
|
96
|
-
|
|
97
|
-
}
|
|
98
|
-
|
|
99
|
-
return columns
|
|
109
|
+
return this.columnsFromRows(columnRows)
|
|
100
110
|
}
|
|
101
111
|
|
|
102
112
|
async getForeignKeys() {
|
|
@@ -17,10 +17,12 @@ export default class VelociousDatabaseDriversMysqlTable extends BaseTable {
|
|
|
17
17
|
this.driver = driver
|
|
18
18
|
}
|
|
19
19
|
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
20
|
+
/**
|
|
21
|
+
* Builds this table's column objects from raw `SHOW FULL COLUMNS`-shaped rows.
|
|
22
|
+
* @param {import("../base.js").QueryResultType} columnRows - Raw column rows for this table.
|
|
23
|
+
* @returns {Array<import("../base-column.js").default>} - Columns built from the rows.
|
|
24
|
+
*/
|
|
25
|
+
columnsFromRows(columnRows) {
|
|
24
26
|
const columns = []
|
|
25
27
|
|
|
26
28
|
for (const data of columnRows) {
|
|
@@ -32,6 +34,14 @@ export default class VelociousDatabaseDriversMysqlTable extends BaseTable {
|
|
|
32
34
|
return columns
|
|
33
35
|
}
|
|
34
36
|
|
|
37
|
+
async getColumns() {
|
|
38
|
+
const columnRows = await this.getDriver()._cachedTableSchemaMetadata(this.getName(), "columns", async () => {
|
|
39
|
+
return await this.driver.query(`SHOW FULL COLUMNS FROM \`${this.getName()}\``)
|
|
40
|
+
})
|
|
41
|
+
|
|
42
|
+
return this.columnsFromRows(columnRows)
|
|
43
|
+
}
|
|
44
|
+
|
|
35
45
|
async getForeignKeys() {
|
|
36
46
|
const foreignKeyRows = await this.getDriver()._cachedTableSchemaMetadata(this.getName(), "foreignKeys", async () => {
|
|
37
47
|
const sql = `
|
|
@@ -62,6 +62,23 @@ export default class VelociousDatabaseDriversPgsqlTable extends BaseTable {
|
|
|
62
62
|
this.driver = driver
|
|
63
63
|
}
|
|
64
64
|
|
|
65
|
+
/**
|
|
66
|
+
* Builds this table's column objects from raw `information_schema.columns`-shaped rows.
|
|
67
|
+
* @param {import("../base.js").QueryResultType} columnRows - Raw column rows for this table.
|
|
68
|
+
* @returns {Array<import("../base-column.js").default>} - Columns built from the rows.
|
|
69
|
+
*/
|
|
70
|
+
columnsFromRows(columnRows) {
|
|
71
|
+
const columns = []
|
|
72
|
+
|
|
73
|
+
for (const data of columnRows) {
|
|
74
|
+
const column = new Column(this, data)
|
|
75
|
+
|
|
76
|
+
columns.push(column)
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
return columns
|
|
80
|
+
}
|
|
81
|
+
|
|
65
82
|
async getColumns() {
|
|
66
83
|
const columnRows = await this.getDriver()._cachedTableSchemaMetadata(this.getName(), "columns", async () => {
|
|
67
84
|
const result = await this.driver.query(`
|
|
@@ -91,15 +108,8 @@ export default class VelociousDatabaseDriversPgsqlTable extends BaseTable {
|
|
|
91
108
|
`)
|
|
92
109
|
return result
|
|
93
110
|
})
|
|
94
|
-
const columns = []
|
|
95
|
-
|
|
96
|
-
for (const data of columnRows) {
|
|
97
|
-
const column = new Column(this, data)
|
|
98
111
|
|
|
99
|
-
|
|
100
|
-
}
|
|
101
|
-
|
|
102
|
-
return columns
|
|
112
|
+
return this.columnsFromRows(columnRows)
|
|
103
113
|
}
|
|
104
114
|
|
|
105
115
|
async getForeignKeys() {
|
|
@@ -19,13 +19,11 @@ export default class VelociousDatabaseDriversSqliteTable extends BaseTable {
|
|
|
19
19
|
}
|
|
20
20
|
|
|
21
21
|
/**
|
|
22
|
-
*
|
|
23
|
-
* @
|
|
22
|
+
* Builds this table's column objects from raw `PRAGMA table_info` rows.
|
|
23
|
+
* @param {import("../base.js").QueryResultType} columnRows - Raw column rows for this table.
|
|
24
|
+
* @returns {Array<import("../base-column.js").default>} - Columns built from the rows.
|
|
24
25
|
*/
|
|
25
|
-
|
|
26
|
-
const columnRows = await this.getDriver()._cachedTableSchemaMetadata(this.getName(), "columns", async () => {
|
|
27
|
-
return await this.driver.query(`PRAGMA table_info('${this.getName()}')`)
|
|
28
|
-
})
|
|
26
|
+
columnsFromRows(columnRows) {
|
|
29
27
|
const columns = []
|
|
30
28
|
|
|
31
29
|
for (const columnData of columnRows) {
|
|
@@ -37,6 +35,18 @@ export default class VelociousDatabaseDriversSqliteTable extends BaseTable {
|
|
|
37
35
|
return columns
|
|
38
36
|
}
|
|
39
37
|
|
|
38
|
+
/**
|
|
39
|
+
* Runs get columns.
|
|
40
|
+
* @returns {Promise<Array<import("../base-column.js").default>>} - Resolves with the columns.
|
|
41
|
+
*/
|
|
42
|
+
async getColumns() {
|
|
43
|
+
const columnRows = await this.getDriver()._cachedTableSchemaMetadata(this.getName(), "columns", async () => {
|
|
44
|
+
return await this.driver.query(`PRAGMA table_info('${this.getName()}')`)
|
|
45
|
+
})
|
|
46
|
+
|
|
47
|
+
return this.columnsFromRows(columnRows)
|
|
48
|
+
}
|
|
49
|
+
|
|
40
50
|
async getForeignKeys() {
|
|
41
51
|
const foreignKeysData = await this.getDriver()._cachedTableSchemaMetadata(this.getName(), "foreignKeys", async () => {
|
|
42
52
|
return await this.driver.query(`SELECT * FROM pragma_foreign_key_list(${this.driver.quote(this.getName())})`)
|
|
@@ -26,7 +26,8 @@ import Current from "../current.js"
|
|
|
26
26
|
* @property {string[]} [keyColumns] - Columns the aggregate is grouped by (for example `["docker_server_id"]`). Empty means a single grand-total row.
|
|
27
27
|
* @property {Record<string, AggregateSpec>} aggregates - Output column name to aggregate. `"SUM"` is shorthand for `{op: "SUM", column: <output name>}`; use `{op: "COUNT", column: "*"}` for `COUNT(*)`.
|
|
28
28
|
* @property {AbortSignal} [signal] - Signal passed to each aggregate database query.
|
|
29
|
-
* @property {(context: SubqueryContext) => string} subquery - Builds one tenant's inner `SELECT`, which must select every `keyColumns` entry plus every aggregate source column.
|
|
29
|
+
* @property {(context: SubqueryContext) => string} subquery - Builds one tenant's inner `SELECT`, which must select every `keyColumns` entry plus every aggregate source column. In `rowIdentity` mode its own select list is the result set.
|
|
30
|
+
* @property {boolean} [rowIdentity] - Opt out of the aggregation shape entirely when each tenant's `subquery` already returns the result rows and its own select list is the result. Pass `aggregates: {}` together with `rowIdentity: true`: every branch is emitted as written and joined with `UNION ALL` (each arm parenthesised when there is more than one), with no `SELECT * FROM (...)` derived table and no outer `SELECT ... GROUP BY`, and the rows come back exactly as the engines produced them. `keyColumns` must be omitted — nothing is grouped or merged, so duplicates across tenants are kept. Combining it with aggregates or key columns throws.
|
|
30
31
|
* @property {import("../configuration.js").default} [configuration] - Configuration to run against. Defaults to the current configuration.
|
|
31
32
|
*/
|
|
32
33
|
|
|
@@ -46,6 +47,13 @@ const DERIVED_TABLE_ALIAS = "velocious_tenant_aggregate"
|
|
|
46
47
|
* (PostgreSQL/SQLite). Results from every server are merged with the aggregate's own operation, so
|
|
47
48
|
* callers get one combined result set regardless of how the tenants are distributed. Reached through
|
|
48
49
|
* `Tenant.aggregateAcross`.
|
|
50
|
+
*
|
|
51
|
+
* The aggregation shape wraps every branch in `SELECT * FROM (...)` and groups the union, which is
|
|
52
|
+
* what makes `SUM`/`COUNT`/`MAX`/`MIN` mergeable across branches. A caller whose branches already
|
|
53
|
+
* return the result rows does not need it and pays for it: an engine that cannot merge a branch into
|
|
54
|
+
* the union materialises each wrapper as a temp table, and the outer grouping adds one more.
|
|
55
|
+
* `rowIdentity: true` emits the branches as written instead (parenthesised inside a multi-branch
|
|
56
|
+
* UNION), so the union streams with no derived table and no grouping pass.
|
|
49
57
|
*/
|
|
50
58
|
export default class TenantAggregator {
|
|
51
59
|
/**
|
|
@@ -56,12 +64,25 @@ export default class TenantAggregator {
|
|
|
56
64
|
this.options = options
|
|
57
65
|
this.configuration = options.configuration ?? Current.configuration()
|
|
58
66
|
this.keyColumns = options.keyColumns ?? []
|
|
67
|
+
this.rowIdentity = options.rowIdentity === true
|
|
68
|
+
|
|
69
|
+
// `rowIdentity` is an explicit opt-out of the aggregation shape, so it is checked before the
|
|
70
|
+
// aggregate specs are normalized: an omitted `aggregates` is the natural mistake here and should
|
|
71
|
+
// read as a contract error, not as a crash inside the normalizer.
|
|
72
|
+
if (this.rowIdentity && options.aggregates === undefined) {
|
|
73
|
+
throw new Error("Tenant.aggregateAcross rowIdentity mode requires `aggregates: {}` — pass it explicitly to declare that nothing is aggregated.")
|
|
74
|
+
}
|
|
75
|
+
|
|
59
76
|
this.aggregates = this._normalizeAggregates(options.aggregates)
|
|
77
|
+
|
|
78
|
+
if (this.rowIdentity) {
|
|
79
|
+
this._assertRowIdentityContract()
|
|
80
|
+
}
|
|
60
81
|
}
|
|
61
82
|
|
|
62
83
|
/**
|
|
63
84
|
* Resolves the tenant list, runs the aggregate per server, and merges everything.
|
|
64
|
-
* @returns {Promise<Array<Record<string, ReturnType<typeof JSON.parse>>>>} - One merged row per distinct key-column combination.
|
|
85
|
+
* @returns {Promise<Array<Record<string, ReturnType<typeof JSON.parse>>>>} - One merged row per distinct key-column combination, or — in `rowIdentity` mode — the rows every branch produced.
|
|
65
86
|
*/
|
|
66
87
|
async run() {
|
|
67
88
|
const resolvedTenants = await this._resolveTenants()
|
|
@@ -77,6 +98,10 @@ export default class TenantAggregator {
|
|
|
77
98
|
rows.push(...groupRows)
|
|
78
99
|
}
|
|
79
100
|
|
|
101
|
+
// `rowIdentity` branches are already the result rows: merging them by key would collapse
|
|
102
|
+
// distinct tenants' rows into one, which is exactly what the caller opted out of.
|
|
103
|
+
if (this.rowIdentity) return rows
|
|
104
|
+
|
|
80
105
|
return this._mergeRows(rows)
|
|
81
106
|
}
|
|
82
107
|
|
|
@@ -203,7 +228,22 @@ export default class TenantAggregator {
|
|
|
203
228
|
}
|
|
204
229
|
|
|
205
230
|
/**
|
|
206
|
-
*
|
|
231
|
+
* Rejects the option combinations that would silently change what a `rowIdentity` run returns.
|
|
232
|
+
* @returns {void}
|
|
233
|
+
*/
|
|
234
|
+
_assertRowIdentityContract() {
|
|
235
|
+
if (Object.keys(this.aggregates).length > 0) {
|
|
236
|
+
throw new Error("Tenant.aggregateAcross cannot combine rowIdentity with aggregates — rowIdentity returns every branch row as it was read, so there is nothing to aggregate.")
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
if (this.keyColumns.length > 0) {
|
|
240
|
+
throw new Error("Tenant.aggregateAcross cannot combine rowIdentity with keyColumns — no GROUP BY is emitted in rowIdentity mode, so the subquery's own select list is the result.")
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
/**
|
|
245
|
+
* Builds the aggregate SQL: an outer `GROUP BY` over a `UNION ALL` of each entry's subquery, or —
|
|
246
|
+
* in `rowIdentity` mode — the entry subqueries joined by `UNION ALL` with no wrapper at all.
|
|
207
247
|
* @param {{connection: import("../database/drivers/base.js").default, entries: ResolvedTenant[], qualified: boolean}} args - Connection, tenants, and whether to qualify tables with their database name.
|
|
208
248
|
* @returns {string} - Executable aggregate SQL.
|
|
209
249
|
*/
|
|
@@ -222,6 +262,19 @@ export default class TenantAggregator {
|
|
|
222
262
|
return this.options.subquery({connection, quote, table, tenant: entry.tenant})
|
|
223
263
|
})
|
|
224
264
|
|
|
265
|
+
// Row-identity branches are the result rows, so they are unioned as they are: no derived-table
|
|
266
|
+
// wrapper and no outer grouping. A multi-branch union only reaches a driver that reported
|
|
267
|
+
// `supportsCrossDatabaseReferences()` (MySQL/MariaDB), and there each arm is parenthesised: that
|
|
268
|
+
// keeps a branch's own `ORDER BY`/`LIMIT` legal inside the UNION — unparenthesised it is a
|
|
269
|
+
// `ER_PARSE_ERROR` — while costing the same as a bare union (measured on MariaDB 12.2: 20
|
|
270
|
+
// parenthesised branches report 0 `Created_tmp_tables` and no `DERIVED` plan row). A single branch
|
|
271
|
+
// needs no arm parentheses, and the fan-out drivers (PostgreSQL/SQLite/MSSQL) never get a UNION.
|
|
272
|
+
if (this.rowIdentity) {
|
|
273
|
+
const branches = subqueries.map((subquery) => subqueries.length === 1 ? subquery : `(${subquery})`)
|
|
274
|
+
|
|
275
|
+
return branches.join("\nUNION ALL\n")
|
|
276
|
+
}
|
|
277
|
+
|
|
225
278
|
const selectParts = this.keyColumns.map((keyColumn) => options.quoteColumnName(keyColumn))
|
|
226
279
|
|
|
227
280
|
for (const [name, spec] of Object.entries(this.aggregates)) {
|
package/src/tenants/tenant.js
CHANGED
|
@@ -136,9 +136,11 @@ export default class Tenant {
|
|
|
136
136
|
* `UNION ALL` where the driver supports it (MySQL/MSSQL) or one query per tenant otherwise
|
|
137
137
|
* (PostgreSQL/SQLite), and merged with each aggregate's own operation. The caller writes only one
|
|
138
138
|
* per-tenant subquery and declares the key columns and aggregates; see
|
|
139
|
-
* {@link import("./tenant-aggregator.js").TenantAggregateOptions}.
|
|
139
|
+
* {@link import("./tenant-aggregator.js").TenantAggregateOptions}. `aggregates: {}` with
|
|
140
|
+
* `rowIdentity: true` skips the aggregation shape instead: every branch is unioned verbatim and its
|
|
141
|
+
* rows are returned as read, with no derived-table wrapper and no outer `GROUP BY`.
|
|
140
142
|
* @param {import("./tenant-aggregator.js").TenantAggregateOptions} options - Aggregate configuration.
|
|
141
|
-
* @returns {Promise<Array<Record<string, ReturnType<typeof JSON.parse>>>>} - One merged row per distinct key-column combination.
|
|
143
|
+
* @returns {Promise<Array<Record<string, ReturnType<typeof JSON.parse>>>>} - One merged row per distinct key-column combination, or the rows every branch produced in `rowIdentity` mode.
|
|
142
144
|
*/
|
|
143
145
|
static async aggregateAcross(options) {
|
|
144
146
|
return await new TenantAggregator(options).run()
|