velocious 1.0.758 → 1.0.760
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/build/background-jobs/worker.js +55 -5
- package/build/database/advisory-lock-errors.js +3 -2
- package/build/database/advisory-lock-lock-server-backend.js +19 -7
- package/build/database/advisory-lock-runner.js +49 -19
- package/build/database/drivers/base-table.js +13 -0
- package/build/database/drivers/mssql/table.js +18 -8
- package/build/database/drivers/mysql/table.js +14 -4
- package/build/database/drivers/pgsql/table.js +18 -8
- package/build/database/drivers/sqlite/table.js +16 -6
- package/build/src/background-jobs/worker.d.ts +25 -2
- package/build/src/background-jobs/worker.d.ts.map +1 -1
- package/build/src/background-jobs/worker.js +54 -6
- package/build/src/database/advisory-lock-errors.d.ts +4 -2
- package/build/src/database/advisory-lock-errors.d.ts.map +1 -1
- package/build/src/database/advisory-lock-errors.js +4 -3
- package/build/src/database/advisory-lock-lock-server-backend.d.ts +19 -10
- package/build/src/database/advisory-lock-lock-server-backend.d.ts.map +1 -1
- package/build/src/database/advisory-lock-lock-server-backend.js +19 -8
- package/build/src/database/advisory-lock-runner.d.ts +3 -0
- package/build/src/database/advisory-lock-runner.d.ts.map +1 -1
- package/build/src/database/advisory-lock-runner.js +46 -20
- package/build/src/database/drivers/base-table.d.ts +8 -0
- package/build/src/database/drivers/base-table.d.ts.map +1 -1
- package/build/src/database/drivers/base-table.js +12 -1
- package/build/src/database/drivers/mssql/table.d.ts +7 -2
- package/build/src/database/drivers/mssql/table.d.ts.map +1 -1
- package/build/src/database/drivers/mssql/table.js +15 -7
- package/build/src/database/drivers/mysql/table.d.ts +7 -2
- package/build/src/database/drivers/mysql/table.d.ts.map +1 -1
- package/build/src/database/drivers/mysql/table.js +13 -5
- package/build/src/database/drivers/pgsql/table.d.ts +7 -2
- package/build/src/database/drivers/pgsql/table.d.ts.map +1 -1
- package/build/src/database/drivers/pgsql/table.js +15 -7
- package/build/src/database/drivers/sqlite/table.d.ts +6 -0
- package/build/src/database/drivers/sqlite/table.d.ts.map +1 -1
- package/build/src/database/drivers/sqlite/table.js +15 -7
- package/build/src/tenants/tenant-aggregator.d.ts +21 -3
- package/build/src/tenants/tenant-aggregator.d.ts.map +1 -1
- package/build/src/tenants/tenant-aggregator.js +50 -4
- package/build/src/tenants/tenant.d.ts +4 -2
- package/build/src/tenants/tenant.d.ts.map +1 -1
- package/build/src/tenants/tenant.js +5 -3
- package/build/tenants/tenant-aggregator.js +56 -3
- package/build/tenants/tenant.js +4 -2
- package/package.json +1 -1
- package/src/background-jobs/worker.js +55 -5
- package/src/database/advisory-lock-errors.js +3 -2
- package/src/database/advisory-lock-lock-server-backend.js +19 -7
- package/src/database/advisory-lock-runner.js +49 -19
- package/src/database/drivers/base-table.js +13 -0
- package/src/database/drivers/mssql/table.js +18 -8
- package/src/database/drivers/mysql/table.js +14 -4
- package/src/database/drivers/pgsql/table.js +18 -8
- package/src/database/drivers/sqlite/table.js +16 -6
- package/src/tenants/tenant-aggregator.js +56 -3
- package/src/tenants/tenant.js +4 -2
package/README.md
CHANGED
|
@@ -3078,7 +3078,7 @@ Tenant lifecycle commands print start and final counts, report each completed te
|
|
|
3078
3078
|
|
|
3079
3079
|
`afterMigrateTenant` hooks run inside the active default and tenant database connection scope for the tenant being migrated.
|
|
3080
3080
|
|
|
3081
|
-
At runtime, the apartment-style `Tenant` façade (`velocious/build/src/tenants/tenant.js`) is the single entry point: `Tenant.with(tenant, callback)` / `Tenant.current()` to switch into and read a Node async context, `Tenant.handle(tenant)` to deeply capture immutable application and physical database identity for overlapping browser/native work, `Tenant.each({identifier, callback, parallel?, filter?})` to run a callback within every provider-listed tenant, and `Tenant.drop({identifier, tenant})` (plus the `db:tenants:drop` CLI command) to drop a tenant's database through the provider's `dropDatabase` hook. SQLite handles additionally expose framework-owned `open`, `flush`, `close`, `delete`, `inspect`, and `withPin` lifecycle methods; `frontendTenantSqlite.maxOpenHandles` bounds resident identities and clean, idle, unpinned handles are evicted least-recently-used. `Tenant.handle(...).databaseOperation(...)` and `.transaction(...)` use bounded pool-owned checkouts and pin model/query/write/association/preload/audit/attachment/raw work to the captured database even if a later UI project switch changes ambient tenant state. The handle also builds tenant-bound live-query sources, exposes an opaque physical identity for filtered record-change subscriptions, and binds project `SyncClient` instances. `Model.usingTenant(tenant)` uses the same safe core and adds eager helpers plus general `databaseOperation`/`transaction` model callbacks; eager records preserve legacy ambient Node follow-up semantics, while browser/native follow-up database work belongs inside the callback APIs. Inactive identifiers, mixed physical tenants, unsupported/cyclic descriptors, expired operations, unscoped tenant event subscriptions, and stale/cross-tenant sync state fail closed. `Tenant.with` and `Tenant.each` retain their connection-establishing and model-initializing ambient behavior for Node request/job flows. `Tenant.aggregateAcross({identifier, aggregates, keyColumns, subquery, tenants?, filter?})` runs one aggregate over the same table across many tenant databases and returns the merged result — grouping tenants by server and using a single cross-database `UNION ALL` where the driver supports two-part `` `database`.`table` `` references (MySQL/MariaDB) or one query per tenant otherwise (PostgreSQL/SQLite/MSSQL).
|
|
3081
|
+
At runtime, the apartment-style `Tenant` façade (`velocious/build/src/tenants/tenant.js`) is the single entry point: `Tenant.with(tenant, callback)` / `Tenant.current()` to switch into and read a Node async context, `Tenant.handle(tenant)` to deeply capture immutable application and physical database identity for overlapping browser/native work, `Tenant.each({identifier, callback, parallel?, filter?})` to run a callback within every provider-listed tenant, and `Tenant.drop({identifier, tenant})` (plus the `db:tenants:drop` CLI command) to drop a tenant's database through the provider's `dropDatabase` hook. SQLite handles additionally expose framework-owned `open`, `flush`, `close`, `delete`, `inspect`, and `withPin` lifecycle methods; `frontendTenantSqlite.maxOpenHandles` bounds resident identities and clean, idle, unpinned handles are evicted least-recently-used. `Tenant.handle(...).databaseOperation(...)` and `.transaction(...)` use bounded pool-owned checkouts and pin model/query/write/association/preload/audit/attachment/raw work to the captured database even if a later UI project switch changes ambient tenant state. The handle also builds tenant-bound live-query sources, exposes an opaque physical identity for filtered record-change subscriptions, and binds project `SyncClient` instances. `Model.usingTenant(tenant)` uses the same safe core and adds eager helpers plus general `databaseOperation`/`transaction` model callbacks; eager records preserve legacy ambient Node follow-up semantics, while browser/native follow-up database work belongs inside the callback APIs. Inactive identifiers, mixed physical tenants, unsupported/cyclic descriptors, expired operations, unscoped tenant event subscriptions, and stale/cross-tenant sync state fail closed. `Tenant.with` and `Tenant.each` retain their connection-establishing and model-initializing ambient behavior for Node request/job flows. `Tenant.aggregateAcross({identifier, aggregates, keyColumns, subquery, tenants?, filter?, rowIdentity?})` runs one aggregate over the same table across many tenant databases and returns the merged result — grouping tenants by server and using a single cross-database `UNION ALL` where the driver supports two-part `` `database`.`table` `` references (MySQL/MariaDB) or one query per tenant otherwise (PostgreSQL/SQLite/MSSQL). `aggregates: {}` with `rowIdentity: true` skips the aggregation shape for callers whose branches already return the rows: every branch is unioned as written, with no derived-table wrapper, no outer grouping and no per-key merge, so each tenant's rows come back as read ([row-identity rows](docs/tenant-databases.md#row-identity-rows-instead-of-an-aggregate)).
|
|
3082
3082
|
|
|
3083
3083
|
`SchemaCloner` adds a missing auto-increment column and its separate source unique index in one schema alteration, including on MySQL/MariaDB where an auto-increment column must be keyed when it is created.
|
|
3084
3084
|
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
import net from "net"
|
|
4
4
|
import { fork, spawn } from "node:child_process"
|
|
5
|
+
import timeout, {TimeoutError} from "awaitery/build/timeout.js"
|
|
5
6
|
import JsonSocket from "./json-socket.js"
|
|
6
7
|
import BackgroundJobRegistry from "./job-registry.js"
|
|
7
8
|
import configurationResolver from "../configuration-resolver.js"
|
|
@@ -99,6 +100,20 @@ const HEARTBEAT_INTERVAL_MS = 15000
|
|
|
99
100
|
const CHILD_ACCEPTANCE_REPORT_MAX_DURATION_MS = 10000
|
|
100
101
|
/** TCP keepalive so a half-open connection to the main surfaces as a close. */
|
|
101
102
|
const SOCKET_KEEPALIVE_MS = 10000
|
|
103
|
+
/**
|
|
104
|
+
* Bound on one owned shutdown close (`configuration.shutdown()`, the Beacon
|
|
105
|
+
* disconnect, the database close). Each of them is somebody else's code — the
|
|
106
|
+
* application's initializers, the Beacon client, the driver pool — and the
|
|
107
|
+
* worker's retirement contract is that the generation closes itself, so none of
|
|
108
|
+
* them may be waited on without end. Production held background-jobs-worker pid
|
|
109
|
+
* 1363854 (release 20261122033406) alive 5 h past its retirement with its Beacon
|
|
110
|
+
* socket still open and its Socketduct durable log sink still open, because the
|
|
111
|
+
* application teardown before `disconnectBeacon()` never settled and nothing
|
|
112
|
+
* bounded it, so the CLI never reached its exit. 30 s leaves room for this
|
|
113
|
+
* deployment's 4 s Socketduct stream-drain bound plus a durable-log flush, and a
|
|
114
|
+
* close that runs out is reported as the named step that ran out.
|
|
115
|
+
*/
|
|
116
|
+
const WORKER_SHUTDOWN_CLOSE_TIMEOUT_MS = 30000
|
|
102
117
|
/**
|
|
103
118
|
* Execution modes.
|
|
104
119
|
* @type {import("./types.js").BackgroundJobExecutionMode[]} */
|
|
@@ -262,6 +277,7 @@ export default class BackgroundJobsWorker {
|
|
|
262
277
|
* @param {number} [args.reconnectDelayMs] - Delay before reconnecting an established worker connection (default: 1000).
|
|
263
278
|
* @param {number} [args.jobTimeoutMs] - Override the wall-clock timeout for forked and pooled jobs from `configuration.getBackgroundJobsConfig()`. `0` disables it.
|
|
264
279
|
* @param {number | null} [args.workerShutdownTimeoutMs] - Override the wall-clock cap on one whole shutdown drain from `configuration.getBackgroundJobsConfig().workerShutdownTimeoutMs`. `null` drains without a bound.
|
|
280
|
+
* @param {number} [args.shutdownCloseTimeoutMs] - Override the bound on one owned shutdown close (application teardown, Beacon disconnect, database close) in ms (default 30000).
|
|
265
281
|
* @param {number} [args.pooledRunnerSilenceTimeoutMs] - Override the per-child IPC-silence bound for pooled children with in-flight jobs from `configuration.getBackgroundJobsConfig()` (default 120000ms). `0` disables the watchdog.
|
|
266
282
|
* @param {boolean} [args.closeDatabaseConnectionsOnStop] - Whether stop owns closing the configuration's database pools (default true).
|
|
267
283
|
* @param {() => void | Promise<void>} [args.onStopped] - Lifecycle hook invoked after the worker finishes stopping.
|
|
@@ -270,7 +286,7 @@ export default class BackgroundJobsWorker {
|
|
|
270
286
|
* @param {(observation: import("./types.js").PooledChildMemoryObservation) => void | Promise<void>} [args.onPooledRunnerMemoryObservation] - Explicit pooled-child memory observation hook. Every validated observation (periodic while a child has in-flight jobs, plus on demand) is forwarded here, in addition to the worker's compact stderr log line, so an application can route memory diagnostics (e.g. to a bug reporter) without parsing logs.
|
|
271
287
|
* @param {(payload: import("./types.js").BackgroundJobPayload) => void} [args.onJobAccepted] - Explicit job-acceptance observation hook. Fires the moment the worker accepts a dispatched job payload, before execution mode routing, so an observer can wait for the pickup event instead of polling in-flight sets.
|
|
272
288
|
*/
|
|
273
|
-
constructor({configuration, host, port, generationId, workerInstanceId, maxConcurrentForkedJobs, maxConcurrentInlineJobs, pooledRunnerCount, pooledRunnerConcurrency, pooledRunnerMaxJobs, pooledRunnerMaxRssBytes, pooledRunnerMaxLifetimeMs, runnerMaxOldSpaceSizeMb, runnerMallocArenaMax, forkedChildSigkillGraceMs, heartbeatIntervalMs, generationHandshakeTimeoutMs = DEFAULT_GENERATION_HANDSHAKE_TIMEOUT_MS, reconnectDelayMs = 1000, jobTimeoutMs, workerShutdownTimeoutMs, pooledRunnerSilenceTimeoutMs, closeDatabaseConnectionsOnStop = true, onStopped, onGenerationAccepted, onRetireMessage, onPooledRunnerMemoryObservation, onJobAccepted} = {}) {
|
|
289
|
+
constructor({configuration, host, port, generationId, workerInstanceId, maxConcurrentForkedJobs, maxConcurrentInlineJobs, pooledRunnerCount, pooledRunnerConcurrency, pooledRunnerMaxJobs, pooledRunnerMaxRssBytes, pooledRunnerMaxLifetimeMs, runnerMaxOldSpaceSizeMb, runnerMallocArenaMax, forkedChildSigkillGraceMs, heartbeatIntervalMs, generationHandshakeTimeoutMs = DEFAULT_GENERATION_HANDSHAKE_TIMEOUT_MS, reconnectDelayMs = 1000, jobTimeoutMs, workerShutdownTimeoutMs, shutdownCloseTimeoutMs, pooledRunnerSilenceTimeoutMs, closeDatabaseConnectionsOnStop = true, onStopped, onGenerationAccepted, onRetireMessage, onPooledRunnerMemoryObservation, onJobAccepted} = {}) {
|
|
274
290
|
/**
|
|
275
291
|
* Narrows the runtime value to the documented type.
|
|
276
292
|
* @type {Promise<import("../configuration.js").default>} */
|
|
@@ -362,6 +378,12 @@ export default class BackgroundJobsWorker {
|
|
|
362
378
|
: (typeof workerShutdownTimeoutMs === "number" && Number.isInteger(workerShutdownTimeoutMs) && workerShutdownTimeoutMs > 0 ? workerShutdownTimeoutMs : null)
|
|
363
379
|
/** @type {number | null} */
|
|
364
380
|
this.workerShutdownTimeoutMs = this.workerShutdownTimeoutMsOverride ?? null
|
|
381
|
+
/**
|
|
382
|
+
* Bound on one owned shutdown close, in ms. Constructor override wins so a
|
|
383
|
+
* spec can exercise the bound without waiting for the production one.
|
|
384
|
+
* @type {number}
|
|
385
|
+
*/
|
|
386
|
+
this.shutdownCloseTimeoutMs = positiveInteger(shutdownCloseTimeoutMs) || WORKER_SHUTDOWN_CLOSE_TIMEOUT_MS
|
|
365
387
|
this.shouldStop = false
|
|
366
388
|
this.isRetiring = false
|
|
367
389
|
/** @type {Promise<void> | undefined} */
|
|
@@ -723,7 +745,8 @@ export default class BackgroundJobsWorker {
|
|
|
723
745
|
}
|
|
724
746
|
|
|
725
747
|
/**
|
|
726
|
-
* Closes application resources before framework resources when this worker owns them
|
|
748
|
+
* Closes application resources before framework resources when this worker owns them,
|
|
749
|
+
* with every close bounded so the shutdown always reaches its end.
|
|
727
750
|
* @returns {Promise<void>} - Resolves after every owned close succeeds.
|
|
728
751
|
*/
|
|
729
752
|
async _closeConfiguration() {
|
|
@@ -735,16 +758,43 @@ export default class BackgroundJobsWorker {
|
|
|
735
758
|
message: "Background jobs worker application and framework shutdown failed",
|
|
736
759
|
steps: [
|
|
737
760
|
...(this.closeDatabaseConnectionsOnStop
|
|
738
|
-
? [async () => await configuration.shutdown()]
|
|
761
|
+
? [async () => await this._boundedShutdownClose("application shutdown", () => configuration.shutdown())]
|
|
739
762
|
: []),
|
|
740
|
-
async () => await configuration.disconnectBeacon(),
|
|
763
|
+
async () => await this._boundedShutdownClose("Beacon disconnect", () => configuration.disconnectBeacon()),
|
|
741
764
|
...(this.closeDatabaseConnectionsOnStop
|
|
742
|
-
? [async () => await configuration.closeDatabaseConnections()]
|
|
765
|
+
? [async () => await this._boundedShutdownClose("database connection close", () => configuration.closeDatabaseConnections())]
|
|
743
766
|
: [])
|
|
744
767
|
]
|
|
745
768
|
})
|
|
746
769
|
}
|
|
747
770
|
|
|
771
|
+
/**
|
|
772
|
+
* Runs one owned shutdown close inside `shutdownCloseTimeoutMs` and names the step
|
|
773
|
+
* when it runs out.
|
|
774
|
+
*
|
|
775
|
+
* The close is not cancelled — it is left running and simply no longer waited on.
|
|
776
|
+
* That is the whole point: the next close (and then the CLI's exit) can happen at
|
|
777
|
+
* all. A close that never settling is a defect in whatever owns it, and it is
|
|
778
|
+
* reported here as the named step that ran out rather than as a worker that quietly
|
|
779
|
+
* never exits.
|
|
780
|
+
* @param {string} label - The step, named in the failure this reports.
|
|
781
|
+
* @param {(() => Promise<void>)} close - The close to run.
|
|
782
|
+
* @returns {Promise<void>} - Resolves when the close settles; rejects with the close's own error, or a named timeout.
|
|
783
|
+
*/
|
|
784
|
+
async _boundedShutdownClose(label, close) {
|
|
785
|
+
const timeoutMs = this.shutdownCloseTimeoutMs
|
|
786
|
+
|
|
787
|
+
try {
|
|
788
|
+
await timeout({timeout: timeoutMs}, close)
|
|
789
|
+
} catch (error) {
|
|
790
|
+
if (error instanceof TimeoutError) {
|
|
791
|
+
throw new Error(`Background jobs worker ${label} did not finish within ${timeoutMs}ms`, {cause: error})
|
|
792
|
+
}
|
|
793
|
+
|
|
794
|
+
throw error
|
|
795
|
+
}
|
|
796
|
+
}
|
|
797
|
+
|
|
748
798
|
/**
|
|
749
799
|
* Waits for a set of in-flight job promises to settle, optionally bounded by
|
|
750
800
|
* `timeoutMs`. When a bounded window runs out with entries still held, the drain
|
|
@@ -31,15 +31,16 @@ export class AdvisoryLockLostError extends Error {
|
|
|
31
31
|
/**
|
|
32
32
|
* Runs constructor.
|
|
33
33
|
* @param {string} message - Error message.
|
|
34
|
-
* @param {{cause?: Error, currentEpoch: number | null, epoch: number | null, lockName: string}} args -
|
|
34
|
+
* @param {{cause?: Error, currentEpoch: number | null, epoch: number | null, lockName: string, reason: "fenced" | "expired" | "transport"}} args - Loss details.
|
|
35
35
|
*/
|
|
36
|
-
constructor(message, {cause, currentEpoch, epoch, lockName}) {
|
|
36
|
+
constructor(message, {cause, currentEpoch, epoch, lockName, reason}) {
|
|
37
37
|
super(message)
|
|
38
38
|
this.name = "AdvisoryLockLostError"
|
|
39
39
|
this.lockName = lockName
|
|
40
40
|
this.epoch = epoch
|
|
41
41
|
this.currentEpoch = currentEpoch
|
|
42
42
|
this.cause = cause
|
|
43
|
+
this.reason = reason
|
|
43
44
|
}
|
|
44
45
|
}
|
|
45
46
|
|
|
@@ -11,6 +11,8 @@ import {AdvisoryLockServiceUnavailableError} from "./advisory-lock-errors.js"
|
|
|
11
11
|
* @property {() => Promise<void>} connect - Opens the transport.
|
|
12
12
|
* @property {(type: string, args: {onMessage: (body: Record<string, unknown>) => void, params: Record<string, unknown>, timeoutMs: number}) => LockServerConnectionLike} openConnection - Opens the 1:1 connection.
|
|
13
13
|
* @property {() => Promise<void> | void} [close] - Closes the transport.
|
|
14
|
+
* @typedef {"fenced" | "expired" | "transport"} LockLossReason
|
|
15
|
+
* @typedef {{currentEpoch: number | null, epoch: number | null, reason: LockLossReason}} LockLoss
|
|
14
16
|
* @typedef {{epoch: number, name: string, timer: ReturnType<typeof setTimeout> | null, leaseExpiresAt: number, renewing: boolean}} LockServerHold
|
|
15
17
|
* @typedef {{status: "acquired", epoch: number, leaseExpiresAt: number} | {status: "already-held", epoch: number, leaseExpiresAt: number} | {status: "busy"} | {status: "timeout"}} LockServerAcquireResult
|
|
16
18
|
*/
|
|
@@ -76,7 +78,7 @@ export default class AdvisoryLockLockServerBackend {
|
|
|
76
78
|
this.helloWaiter = null
|
|
77
79
|
/** @type {Map<string, LockServerHold>} */
|
|
78
80
|
this.holds = new Map()
|
|
79
|
-
/** @type {Map<string,
|
|
81
|
+
/** @type {Map<string, LockLoss>} */
|
|
80
82
|
this.fencedHolds = new Map()
|
|
81
83
|
/** @type {Map<string, {reject: (error: Error) => void, resolve: (message: Record<string, unknown>) => void, timer: ReturnType<typeof setTimeout> | null}>} */
|
|
82
84
|
this.pending = new Map()
|
|
@@ -289,7 +291,7 @@ export default class AdvisoryLockLockServerBackend {
|
|
|
289
291
|
* Whether this hold was fenced or lost during its critical section.
|
|
290
292
|
* @param {string} name - Lock name.
|
|
291
293
|
* @param {number | null} epoch - Epoch the hold was granted.
|
|
292
|
-
* @returns {
|
|
294
|
+
* @returns {LockLoss | null} - Fencing details when lost.
|
|
293
295
|
*/
|
|
294
296
|
takeLoss(name, epoch) {
|
|
295
297
|
const loss = this.fencedHolds.get(name)
|
|
@@ -302,7 +304,7 @@ export default class AdvisoryLockLockServerBackend {
|
|
|
302
304
|
|
|
303
305
|
const hold = this.holds.get(name)
|
|
304
306
|
|
|
305
|
-
if (hold && hold.epoch !== epoch) return {currentEpoch: hold.epoch, epoch}
|
|
307
|
+
if (hold && hold.epoch !== epoch) return {currentEpoch: hold.epoch, epoch, reason: "fenced"}
|
|
306
308
|
|
|
307
309
|
return null
|
|
308
310
|
}
|
|
@@ -382,7 +384,7 @@ export default class AdvisoryLockLockServerBackend {
|
|
|
382
384
|
|
|
383
385
|
this.holds.delete(name)
|
|
384
386
|
if (hold.timer) clearTimeout(hold.timer)
|
|
385
|
-
this.fencedHolds.set(name, {currentEpoch: Number(body.epoch ?? 0), epoch: hold.epoch})
|
|
387
|
+
this.fencedHolds.set(name, {currentEpoch: Number(body.epoch ?? 0), epoch: hold.epoch, reason: "fenced"})
|
|
386
388
|
|
|
387
389
|
return
|
|
388
390
|
}
|
|
@@ -516,7 +518,7 @@ export default class AdvisoryLockLockServerBackend {
|
|
|
516
518
|
if (status === "lost" || status === "not-found") {
|
|
517
519
|
this.holds.delete(name)
|
|
518
520
|
if (hold.timer) clearTimeout(hold.timer)
|
|
519
|
-
this.fencedHolds.set(name, {currentEpoch: Number(response.currentEpoch ?? 0), epoch})
|
|
521
|
+
this.fencedHolds.set(name, {currentEpoch: Number(response.currentEpoch ?? 0), epoch, reason: "fenced"})
|
|
520
522
|
}
|
|
521
523
|
} catch (error) {
|
|
522
524
|
if (!(error instanceof AdvisoryLockServiceUnavailableError)) throw error
|
|
@@ -535,7 +537,7 @@ export default class AdvisoryLockLockServerBackend {
|
|
|
535
537
|
this.markUnavailable(error)
|
|
536
538
|
this.holds.delete(name)
|
|
537
539
|
if (hold.timer) clearTimeout(hold.timer)
|
|
538
|
-
this.fencedHolds.set(name, {currentEpoch: null, epoch})
|
|
540
|
+
this.fencedHolds.set(name, {currentEpoch: null, epoch, reason: this.connection ? "expired" : "transport"})
|
|
539
541
|
this.reportError(error)
|
|
540
542
|
} finally {
|
|
541
543
|
hold.renewing = false
|
|
@@ -553,6 +555,16 @@ export default class AdvisoryLockLockServerBackend {
|
|
|
553
555
|
this.status.unavailableUntil = Date.now() + this.reprobeCooldownMs
|
|
554
556
|
}
|
|
555
557
|
|
|
558
|
+
/**
|
|
559
|
+
* How long this backend still refuses to re-probe, in milliseconds. `0` means
|
|
560
|
+
* a probe is allowed on the next call, so there is no bounded reconnect event
|
|
561
|
+
* left to wait for.
|
|
562
|
+
* @returns {number} - Remaining re-probe cooldown, or 0 when one is not pending.
|
|
563
|
+
*/
|
|
564
|
+
reprobeCooldownRemainingMs() {
|
|
565
|
+
return Math.max(0, this.status.unavailableUntil - Date.now())
|
|
566
|
+
}
|
|
567
|
+
|
|
556
568
|
/**
|
|
557
569
|
* Drops transport state and marks every outstanding hold lost.
|
|
558
570
|
* @returns {void}
|
|
@@ -560,7 +572,7 @@ export default class AdvisoryLockLockServerBackend {
|
|
|
560
572
|
teardown() {
|
|
561
573
|
for (const [name, hold] of this.holds) {
|
|
562
574
|
if (hold.timer) clearTimeout(hold.timer)
|
|
563
|
-
this.fencedHolds.set(name, {currentEpoch: null, epoch: hold.epoch})
|
|
575
|
+
this.fencedHolds.set(name, {currentEpoch: null, epoch: hold.epoch, reason: "transport"})
|
|
564
576
|
}
|
|
565
577
|
|
|
566
578
|
this.holds.clear()
|
|
@@ -367,6 +367,9 @@ export default class AdvisoryLockRunner {
|
|
|
367
367
|
* A name whose minimum tier is `lock-server` is never served by a weaker
|
|
368
368
|
* tier: an absent or unreachable server raises
|
|
369
369
|
* `AdvisoryLockBackendUnavailableError` instead of reporting a fallback.
|
|
370
|
+
* A mandated name still waits for the backend's own bounded re-probe window
|
|
371
|
+
* whenever this acquisition's budget covers it, so a lock-server restart is
|
|
372
|
+
* not mistaken for an unavailable backend.
|
|
370
373
|
*
|
|
371
374
|
* `null` means "this tier did not serve the acquisition — continue with the
|
|
372
375
|
* next eligible tier". A returned value means the lock-server tier answered,
|
|
@@ -400,29 +403,55 @@ export default class AdvisoryLockRunner {
|
|
|
400
403
|
return null
|
|
401
404
|
}
|
|
402
405
|
|
|
406
|
+
// A mandated tier may only be abandoned once the backend has no bounded
|
|
407
|
+
// reconnect event left inside this acquisition's own budget. The backend
|
|
408
|
+
// publishes that window (`reprobeCooldownRemainingMs`), so the wait is
|
|
409
|
+
// bounded by the backend's own re-probe rather than by a retry policy.
|
|
410
|
+
const acquireBudgetMs = this.acquireTimeoutMs(args)
|
|
411
|
+
const transportWaitBudgetMs = mode === "wait" && acquireBudgetMs !== null ? acquireBudgetMs : null
|
|
403
412
|
/** @type {boolean} */
|
|
404
413
|
let acquired
|
|
414
|
+
/** @type {number} */
|
|
415
|
+
let waitedForTransportMs = 0
|
|
405
416
|
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
acquired = await backend.tryAcquire(name)
|
|
411
|
-
}
|
|
412
|
-
} catch (error) {
|
|
413
|
-
if (!(error instanceof AdvisoryLockServiceUnavailableError)) throw error
|
|
417
|
+
for (;;) {
|
|
418
|
+
try {
|
|
419
|
+
if (mode === "wait") {
|
|
420
|
+
const remainingBudgetMs = acquireBudgetMs === null ? null : Math.max(0, acquireBudgetMs - waitedForTransportMs)
|
|
414
421
|
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
reasons: [`lock-server unavailable: ${error.message}`]
|
|
420
|
-
})
|
|
421
|
-
}
|
|
422
|
+
acquired = await backend.acquire(name, {timeoutMs: remainingBudgetMs})
|
|
423
|
+
} else {
|
|
424
|
+
acquired = await backend.tryAcquire(name)
|
|
425
|
+
}
|
|
422
426
|
|
|
423
|
-
|
|
427
|
+
break
|
|
428
|
+
} catch (error) {
|
|
429
|
+
if (!(error instanceof AdvisoryLockServiceUnavailableError)) throw error
|
|
424
430
|
|
|
425
|
-
|
|
431
|
+
if (policy.minTier !== "lock-server") {
|
|
432
|
+
this.reportAdvisoryLockFallback({reason: error.message, servedBy: "driver-native", lockName: name})
|
|
433
|
+
|
|
434
|
+
return null
|
|
435
|
+
}
|
|
436
|
+
|
|
437
|
+
const cooldownMs = typeof backend.reprobeCooldownRemainingMs === "function" ? backend.reprobeCooldownRemainingMs() : 0
|
|
438
|
+
// `or-fail`/`defer` declare no wait budget, so their bound is the
|
|
439
|
+
// backend's own single re-probe cycle: one reconnect attempt, never a queue.
|
|
440
|
+
const waitBudgetMs = transportWaitBudgetMs ?? cooldownMs
|
|
441
|
+
|
|
442
|
+
if (cooldownMs <= 0 || waitedForTransportMs + cooldownMs > waitBudgetMs) {
|
|
443
|
+
throw advisoryLockBackendUnavailable({
|
|
444
|
+
lockName: name,
|
|
445
|
+
minTier: policy.minTier,
|
|
446
|
+
reasons: [`lock-server unavailable: ${error.message}`]
|
|
447
|
+
})
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
await new Promise((resolve) => {
|
|
451
|
+
setTimeout(resolve, cooldownMs)
|
|
452
|
+
})
|
|
453
|
+
waitedForTransportMs += cooldownMs
|
|
454
|
+
}
|
|
426
455
|
}
|
|
427
456
|
|
|
428
457
|
if (!acquired) {
|
|
@@ -485,12 +514,13 @@ export default class AdvisoryLockRunner {
|
|
|
485
514
|
|
|
486
515
|
if (loss) {
|
|
487
516
|
throw new AdvisoryLockLostError(
|
|
488
|
-
`Lost the lease for advisory lock ${JSON.stringify(name)} while the critical section was running`,
|
|
517
|
+
`Lost the lease for advisory lock ${JSON.stringify(name)} while the critical section was running (reason=${loss.reason}; epoch=${String(loss.epoch)}; currentEpoch=${String(loss.currentEpoch)})`,
|
|
489
518
|
{
|
|
490
519
|
cause: callbackError instanceof Error ? callbackError : undefined,
|
|
491
520
|
currentEpoch: loss.currentEpoch,
|
|
492
521
|
epoch: lockBackend.epoch ?? null,
|
|
493
|
-
lockName: name
|
|
522
|
+
lockName: name,
|
|
523
|
+
reason: loss.reason
|
|
494
524
|
}
|
|
495
525
|
)
|
|
496
526
|
}
|
|
@@ -9,6 +9,19 @@ export default class VelociousDatabaseDriversBaseTable {
|
|
|
9
9
|
* @type {import("./base.js").default | undefined} */
|
|
10
10
|
driver = undefined
|
|
11
11
|
|
|
12
|
+
/**
|
|
13
|
+
* Builds this table's column objects from raw introspection rows that were already read,
|
|
14
|
+
* so a batched catalog read and a per-table read produce the same columns.
|
|
15
|
+
* @abstract
|
|
16
|
+
* @param {import("./base.js").QueryResultType} columnRows - Raw column rows for this table.
|
|
17
|
+
* @returns {Array<import("./base-column.js").default>} - Columns built from the rows.
|
|
18
|
+
*/
|
|
19
|
+
columnsFromRows(columnRows) {
|
|
20
|
+
void columnRows
|
|
21
|
+
|
|
22
|
+
throw new Error("columnsFromRows not implemented")
|
|
23
|
+
}
|
|
24
|
+
|
|
12
25
|
/**
|
|
13
26
|
* Runs get column by name.
|
|
14
27
|
* @param {string} columnName - Column name.
|
|
@@ -63,6 +63,23 @@ export default class VelociousDatabaseDriversMssqlTable extends BaseTable {
|
|
|
63
63
|
this.driver = driver
|
|
64
64
|
}
|
|
65
65
|
|
|
66
|
+
/**
|
|
67
|
+
* Builds this table's column objects from raw `INFORMATION_SCHEMA.COLUMNS`-shaped rows.
|
|
68
|
+
* @param {import("../base.js").QueryResultType} columnRows - Raw column rows for this table.
|
|
69
|
+
* @returns {Array<import("../base-column.js").default>} - Columns built from the rows.
|
|
70
|
+
*/
|
|
71
|
+
columnsFromRows(columnRows) {
|
|
72
|
+
const columns = []
|
|
73
|
+
|
|
74
|
+
for (const data of columnRows) {
|
|
75
|
+
const column = new Column(this, data)
|
|
76
|
+
|
|
77
|
+
columns.push(column)
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
return columns
|
|
81
|
+
}
|
|
82
|
+
|
|
66
83
|
async getColumns() {
|
|
67
84
|
const columnRows = await this.getDriver()._cachedTableSchemaMetadata(this.getName(), "columns", async () => {
|
|
68
85
|
const result = await this.driver.query(`
|
|
@@ -88,15 +105,8 @@ export default class VelociousDatabaseDriversMssqlTable extends BaseTable {
|
|
|
88
105
|
`)
|
|
89
106
|
return result
|
|
90
107
|
})
|
|
91
|
-
const columns = []
|
|
92
|
-
|
|
93
|
-
for (const data of columnRows) {
|
|
94
|
-
const column = new Column(this, data)
|
|
95
108
|
|
|
96
|
-
|
|
97
|
-
}
|
|
98
|
-
|
|
99
|
-
return columns
|
|
109
|
+
return this.columnsFromRows(columnRows)
|
|
100
110
|
}
|
|
101
111
|
|
|
102
112
|
async getForeignKeys() {
|
|
@@ -17,10 +17,12 @@ export default class VelociousDatabaseDriversMysqlTable extends BaseTable {
|
|
|
17
17
|
this.driver = driver
|
|
18
18
|
}
|
|
19
19
|
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
20
|
+
/**
|
|
21
|
+
* Builds this table's column objects from raw `SHOW FULL COLUMNS`-shaped rows.
|
|
22
|
+
* @param {import("../base.js").QueryResultType} columnRows - Raw column rows for this table.
|
|
23
|
+
* @returns {Array<import("../base-column.js").default>} - Columns built from the rows.
|
|
24
|
+
*/
|
|
25
|
+
columnsFromRows(columnRows) {
|
|
24
26
|
const columns = []
|
|
25
27
|
|
|
26
28
|
for (const data of columnRows) {
|
|
@@ -32,6 +34,14 @@ export default class VelociousDatabaseDriversMysqlTable extends BaseTable {
|
|
|
32
34
|
return columns
|
|
33
35
|
}
|
|
34
36
|
|
|
37
|
+
async getColumns() {
|
|
38
|
+
const columnRows = await this.getDriver()._cachedTableSchemaMetadata(this.getName(), "columns", async () => {
|
|
39
|
+
return await this.driver.query(`SHOW FULL COLUMNS FROM \`${this.getName()}\``)
|
|
40
|
+
})
|
|
41
|
+
|
|
42
|
+
return this.columnsFromRows(columnRows)
|
|
43
|
+
}
|
|
44
|
+
|
|
35
45
|
async getForeignKeys() {
|
|
36
46
|
const foreignKeyRows = await this.getDriver()._cachedTableSchemaMetadata(this.getName(), "foreignKeys", async () => {
|
|
37
47
|
const sql = `
|
|
@@ -62,6 +62,23 @@ export default class VelociousDatabaseDriversPgsqlTable extends BaseTable {
|
|
|
62
62
|
this.driver = driver
|
|
63
63
|
}
|
|
64
64
|
|
|
65
|
+
/**
|
|
66
|
+
* Builds this table's column objects from raw `information_schema.columns`-shaped rows.
|
|
67
|
+
* @param {import("../base.js").QueryResultType} columnRows - Raw column rows for this table.
|
|
68
|
+
* @returns {Array<import("../base-column.js").default>} - Columns built from the rows.
|
|
69
|
+
*/
|
|
70
|
+
columnsFromRows(columnRows) {
|
|
71
|
+
const columns = []
|
|
72
|
+
|
|
73
|
+
for (const data of columnRows) {
|
|
74
|
+
const column = new Column(this, data)
|
|
75
|
+
|
|
76
|
+
columns.push(column)
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
return columns
|
|
80
|
+
}
|
|
81
|
+
|
|
65
82
|
async getColumns() {
|
|
66
83
|
const columnRows = await this.getDriver()._cachedTableSchemaMetadata(this.getName(), "columns", async () => {
|
|
67
84
|
const result = await this.driver.query(`
|
|
@@ -91,15 +108,8 @@ export default class VelociousDatabaseDriversPgsqlTable extends BaseTable {
|
|
|
91
108
|
`)
|
|
92
109
|
return result
|
|
93
110
|
})
|
|
94
|
-
const columns = []
|
|
95
|
-
|
|
96
|
-
for (const data of columnRows) {
|
|
97
|
-
const column = new Column(this, data)
|
|
98
111
|
|
|
99
|
-
|
|
100
|
-
}
|
|
101
|
-
|
|
102
|
-
return columns
|
|
112
|
+
return this.columnsFromRows(columnRows)
|
|
103
113
|
}
|
|
104
114
|
|
|
105
115
|
async getForeignKeys() {
|
|
@@ -19,13 +19,11 @@ export default class VelociousDatabaseDriversSqliteTable extends BaseTable {
|
|
|
19
19
|
}
|
|
20
20
|
|
|
21
21
|
/**
|
|
22
|
-
*
|
|
23
|
-
* @
|
|
22
|
+
* Builds this table's column objects from raw `PRAGMA table_info` rows.
|
|
23
|
+
* @param {import("../base.js").QueryResultType} columnRows - Raw column rows for this table.
|
|
24
|
+
* @returns {Array<import("../base-column.js").default>} - Columns built from the rows.
|
|
24
25
|
*/
|
|
25
|
-
|
|
26
|
-
const columnRows = await this.getDriver()._cachedTableSchemaMetadata(this.getName(), "columns", async () => {
|
|
27
|
-
return await this.driver.query(`PRAGMA table_info('${this.getName()}')`)
|
|
28
|
-
})
|
|
26
|
+
columnsFromRows(columnRows) {
|
|
29
27
|
const columns = []
|
|
30
28
|
|
|
31
29
|
for (const columnData of columnRows) {
|
|
@@ -37,6 +35,18 @@ export default class VelociousDatabaseDriversSqliteTable extends BaseTable {
|
|
|
37
35
|
return columns
|
|
38
36
|
}
|
|
39
37
|
|
|
38
|
+
/**
|
|
39
|
+
* Runs get columns.
|
|
40
|
+
* @returns {Promise<Array<import("../base-column.js").default>>} - Resolves with the columns.
|
|
41
|
+
*/
|
|
42
|
+
async getColumns() {
|
|
43
|
+
const columnRows = await this.getDriver()._cachedTableSchemaMetadata(this.getName(), "columns", async () => {
|
|
44
|
+
return await this.driver.query(`PRAGMA table_info('${this.getName()}')`)
|
|
45
|
+
})
|
|
46
|
+
|
|
47
|
+
return this.columnsFromRows(columnRows)
|
|
48
|
+
}
|
|
49
|
+
|
|
40
50
|
async getForeignKeys() {
|
|
41
51
|
const foreignKeysData = await this.getDriver()._cachedTableSchemaMetadata(this.getName(), "foreignKeys", async () => {
|
|
42
52
|
return await this.driver.query(`SELECT * FROM pragma_foreign_key_list(${this.driver.quote(this.getName())})`)
|
|
@@ -234,6 +234,12 @@ export default class BackgroundJobsWorker {
|
|
|
234
234
|
workerShutdownTimeoutMsOverride: number | null | undefined;
|
|
235
235
|
/** @type {number | null} */
|
|
236
236
|
workerShutdownTimeoutMs: number | null;
|
|
237
|
+
/**
|
|
238
|
+
* Bound on one owned shutdown close, in ms. Constructor override wins so a
|
|
239
|
+
* spec can exercise the bound without waiting for the production one.
|
|
240
|
+
* @type {number}
|
|
241
|
+
*/
|
|
242
|
+
shutdownCloseTimeoutMs: number;
|
|
237
243
|
shouldStop: boolean;
|
|
238
244
|
isRetiring: boolean;
|
|
239
245
|
/** @type {Promise<void> | undefined} */
|
|
@@ -348,6 +354,7 @@ export default class BackgroundJobsWorker {
|
|
|
348
354
|
* @param {number} [args.reconnectDelayMs] - Delay before reconnecting an established worker connection (default: 1000).
|
|
349
355
|
* @param {number} [args.jobTimeoutMs] - Override the wall-clock timeout for forked and pooled jobs from `configuration.getBackgroundJobsConfig()`. `0` disables it.
|
|
350
356
|
* @param {number | null} [args.workerShutdownTimeoutMs] - Override the wall-clock cap on one whole shutdown drain from `configuration.getBackgroundJobsConfig().workerShutdownTimeoutMs`. `null` drains without a bound.
|
|
357
|
+
* @param {number} [args.shutdownCloseTimeoutMs] - Override the bound on one owned shutdown close (application teardown, Beacon disconnect, database close) in ms (default 30000).
|
|
351
358
|
* @param {number} [args.pooledRunnerSilenceTimeoutMs] - Override the per-child IPC-silence bound for pooled children with in-flight jobs from `configuration.getBackgroundJobsConfig()` (default 120000ms). `0` disables the watchdog.
|
|
352
359
|
* @param {boolean} [args.closeDatabaseConnectionsOnStop] - Whether stop owns closing the configuration's database pools (default true).
|
|
353
360
|
* @param {() => void | Promise<void>} [args.onStopped] - Lifecycle hook invoked after the worker finishes stopping.
|
|
@@ -356,7 +363,7 @@ export default class BackgroundJobsWorker {
|
|
|
356
363
|
* @param {(observation: import("./types.js").PooledChildMemoryObservation) => void | Promise<void>} [args.onPooledRunnerMemoryObservation] - Explicit pooled-child memory observation hook. Every validated observation (periodic while a child has in-flight jobs, plus on demand) is forwarded here, in addition to the worker's compact stderr log line, so an application can route memory diagnostics (e.g. to a bug reporter) without parsing logs.
|
|
357
364
|
* @param {(payload: import("./types.js").BackgroundJobPayload) => void} [args.onJobAccepted] - Explicit job-acceptance observation hook. Fires the moment the worker accepts a dispatched job payload, before execution mode routing, so an observer can wait for the pickup event instead of polling in-flight sets.
|
|
358
365
|
*/
|
|
359
|
-
constructor({ configuration, host, port, generationId, workerInstanceId, maxConcurrentForkedJobs, maxConcurrentInlineJobs, pooledRunnerCount, pooledRunnerConcurrency, pooledRunnerMaxJobs, pooledRunnerMaxRssBytes, pooledRunnerMaxLifetimeMs, runnerMaxOldSpaceSizeMb, runnerMallocArenaMax, forkedChildSigkillGraceMs, heartbeatIntervalMs, generationHandshakeTimeoutMs, reconnectDelayMs, jobTimeoutMs, workerShutdownTimeoutMs, pooledRunnerSilenceTimeoutMs, closeDatabaseConnectionsOnStop, onStopped, onGenerationAccepted, onRetireMessage, onPooledRunnerMemoryObservation, onJobAccepted }?: {
|
|
366
|
+
constructor({ configuration, host, port, generationId, workerInstanceId, maxConcurrentForkedJobs, maxConcurrentInlineJobs, pooledRunnerCount, pooledRunnerConcurrency, pooledRunnerMaxJobs, pooledRunnerMaxRssBytes, pooledRunnerMaxLifetimeMs, runnerMaxOldSpaceSizeMb, runnerMallocArenaMax, forkedChildSigkillGraceMs, heartbeatIntervalMs, generationHandshakeTimeoutMs, reconnectDelayMs, jobTimeoutMs, workerShutdownTimeoutMs, shutdownCloseTimeoutMs, pooledRunnerSilenceTimeoutMs, closeDatabaseConnectionsOnStop, onStopped, onGenerationAccepted, onRetireMessage, onPooledRunnerMemoryObservation, onJobAccepted }?: {
|
|
360
367
|
configuration?: import("../configuration.js").default;
|
|
361
368
|
host?: string;
|
|
362
369
|
port?: number;
|
|
@@ -377,6 +384,7 @@ export default class BackgroundJobsWorker {
|
|
|
377
384
|
reconnectDelayMs?: number;
|
|
378
385
|
jobTimeoutMs?: number;
|
|
379
386
|
workerShutdownTimeoutMs?: number | null;
|
|
387
|
+
shutdownCloseTimeoutMs?: number;
|
|
380
388
|
pooledRunnerSilenceTimeoutMs?: number;
|
|
381
389
|
closeDatabaseConnectionsOnStop?: boolean;
|
|
382
390
|
onStopped?: () => void | Promise<void>;
|
|
@@ -451,10 +459,25 @@ export default class BackgroundJobsWorker {
|
|
|
451
459
|
*/
|
|
452
460
|
_remainingDrainMs(deadlineMs: number | undefined): number | undefined;
|
|
453
461
|
/**
|
|
454
|
-
* Closes application resources before framework resources when this worker owns them
|
|
462
|
+
* Closes application resources before framework resources when this worker owns them,
|
|
463
|
+
* with every close bounded so the shutdown always reaches its end.
|
|
455
464
|
* @returns {Promise<void>} - Resolves after every owned close succeeds.
|
|
456
465
|
*/
|
|
457
466
|
_closeConfiguration(): Promise<void>;
|
|
467
|
+
/**
|
|
468
|
+
* Runs one owned shutdown close inside `shutdownCloseTimeoutMs` and names the step
|
|
469
|
+
* when it runs out.
|
|
470
|
+
*
|
|
471
|
+
* The close is not cancelled — it is left running and simply no longer waited on.
|
|
472
|
+
* That is the whole point: the next close (and then the CLI's exit) can happen at
|
|
473
|
+
* all. A close that never settling is a defect in whatever owns it, and it is
|
|
474
|
+
* reported here as the named step that ran out rather than as a worker that quietly
|
|
475
|
+
* never exits.
|
|
476
|
+
* @param {string} label - The step, named in the failure this reports.
|
|
477
|
+
* @param {(() => Promise<void>)} close - The close to run.
|
|
478
|
+
* @returns {Promise<void>} - Resolves when the close settles; rejects with the close's own error, or a named timeout.
|
|
479
|
+
*/
|
|
480
|
+
_boundedShutdownClose(label: string, close: (() => Promise<void>)): Promise<void>;
|
|
458
481
|
/**
|
|
459
482
|
* Waits for a set of in-flight job promises to settle, optionally bounded by
|
|
460
483
|
* `timeoutMs`. When a bounded window runs out with entries still held, the drain
|