velocious 1.0.556 → 1.0.558
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -2
- package/build/background-jobs/forked-runner-child.js +20 -2
- package/build/background-jobs/main.js +35 -12
- package/build/background-jobs/pooled-runner-child.js +20 -2
- package/build/background-jobs/runner-graceful-shutdown.js +73 -0
- package/build/background-jobs/worker.js +53 -28
- package/build/configuration.js +77 -12
- package/build/database/advisory-lock-runner.js +12 -3
- package/build/database/record/index.js +12 -0
- package/build/frontend-models/base.js +192 -41
- package/build/http-client/websocket-client.js +48 -0
- package/build/src/background-jobs/forked-runner-child.js +20 -3
- package/build/src/background-jobs/main.d.ts +14 -1
- package/build/src/background-jobs/main.d.ts.map +1 -1
- package/build/src/background-jobs/main.js +37 -14
- package/build/src/background-jobs/pooled-runner-child.js +19 -3
- package/build/src/background-jobs/runner-graceful-shutdown.d.ts +49 -0
- package/build/src/background-jobs/runner-graceful-shutdown.d.ts.map +1 -0
- package/build/src/background-jobs/runner-graceful-shutdown.js +70 -0
- package/build/src/background-jobs/worker.d.ts +18 -1
- package/build/src/background-jobs/worker.d.ts.map +1 -1
- package/build/src/background-jobs/worker.js +55 -31
- package/build/src/configuration.d.ts +29 -0
- package/build/src/configuration.d.ts.map +1 -1
- package/build/src/configuration.js +71 -11
- package/build/src/database/advisory-lock-runner.d.ts.map +1 -1
- package/build/src/database/advisory-lock-runner.js +12 -4
- package/build/src/database/record/index.d.ts.map +1 -1
- package/build/src/database/record/index.js +9 -1
- package/build/src/frontend-models/base.d.ts +13 -4
- package/build/src/frontend-models/base.d.ts.map +1 -1
- package/build/src/frontend-models/base.js +173 -44
- package/build/src/http-client/websocket-client.d.ts +3 -0
- package/build/src/http-client/websocket-client.d.ts.map +1 -1
- package/build/src/http-client/websocket-client.js +42 -1
- package/build/src/tenants/tenant.d.ts +20 -9
- package/build/src/tenants/tenant.d.ts.map +1 -1
- package/build/src/tenants/tenant.js +47 -12
- package/build/src/testing/factory/factory-registry.d.ts +22 -3
- package/build/src/testing/factory/factory-registry.d.ts.map +1 -1
- package/build/src/testing/factory/factory-registry.js +32 -9
- package/build/src/testing/factory/factory-runner.d.ts +16 -4
- package/build/src/testing/factory/factory-runner.d.ts.map +1 -1
- package/build/src/testing/factory/factory-runner.js +33 -10
- package/build/src/utils/shutdown-lifecycle.d.ts +12 -0
- package/build/src/utils/shutdown-lifecycle.d.ts.map +1 -0
- package/build/src/utils/shutdown-lifecycle.js +36 -0
- package/build/tenants/tenant.js +51 -11
- package/build/testing/factory/factory-registry.js +33 -8
- package/build/testing/factory/factory-runner.js +38 -12
- package/build/tsconfig.tsbuildinfo +1 -1
- package/build/utils/shutdown-lifecycle.js +41 -0
- package/package.json +5 -4
- package/src/background-jobs/forked-runner-child.js +20 -2
- package/src/background-jobs/main.js +35 -12
- package/src/background-jobs/pooled-runner-child.js +20 -2
- package/src/background-jobs/runner-graceful-shutdown.js +73 -0
- package/src/background-jobs/worker.js +53 -28
- package/src/configuration.js +77 -12
- package/src/database/advisory-lock-runner.js +12 -3
- package/src/database/record/index.js +12 -0
- package/src/frontend-models/base.js +192 -41
- package/src/http-client/websocket-client.js +48 -0
- package/src/tenants/tenant.js +51 -11
- package/src/testing/factory/factory-registry.js +33 -8
- package/src/testing/factory/factory-runner.js +38 -12
- package/src/utils/shutdown-lifecycle.js +41 -0
package/README.md
CHANGED
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
* Migrations for schema changes and UTC datetime storage (see [docs/database-migrations.md](docs/database-migrations.md))
|
|
11
11
|
* External packages (engines) that contribute data models, frontend-model resources and migrations to a consuming app (see [docs/packages.md](docs/packages.md))
|
|
12
12
|
* Controllers and views for HTTP endpoints
|
|
13
|
-
* Frontend-model transport for creating, updating, querying, and subscribing to query-filtered lifecycle events over HTTP/WebSocket, with structured per-attribute validation error responses (see [docs/frontend-models.md](docs/frontend-models.md))
|
|
13
|
+
* Frontend-model transport for creating, updating, querying, and subscribing to query-filtered lifecycle events over HTTP/WebSocket, with structured per-attribute validation error responses and one-budget WebSocket startup controls (see [docs/frontend-models.md](docs/frontend-models.md) and [docs/websocket-channels.md](docs/websocket-channels.md))
|
|
14
14
|
* Client-side offline sync mutation logs and frontend-model optimistic queueing primitives (see [docs/offline-sync.md](docs/offline-sync.md))
|
|
15
15
|
* Declarative client sync scopes with per-scope cursors, automatic mutation tracking, realtime delivery, and `sync`/`pull` progress reporting for "X of Y" import screens (see [docs/sync-client.md](docs/sync-client.md))
|
|
16
16
|
* Reactive `useLiveQuery(Model.where(...))` queries that stay current from committed local model changes across local writes, pulls, and realtime (see [docs/live-queries.md](docs/live-queries.md))
|
|
@@ -75,6 +75,11 @@ Baselines are generated against a fresh checkout (no generated dummy `configurat
|
|
|
75
75
|
|
|
76
76
|
# Testing
|
|
77
77
|
|
|
78
|
+
The dummy database configurations require `MSSQL_SA_PASSWORD` whenever they include
|
|
79
|
+
the shared MSSQL test database. Set it in the local process environment rather than
|
|
80
|
+
writing the password into `spec/dummy/src/config/configuration*.js`. TensorBuzz CI
|
|
81
|
+
provides one shared test value to both the build and MSSQL service environments.
|
|
82
|
+
|
|
78
83
|
Tag tests to filter runs.
|
|
79
84
|
|
|
80
85
|
```js
|
|
@@ -2139,6 +2144,15 @@ Inline jobs share the worker process and run concurrently up to `maxConcurrentIn
|
|
|
2139
2144
|
new BackgroundJobsWorker({configuration, maxConcurrentInlineJobs: 8})
|
|
2140
2145
|
```
|
|
2141
2146
|
|
|
2147
|
+
Standalone background-jobs main and worker processes close their configuration's
|
|
2148
|
+
database pools during `stop()`. Embedded/test callers that share externally owned
|
|
2149
|
+
pools can pass `closeDatabaseConnectionsOnStop: false` to `BackgroundJobsMain` or
|
|
2150
|
+
`BackgroundJobsWorker`; sockets and Beacon still shut down normally. An async
|
|
2151
|
+
`onStopped` constructor hook can coordinate externally owned cleanup after that
|
|
2152
|
+
shutdown without wrapping the service's `stop()` method. Repeated `stop()` calls
|
|
2153
|
+
share one lifecycle and invoke the hook once; dual shutdown/hook failures reject
|
|
2154
|
+
with an `AggregateError` ordered with the shutdown failure first.
|
|
2155
|
+
|
|
2142
2156
|
## Scheduled jobs
|
|
2143
2157
|
|
|
2144
2158
|
Velocious can enqueue recurring jobs from the `background-jobs-main` process. Configure them with `scheduledBackgroundJobs` using Sidekiq Scheduler-style `every` arrays:
|
|
@@ -2383,7 +2397,7 @@ Tenant lifecycle commands print start and final counts, report each completed te
|
|
|
2383
2397
|
|
|
2384
2398
|
`afterMigrateTenant` hooks run inside the active default and tenant database connection scope for the tenant being migrated.
|
|
2385
2399
|
|
|
2386
|
-
At runtime, the apartment-style `Tenant` façade (`velocious/build/src/tenants/tenant.js`) is the single entry point: `Tenant.with(tenant, callback)` / `Tenant.current()` to switch into and read a tenant context, `Tenant.each({identifier, callback, parallel?, filter?})` to run a callback within every provider-listed tenant, and `Tenant.drop({identifier, tenant})` (plus the `db:tenants:drop` CLI command) to drop a tenant's database through the provider's `dropDatabase` hook. `Tenant.with` and `Tenant.each` run their callbacks inside `ensureConnections`, so switching into a tenant establishes its database connections (global + tenant) and passes
|
|
2400
|
+
At runtime, the apartment-style `Tenant` façade (`velocious/build/src/tenants/tenant.js`) is the single entry point: `Tenant.with(tenant, callback)` / `Tenant.current()` to switch into and read a tenant context, `Tenant.each({identifier, callback, parallel?, filter?})` to run a callback within every provider-listed tenant, and `Tenant.drop({identifier, tenant})` (plus the `db:tenants:drop` CLI command) to drop a tenant's database through the provider's `dropDatabase` hook. `Tenant.with` and `Tenant.each` run their callbacks inside `ensureConnections`, so switching into a tenant establishes its database connections (global + tenant), initializes registered tenant-switched models whose tables exist, and passes the connections to the callback — the tenant is immediately queryable without caller-owned connection or model initialization (already-open connections and in-progress model initialization promises are reused). Models for absent optional tables stay deferred. `Tenant.with` is generic over its callback's return type, so a value returned from inside the tenant context comes back to the caller with its type preserved (no cast needed). `Tenant.aggregateAcross({identifier, aggregates, keyColumns, subquery, tenants?, filter?})` runs one aggregate over the same table across many tenant databases and returns the merged result — grouping tenants by server and using a single cross-database `UNION ALL` where the driver supports two-part `` `database`.`table` `` references (MySQL/MariaDB) or one query per tenant otherwise (PostgreSQL/SQLite/MSSQL).
|
|
2387
2401
|
|
|
2388
2402
|
`SchemaCloner` adds a missing auto-increment column and its separate source unique index in one schema alteration, including on MySQL/MariaDB where an auto-increment column must be keyed when it is created.
|
|
2389
2403
|
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
// @ts-check
|
|
2
2
|
|
|
3
3
|
import runJobPayload from "./job-runner.js"
|
|
4
|
+
import { closeRunnerConnections, currentConfigurationOrNull } from "./runner-graceful-shutdown.js"
|
|
4
5
|
import setRunnerProcessTitle from "./runner-process-title.js"
|
|
5
6
|
|
|
6
7
|
// Name the process so `ps`/`top`/`htop` can identify forked job runners at a
|
|
@@ -10,6 +11,23 @@ import setRunnerProcessTitle from "./runner-process-title.js"
|
|
|
10
11
|
setRunnerProcessTitle()
|
|
11
12
|
|
|
12
13
|
let finishing = false
|
|
14
|
+
let shuttingDown = false
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Closes the runner's connections — releasing any advisory lock a killed-mid-job
|
|
18
|
+
* job still holds — before exiting on shutdown, instead of leaving a half-open
|
|
19
|
+
* session that keeps the lock until the DB server's `wait_timeout`. Normal
|
|
20
|
+
* completion (`finish`) already released its locks via the job's own lock scope.
|
|
21
|
+
* @param {number} exitCode - Process exit code.
|
|
22
|
+
* @returns {Promise<void>}
|
|
23
|
+
*/
|
|
24
|
+
async function shutdownRunner(exitCode) {
|
|
25
|
+
if (shuttingDown || finishing) return
|
|
26
|
+
shuttingDown = true
|
|
27
|
+
|
|
28
|
+
await closeRunnerConnections(currentConfigurationOrNull())
|
|
29
|
+
process.exit(exitCode)
|
|
30
|
+
}
|
|
13
31
|
|
|
14
32
|
/**
|
|
15
33
|
* Runs is job message.
|
|
@@ -82,11 +100,11 @@ async function handleJobMessage(message) {
|
|
|
82
100
|
}
|
|
83
101
|
|
|
84
102
|
for (const signal of ["SIGTERM", "SIGINT"]) {
|
|
85
|
-
process.once(signal, () =>
|
|
103
|
+
process.once(signal, () => void shutdownRunner(1))
|
|
86
104
|
}
|
|
87
105
|
|
|
88
106
|
process.once("disconnect", () => {
|
|
89
|
-
if (!finishing)
|
|
107
|
+
if (!finishing) void shutdownRunner(0)
|
|
90
108
|
})
|
|
91
109
|
|
|
92
110
|
process.once("message", (message) => {
|
|
@@ -7,6 +7,7 @@ import BackgroundJobsStore from "./store.js"
|
|
|
7
7
|
import Logger from "../logger.js"
|
|
8
8
|
import PruneTerminalBackgroundJobsJob from "../jobs/prune-terminal-background-jobs.js"
|
|
9
9
|
import VelociousError from "../velocious-error.js"
|
|
10
|
+
import shutdownLifecycle from "../utils/shutdown-lifecycle.js"
|
|
10
11
|
|
|
11
12
|
/**
|
|
12
13
|
* Channel used by `background-jobs-main` to coordinate dispatch wake-ups
|
|
@@ -60,9 +61,13 @@ export default class BackgroundJobsMain {
|
|
|
60
61
|
* @param {number} [args.port] - Port.
|
|
61
62
|
* @param {number} [args.workerStaleTimeoutMs] - Override how long a silent worker may go before being dropped (default 60000ms).
|
|
62
63
|
* @param {number} [args.workerLivenessSweepMs] - Override how often stale workers are swept for (default 15000ms).
|
|
64
|
+
* @param {boolean} [args.closeDatabaseConnectionsOnStop] - Whether stop owns closing the configuration's database pools (default true).
|
|
65
|
+
* @param {() => void | Promise<void>} [args.onStopped] - Lifecycle hook invoked after the main process finishes stopping.
|
|
63
66
|
*/
|
|
64
|
-
constructor({configuration, host, port, workerStaleTimeoutMs, workerLivenessSweepMs}) {
|
|
67
|
+
constructor({configuration, host, port, workerStaleTimeoutMs, workerLivenessSweepMs, closeDatabaseConnectionsOnStop = true, onStopped}) {
|
|
65
68
|
this.configuration = configuration
|
|
69
|
+
this.closeDatabaseConnectionsOnStop = closeDatabaseConnectionsOnStop
|
|
70
|
+
this.onStopped = onStopped
|
|
66
71
|
const config = configuration.getBackgroundJobsConfig()
|
|
67
72
|
this.host = host || config.host
|
|
68
73
|
this.port = typeof port === "number" ? port : config.port
|
|
@@ -123,6 +128,8 @@ export default class BackgroundJobsMain {
|
|
|
123
128
|
this._draining = false
|
|
124
129
|
this._redrainQueued = false
|
|
125
130
|
this._stopped = false
|
|
131
|
+
/** @type {Promise<void> | undefined} */
|
|
132
|
+
this.stopPromise = undefined
|
|
126
133
|
/**
|
|
127
134
|
* Narrows the runtime value to the documented type.
|
|
128
135
|
* @type {(() => void) | undefined} */
|
|
@@ -143,6 +150,7 @@ export default class BackgroundJobsMain {
|
|
|
143
150
|
*/
|
|
144
151
|
async start() {
|
|
145
152
|
this._stopped = false
|
|
153
|
+
this.stopPromise = undefined
|
|
146
154
|
this.configuration.setCurrent()
|
|
147
155
|
await this.configuration.initialize({type: "background-jobs-main"})
|
|
148
156
|
await this.configuration.connectBeacon({peerType: "background-jobs-main"})
|
|
@@ -214,18 +222,33 @@ export default class BackgroundJobsMain {
|
|
|
214
222
|
* Runs stop.
|
|
215
223
|
* @returns {Promise<void>} - Resolves when closed.
|
|
216
224
|
*/
|
|
217
|
-
|
|
225
|
+
stop() {
|
|
226
|
+
if (!this.stopPromise) this.stopPromise = this._stop()
|
|
227
|
+
|
|
228
|
+
return this.stopPromise
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
/**
|
|
232
|
+
* Runs the main-process shutdown lifecycle once.
|
|
233
|
+
* @returns {Promise<void>} - Resolves when closed.
|
|
234
|
+
*/
|
|
235
|
+
async _stop() {
|
|
218
236
|
this._stopped = true
|
|
219
237
|
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
238
|
+
await shutdownLifecycle({
|
|
239
|
+
onStopped: this.onStopped,
|
|
240
|
+
shutdown: async () => {
|
|
241
|
+
this._closeWorkers()
|
|
242
|
+
this._clearTimers()
|
|
243
|
+
this._disconnectBeaconHandlers()
|
|
244
|
+
await this.scheduler?.stop()
|
|
245
|
+
try {
|
|
246
|
+
await this._drainWorkerHandoffAdoptions()
|
|
247
|
+
} finally {
|
|
248
|
+
await this._stopBeaconAndServer()
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
})
|
|
229
252
|
}
|
|
230
253
|
|
|
231
254
|
/**
|
|
@@ -287,7 +310,7 @@ export default class BackgroundJobsMain {
|
|
|
287
310
|
try {
|
|
288
311
|
await this._closeServer()
|
|
289
312
|
} finally {
|
|
290
|
-
await this.configuration.closeDatabaseConnections()
|
|
313
|
+
if (this.closeDatabaseConnectionsOnStop) await this.configuration.closeDatabaseConnections()
|
|
291
314
|
}
|
|
292
315
|
}
|
|
293
316
|
|
|
@@ -1,12 +1,30 @@
|
|
|
1
1
|
// @ts-check
|
|
2
2
|
|
|
3
3
|
import runJobPayload, { BackgroundJobPerformedFailure } from "./job-runner.js"
|
|
4
|
+
import { closeRunnerConnections, currentConfigurationOrNull } from "./runner-graceful-shutdown.js"
|
|
4
5
|
import setRunnerProcessTitle from "./runner-process-title.js"
|
|
5
6
|
|
|
6
7
|
const BASE_PROCESS_TITLE = "velocious background-jobs-runner"
|
|
7
8
|
|
|
8
9
|
setRunnerProcessTitle()
|
|
9
10
|
|
|
11
|
+
let shuttingDown = false
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* Closes the runner's connections — releasing any advisory lock a killed-mid-pass
|
|
15
|
+
* job still holds — before exiting, instead of leaving a half-open session that
|
|
16
|
+
* keeps the lock until the DB server's `wait_timeout`.
|
|
17
|
+
* @param {number} exitCode - Process exit code.
|
|
18
|
+
* @returns {Promise<void>}
|
|
19
|
+
*/
|
|
20
|
+
async function shutdownRunner(exitCode) {
|
|
21
|
+
if (shuttingDown) return
|
|
22
|
+
shuttingDown = true
|
|
23
|
+
|
|
24
|
+
await closeRunnerConnections(currentConfigurationOrNull())
|
|
25
|
+
process.exit(exitCode)
|
|
26
|
+
}
|
|
27
|
+
|
|
10
28
|
/**
|
|
11
29
|
* Ids of jobs currently running in this child. A pooled child runs up to
|
|
12
30
|
* `pooledRunnerConcurrency` jobs at once (the worker only dispatches within that
|
|
@@ -111,5 +129,5 @@ function handleMessage(message) {
|
|
|
111
129
|
}
|
|
112
130
|
|
|
113
131
|
process.on("message", (message) => handleMessage(message))
|
|
114
|
-
process.once("disconnect", () =>
|
|
115
|
-
for (const signal of ["SIGTERM", "SIGINT"]) process.once(signal, () =>
|
|
132
|
+
process.once("disconnect", () => void shutdownRunner(0))
|
|
133
|
+
for (const signal of ["SIGTERM", "SIGINT"]) process.once(signal, () => void shutdownRunner(1))
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
// @ts-check
|
|
2
|
+
|
|
3
|
+
import timeout from "awaitery/build/timeout.js"
|
|
4
|
+
|
|
5
|
+
import Configuration, {CurrentConfigurationNotSetError} from "../configuration.js"
|
|
6
|
+
|
|
7
|
+
/** Bounded grace for closing framework connections on shutdown before forcing exit. */
|
|
8
|
+
const SHUTDOWN_CLOSE_TIMEOUT_MS = 5000
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* The subset of a configuration a runner closes on shutdown. Typed structurally so
|
|
12
|
+
* the shutdown path stays typechecked without a broad cast, and a future signature
|
|
13
|
+
* drift surfaces at the call sites (and in tests) instead of hiding behind `any`.
|
|
14
|
+
* @typedef {{disconnectBeacon: () => Promise<void>, closeDatabaseConnections: () => Promise<void>}} RunnerCloseableConfiguration
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* Gracefully closes a background-job runner's framework connections (beacon +
|
|
19
|
+
* database) on shutdown, so the database server ends the session and releases any
|
|
20
|
+
* advisory lock the runner still holds *immediately* — instead of leaving a
|
|
21
|
+
* half-open session that keeps the lock (e.g. the build-planner lock) held until
|
|
22
|
+
* the server's idle `wait_timeout` (hours).
|
|
23
|
+
*
|
|
24
|
+
* A runner child otherwise exits abruptly on shutdown (`process.exit`) and relies
|
|
25
|
+
* on the OS tearing down its sockets. A named lock releases only when its owning
|
|
26
|
+
* session ends, and an abrupt exit / container teardown does not reliably deliver a
|
|
27
|
+
* clean disconnect to the server — so the lock leaks. Sending a real close here
|
|
28
|
+
* (`COM_QUIT` via `closeDatabaseConnections`) makes the server end the session and
|
|
29
|
+
* release the lock deterministically.
|
|
30
|
+
*
|
|
31
|
+
* The beacon disconnect and the database close run independently, each bounded by
|
|
32
|
+
* `closeTimeoutMs`: the database close releases the locks and must still run even if
|
|
33
|
+
* the beacon disconnect hangs (a wedged beacon socket during teardown), and neither
|
|
34
|
+
* may block the exit forever. A failed or timed-out close is thrown, not swallowed —
|
|
35
|
+
* the caller surfaces it (a lock that failed to release must be visible).
|
|
36
|
+
* @param {RunnerCloseableConfiguration | null} configuration - Configuration whose connections to close; null when none is set (nothing to close).
|
|
37
|
+
* @param {number} [closeTimeoutMs] - Max time to spend on each close before giving up.
|
|
38
|
+
* @returns {Promise<void>} - Resolves once both closes have settled; rejects if either failed.
|
|
39
|
+
*/
|
|
40
|
+
export async function closeRunnerConnections(configuration, closeTimeoutMs = SHUTDOWN_CLOSE_TIMEOUT_MS) {
|
|
41
|
+
if (!configuration) return
|
|
42
|
+
|
|
43
|
+
const [beaconResult, databaseResult] = await Promise.allSettled([
|
|
44
|
+
timeout({timeout: closeTimeoutMs}, () => configuration.disconnectBeacon()),
|
|
45
|
+
timeout({timeout: closeTimeoutMs}, () => configuration.closeDatabaseConnections())
|
|
46
|
+
])
|
|
47
|
+
|
|
48
|
+
/** @type {unknown[]} */
|
|
49
|
+
const errors = []
|
|
50
|
+
|
|
51
|
+
if (beaconResult.status == "rejected") errors.push(beaconResult.reason)
|
|
52
|
+
if (databaseResult.status == "rejected") errors.push(databaseResult.reason)
|
|
53
|
+
|
|
54
|
+
if (errors.length == 1) throw errors[0]
|
|
55
|
+
if (errors.length > 1) throw new AggregateError(errors, "Failed to close background-job runner connections on shutdown")
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* The current configuration, or null when none has been set yet — a runner that is
|
|
60
|
+
* signalled before it runs any job holds no connections (and no locks) to close.
|
|
61
|
+
* Only that expected "not set yet" case is treated as null; any other error is a real
|
|
62
|
+
* fault and is rethrown rather than masked.
|
|
63
|
+
* @returns {Configuration | null} - The current configuration, or null when none is set.
|
|
64
|
+
*/
|
|
65
|
+
export function currentConfigurationOrNull() {
|
|
66
|
+
try {
|
|
67
|
+
return Configuration.current()
|
|
68
|
+
} catch (error) {
|
|
69
|
+
if (error instanceof CurrentConfigurationNotSetError) return null
|
|
70
|
+
|
|
71
|
+
throw error
|
|
72
|
+
}
|
|
73
|
+
}
|
|
@@ -8,6 +8,7 @@ import configurationResolver from "../configuration-resolver.js"
|
|
|
8
8
|
import BackgroundJobsStatusReporter from "./status-reporter.js"
|
|
9
9
|
import {randomUUID} from "crypto"
|
|
10
10
|
import {fileURLToPath} from "node:url"
|
|
11
|
+
import shutdownLifecycle from "../utils/shutdown-lifecycle.js"
|
|
11
12
|
|
|
12
13
|
/** Grace period after SIGTERM before a lingering process runner is SIGKILLed. */
|
|
13
14
|
const FORKED_CHILD_SIGKILL_GRACE_MS = 5000
|
|
@@ -73,8 +74,10 @@ export default class BackgroundJobsWorker {
|
|
|
73
74
|
* @param {number} [args.forkedChildSigkillGraceMs] - Override the grace period between SIGTERM and SIGKILL when reaping lingering process runners on stop.
|
|
74
75
|
* @param {number} [args.heartbeatIntervalMs] - Override the liveness heartbeat interval (default 15000ms).
|
|
75
76
|
* @param {number} [args.jobTimeoutMs] - Override the wall-clock timeout for forked and pooled jobs from `configuration.getBackgroundJobsConfig()`. `0` disables it.
|
|
77
|
+
* @param {boolean} [args.closeDatabaseConnectionsOnStop] - Whether stop owns closing the configuration's database pools (default true).
|
|
78
|
+
* @param {() => void | Promise<void>} [args.onStopped] - Lifecycle hook invoked after the worker finishes stopping.
|
|
76
79
|
*/
|
|
77
|
-
constructor({configuration, host, port, maxConcurrentForkedJobs, maxConcurrentInlineJobs, pooledRunnerCount, pooledRunnerConcurrency, pooledRunnerMaxJobs, pooledRunnerMaxRssBytes, pooledRunnerMaxLifetimeMs, forkedChildSigkillGraceMs, heartbeatIntervalMs, jobTimeoutMs} = {}) {
|
|
80
|
+
constructor({configuration, host, port, maxConcurrentForkedJobs, maxConcurrentInlineJobs, pooledRunnerCount, pooledRunnerConcurrency, pooledRunnerMaxJobs, pooledRunnerMaxRssBytes, pooledRunnerMaxLifetimeMs, forkedChildSigkillGraceMs, heartbeatIntervalMs, jobTimeoutMs, closeDatabaseConnectionsOnStop = true, onStopped} = {}) {
|
|
78
81
|
/**
|
|
79
82
|
* Narrows the runtime value to the documented type.
|
|
80
83
|
* @type {Promise<import("../configuration.js").default>} */
|
|
@@ -85,6 +88,8 @@ export default class BackgroundJobsWorker {
|
|
|
85
88
|
this.configuration = undefined
|
|
86
89
|
this.host = host
|
|
87
90
|
this.port = port
|
|
91
|
+
this.closeDatabaseConnectionsOnStop = closeDatabaseConnectionsOnStop
|
|
92
|
+
this.onStopped = onStopped
|
|
88
93
|
/**
|
|
89
94
|
* Constructor override for the inline-job concurrency cap. When unset
|
|
90
95
|
* the cap is read from `configuration.getBackgroundJobsConfig()` in
|
|
@@ -136,6 +141,8 @@ export default class BackgroundJobsWorker {
|
|
|
136
141
|
*/
|
|
137
142
|
this.jobTimeoutMsOverride = typeof jobTimeoutMs === "number" ? jobTimeoutMs : undefined
|
|
138
143
|
this.shouldStop = false
|
|
144
|
+
/** @type {Promise<void> | undefined} */
|
|
145
|
+
this.stopPromise = undefined
|
|
139
146
|
this.workerId = randomUUID()
|
|
140
147
|
this.heartbeatIntervalMs = typeof heartbeatIntervalMs === "number" && heartbeatIntervalMs >= 1
|
|
141
148
|
? heartbeatIntervalMs
|
|
@@ -195,6 +202,8 @@ export default class BackgroundJobsWorker {
|
|
|
195
202
|
* @returns {Promise<void>} - Resolves when connected.
|
|
196
203
|
*/
|
|
197
204
|
async start() {
|
|
205
|
+
this.shouldStop = false
|
|
206
|
+
this.stopPromise = undefined
|
|
198
207
|
this.configuration = await this.configurationPromise
|
|
199
208
|
this.configuration.setCurrent()
|
|
200
209
|
await this.configuration.initialize({type: "background-jobs-worker"})
|
|
@@ -241,37 +250,53 @@ export default class BackgroundJobsWorker {
|
|
|
241
250
|
* @param {number} [args.timeoutMs] - Max wait for in-flight jobs (per phase) in ms.
|
|
242
251
|
* @returns {Promise<void>} - Resolves when stopped.
|
|
243
252
|
*/
|
|
244
|
-
|
|
245
|
-
if (this.
|
|
246
|
-
this.shouldStop = true
|
|
247
|
-
this._stopHeartbeat()
|
|
253
|
+
stop({timeoutMs} = {}) {
|
|
254
|
+
if (!this.stopPromise) this.stopPromise = this._stop({timeoutMs})
|
|
248
255
|
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
if (this.jsonSocket) {
|
|
252
|
-
try {
|
|
253
|
-
this.jsonSocket.send({type: "draining"})
|
|
254
|
-
} catch {
|
|
255
|
-
// Socket may already be closing; nothing to do.
|
|
256
|
-
}
|
|
257
|
-
}
|
|
256
|
+
return this.stopPromise
|
|
257
|
+
}
|
|
258
258
|
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
259
|
+
/**
|
|
260
|
+
* Runs the worker shutdown lifecycle once.
|
|
261
|
+
* @param {object} [args] - Options.
|
|
262
|
+
* @param {number} [args.timeoutMs] - Max wait for in-flight jobs (per phase) in ms.
|
|
263
|
+
* @returns {Promise<void>} - Resolves when stopped.
|
|
264
|
+
*/
|
|
265
|
+
async _stop({timeoutMs} = {}) {
|
|
266
|
+
this.shouldStop = true
|
|
267
|
+
this._stopHeartbeat()
|
|
266
268
|
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
269
|
+
await shutdownLifecycle({
|
|
270
|
+
onStopped: this.onStopped,
|
|
271
|
+
shutdown: async () => {
|
|
272
|
+
// Announce drain so main stops dispatching but keeps the connection
|
|
273
|
+
// open until we close it ourselves below.
|
|
274
|
+
if (this.jsonSocket) {
|
|
275
|
+
try {
|
|
276
|
+
this.jsonSocket.send({type: "draining"})
|
|
277
|
+
} catch {
|
|
278
|
+
// Socket may already be closing; nothing to do.
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
await this._drainInflight(this.inflightInlineJobs, timeoutMs)
|
|
283
|
+
await this._drainInflight(this.inflightPooledJobs, timeoutMs)
|
|
284
|
+
await this._drainInflight(this.inflightProcessJobs, timeoutMs)
|
|
285
|
+
await this._terminateProcessChildren()
|
|
286
|
+
// Give in-flight result reports (now decoupled from job slots) a bounded
|
|
287
|
+
// chance to land before the socket closes.
|
|
288
|
+
await this._drainInflight(this.inflightReports, timeoutMs)
|
|
289
|
+
|
|
290
|
+
if (this.jsonSocket) this.jsonSocket.close()
|
|
291
|
+
if (!this.configuration) return
|
|
292
|
+
|
|
293
|
+
try {
|
|
294
|
+
await this.configuration.disconnectBeacon()
|
|
295
|
+
} finally {
|
|
296
|
+
if (this.closeDatabaseConnectionsOnStop) await this.configuration.closeDatabaseConnections()
|
|
297
|
+
}
|
|
273
298
|
}
|
|
274
|
-
}
|
|
299
|
+
})
|
|
275
300
|
}
|
|
276
301
|
|
|
277
302
|
/**
|
package/build/configuration.js
CHANGED
|
@@ -119,6 +119,15 @@ export default class VelociousConfiguration {
|
|
|
119
119
|
* Close database connections promise.
|
|
120
120
|
* @type {Promise<void> | null} */
|
|
121
121
|
_closeDatabaseConnectionsPromise = null
|
|
122
|
+
|
|
123
|
+
/**
|
|
124
|
+
* Dedicated advisory-lock connections currently holding a lock. These are spawned
|
|
125
|
+
* outside the pools' tracked sets (so a hold-timeout lock survives pool checkouts),
|
|
126
|
+
* so `closeDatabaseConnections` would otherwise walk past them; tracking them here
|
|
127
|
+
* lets a shutdown close them and release the lock instead of orphaning it.
|
|
128
|
+
* @type {Set<import("./database/drivers/base.js").default>} */
|
|
129
|
+
_advisoryLockConnections = new Set()
|
|
130
|
+
|
|
122
131
|
/**
|
|
123
132
|
* Runs current.
|
|
124
133
|
* @returns {VelociousConfiguration} - The current.
|
|
@@ -2986,6 +2995,53 @@ export default class VelociousConfiguration {
|
|
|
2986
2995
|
})
|
|
2987
2996
|
}
|
|
2988
2997
|
|
|
2998
|
+
/**
|
|
2999
|
+
* Registers a dedicated connection that currently holds an advisory lock, so a
|
|
3000
|
+
* shutdown can close it and release the lock. See `_advisoryLockConnections`.
|
|
3001
|
+
* @param {import("./database/drivers/base.js").default} connection - The dedicated lock connection.
|
|
3002
|
+
* @returns {void}
|
|
3003
|
+
*/
|
|
3004
|
+
registerAdvisoryLockConnection(connection) {
|
|
3005
|
+
this._advisoryLockConnections.add(connection)
|
|
3006
|
+
}
|
|
3007
|
+
|
|
3008
|
+
/**
|
|
3009
|
+
* Unregisters a dedicated advisory-lock connection once its lock scope ends and the
|
|
3010
|
+
* connection has been (or is about to be) closed by its owner.
|
|
3011
|
+
* @param {import("./database/drivers/base.js").default} connection - The dedicated lock connection.
|
|
3012
|
+
* @returns {void}
|
|
3013
|
+
*/
|
|
3014
|
+
unregisterAdvisoryLockConnection(connection) {
|
|
3015
|
+
this._advisoryLockConnections.delete(connection)
|
|
3016
|
+
}
|
|
3017
|
+
|
|
3018
|
+
/**
|
|
3019
|
+
* Closes every registered dedicated advisory-lock connection, ending its session so
|
|
3020
|
+
* the DB server releases the lock. Every connection is attempted before any failure
|
|
3021
|
+
* is surfaced, so one stuck close does not leave the others' locks held; a failure is
|
|
3022
|
+
* then thrown (never swallowed), aggregated when more than one connection failed.
|
|
3023
|
+
* @returns {Promise<void>} - Resolves once all have been closed; rejects if any failed.
|
|
3024
|
+
*/
|
|
3025
|
+
async _closeAdvisoryLockConnections() {
|
|
3026
|
+
const connections = [...this._advisoryLockConnections]
|
|
3027
|
+
|
|
3028
|
+
this._advisoryLockConnections.clear()
|
|
3029
|
+
|
|
3030
|
+
/** @type {unknown[]} */
|
|
3031
|
+
const errors = []
|
|
3032
|
+
|
|
3033
|
+
for (const connection of connections) {
|
|
3034
|
+
try {
|
|
3035
|
+
await connection.close()
|
|
3036
|
+
} catch (error) {
|
|
3037
|
+
errors.push(error)
|
|
3038
|
+
}
|
|
3039
|
+
}
|
|
3040
|
+
|
|
3041
|
+
if (errors.length == 1) throw errors[0]
|
|
3042
|
+
if (errors.length > 1) throw new AggregateError(errors, "Failed to close dedicated advisory-lock connections")
|
|
3043
|
+
}
|
|
3044
|
+
|
|
2989
3045
|
/**
|
|
2990
3046
|
* Closes active database connections and clears global connections.
|
|
2991
3047
|
* @returns {Promise<void>} - Resolves when complete.
|
|
@@ -3000,21 +3056,30 @@ export default class VelociousConfiguration {
|
|
|
3000
3056
|
const constructors = new Set()
|
|
3001
3057
|
|
|
3002
3058
|
this._closeDatabaseConnectionsPromise = (async () => {
|
|
3003
|
-
|
|
3004
|
-
|
|
3005
|
-
|
|
3006
|
-
|
|
3059
|
+
try {
|
|
3060
|
+
// Close dedicated advisory-lock connections first: they are spawned outside the
|
|
3061
|
+
// pools' tracked sets, so `pool.closeAll()` would not reach them and a lock held
|
|
3062
|
+
// by a runner torn down mid-pass would leak until the DB server's `wait_timeout`.
|
|
3063
|
+
// Still close the pools even if this throws, so a stuck lock connection does not
|
|
3064
|
+
// leave the rest of the connections open.
|
|
3065
|
+
await this._closeAdvisoryLockConnections()
|
|
3066
|
+
} finally {
|
|
3067
|
+
for (const pool of Object.values(this.databasePools)) {
|
|
3068
|
+
if (!pool) continue
|
|
3069
|
+
|
|
3070
|
+
await pool.closeAll()
|
|
3071
|
+
|
|
3072
|
+
const PoolClass = /** @type {typeof import("./database/pool/base.js").default} */ (pool.constructor)
|
|
3073
|
+
constructors.add(PoolClass)
|
|
3074
|
+
}
|
|
3007
3075
|
|
|
3008
|
-
const PoolClass
|
|
3009
|
-
|
|
3010
|
-
|
|
3076
|
+
for (const PoolClass of constructors) {
|
|
3077
|
+
PoolClass.clearGlobalConnections(this)
|
|
3078
|
+
}
|
|
3011
3079
|
|
|
3012
|
-
|
|
3013
|
-
|
|
3080
|
+
// Allow models to be re-initialized after connections are closed.
|
|
3081
|
+
this._modelsInitialized = false
|
|
3014
3082
|
}
|
|
3015
|
-
|
|
3016
|
-
// Allow models to be re-initialized after connections are closed.
|
|
3017
|
-
this._modelsInitialized = false
|
|
3018
3083
|
})()
|
|
3019
3084
|
|
|
3020
3085
|
try {
|
|
@@ -150,13 +150,22 @@ export default class AdvisoryLockRunner {
|
|
|
150
150
|
async withDedicatedConnection(callback) {
|
|
151
151
|
const connection = await this.configuration.getDatabasePool(this.databaseIdentifier).spawnConnection()
|
|
152
152
|
|
|
153
|
+
// The spawned driver owns its physical connection unless it borrows a shared one
|
|
154
|
+
// via `getConnection`. An owned connection lives outside the pools' tracked sets,
|
|
155
|
+
// so register it while the lock is held: a shutdown then closes it (releasing the
|
|
156
|
+
// lock) instead of orphaning a half-open session until the DB `wait_timeout`.
|
|
157
|
+
const ownsConnection = !connection.getArgs().getConnection
|
|
158
|
+
|
|
159
|
+
if (ownsConnection) this.configuration.registerAdvisoryLockConnection(connection)
|
|
160
|
+
|
|
153
161
|
try {
|
|
154
162
|
return await callback(connection)
|
|
155
163
|
} finally {
|
|
156
|
-
if (
|
|
157
|
-
|
|
158
|
-
} else {
|
|
164
|
+
if (ownsConnection) {
|
|
165
|
+
this.configuration.unregisterAdvisoryLockConnection(connection)
|
|
159
166
|
await connection.close()
|
|
167
|
+
} else {
|
|
168
|
+
await connection.releaseHeldAdvisoryLocks()
|
|
160
169
|
}
|
|
161
170
|
}
|
|
162
171
|
}
|
|
@@ -1710,6 +1710,12 @@ class VelociousDatabaseRecord {
|
|
|
1710
1710
|
*/
|
|
1711
1711
|
static switchesTenantDatabase(databaseIdentifierOrResolver) {
|
|
1712
1712
|
this._tenantDatabaseIdentifierResolver = databaseIdentifierOrResolver
|
|
1713
|
+
|
|
1714
|
+
if (this._translationClass) {
|
|
1715
|
+
const translatedModelClass = this
|
|
1716
|
+
|
|
1717
|
+
this._translationClass.switchesTenantDatabase(({tenant}) => translatedModelClass.getTenantDatabaseIdentifier(tenant))
|
|
1718
|
+
}
|
|
1713
1719
|
}
|
|
1714
1720
|
|
|
1715
1721
|
/**
|
|
@@ -2814,6 +2820,12 @@ class VelociousDatabaseRecord {
|
|
|
2814
2820
|
TranslationClass.setTableName(this.getTranslationsTableName())
|
|
2815
2821
|
TranslationClass.belongsTo(belongsTo)
|
|
2816
2822
|
|
|
2823
|
+
if (this.hasTenantDatabaseIdentifierResolver()) {
|
|
2824
|
+
const translatedModelClass = this
|
|
2825
|
+
|
|
2826
|
+
TranslationClass.switchesTenantDatabase(({tenant}) => translatedModelClass.getTenantDatabaseIdentifier(tenant))
|
|
2827
|
+
}
|
|
2828
|
+
|
|
2817
2829
|
this._translationClass = TranslationClass
|
|
2818
2830
|
|
|
2819
2831
|
return this._translationClass
|