velocious 1.0.719 → 1.0.721
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/background-jobs/main.js +163 -2
- package/build/configuration.js +20 -2
- package/build/database/advisory-lock-file-backend.js +21 -3
- package/build/database/advisory-lock-lock-server-backend.js +12 -2
- package/build/database/advisory-lock-runner.js +91 -1
- package/build/routes/base-route.js +3 -2
- package/build/routes/basic-route.js +33 -8
- package/build/routes/get-route.js +17 -1
- package/build/routes/post-route.js +17 -1
- package/build/routes/resolver.js +5 -3
- package/build/src/background-jobs/main.d.ts +60 -0
- package/build/src/background-jobs/main.d.ts.map +1 -1
- package/build/src/background-jobs/main.js +154 -3
- package/build/src/configuration.d.ts +18 -1
- package/build/src/configuration.d.ts.map +1 -1
- package/build/src/configuration.js +20 -3
- package/build/src/database/advisory-lock-file-backend.d.ts +2 -1
- package/build/src/database/advisory-lock-file-backend.d.ts.map +1 -1
- package/build/src/database/advisory-lock-file-backend.js +18 -5
- package/build/src/database/advisory-lock-lock-server-backend.d.ts +5 -1
- package/build/src/database/advisory-lock-lock-server-backend.d.ts.map +1 -1
- package/build/src/database/advisory-lock-lock-server-backend.js +11 -3
- package/build/src/database/advisory-lock-runner.d.ts +42 -0
- package/build/src/database/advisory-lock-runner.d.ts.map +1 -1
- package/build/src/database/advisory-lock-runner.js +82 -2
- package/build/src/routes/base-route.d.ts +3 -2
- package/build/src/routes/base-route.d.ts.map +1 -1
- package/build/src/routes/base-route.js +4 -3
- package/build/src/routes/basic-route.d.ts +19 -6
- package/build/src/routes/basic-route.d.ts.map +1 -1
- package/build/src/routes/basic-route.js +29 -9
- package/build/src/routes/get-route.d.ts +12 -1
- package/build/src/routes/get-route.d.ts.map +1 -1
- package/build/src/routes/get-route.js +16 -2
- package/build/src/routes/post-route.d.ts +12 -1
- package/build/src/routes/post-route.d.ts.map +1 -1
- package/build/src/routes/post-route.js +16 -2
- package/build/src/routes/resolver.d.ts +2 -1
- package/build/src/routes/resolver.d.ts.map +1 -1
- package/build/src/routes/resolver.js +6 -4
- package/package.json +1 -1
- package/src/background-jobs/main.js +163 -2
- package/src/configuration.js +20 -2
- package/src/database/advisory-lock-file-backend.js +21 -3
- package/src/database/advisory-lock-lock-server-backend.js +12 -2
- package/src/database/advisory-lock-runner.js +91 -1
- package/src/routes/base-route.js +3 -2
- package/src/routes/basic-route.js +33 -8
- package/src/routes/get-route.js +17 -1
- package/src/routes/post-route.js +17 -1
- package/src/routes/resolver.js +5 -3
|
@@ -9,7 +9,7 @@ import Logger from "../logger.js"
|
|
|
9
9
|
import PruneTerminalBackgroundJobsJob from "../jobs/prune-terminal-background-jobs.js"
|
|
10
10
|
import VelociousError from "../velocious-error.js"
|
|
11
11
|
import shutdownLifecycle, { runShutdownSteps } from "../utils/shutdown-lifecycle.js"
|
|
12
|
-
import { validateGenerationId, workerIdBelongsToGeneration } from "./generation-identity.js"
|
|
12
|
+
import { parseGenerationWorkerId, validateGenerationId, workerIdBelongsToGeneration } from "./generation-identity.js"
|
|
13
13
|
import BackgroundJobsLifecycleControlServer from "./lifecycle-control-server.js"
|
|
14
14
|
|
|
15
15
|
/**
|
|
@@ -40,6 +40,8 @@ const WORKER_LIVENESS_SWEEP_MS = 15000
|
|
|
40
40
|
/** Grace for workers from the previous main generation to reconnect and adopt leases. */
|
|
41
41
|
const WORKER_RECONNECT_GRACE_MS = 30000
|
|
42
42
|
const GENERATION_ORPHANED_AFTER_MS = 60 * 60 * 1000
|
|
43
|
+
/** Prefix of the per-generation liveness advisory lock one live main holds for its whole generation. */
|
|
44
|
+
const GENERATION_LIVENESS_LOCK_PREFIX = "background-jobs:generation-liveness"
|
|
43
45
|
const WORKER_RECONNECT_GRACE_VALIDATION_MESSAGE = `workerReconnectGraceMs must be an integer between 0 and ${MAX_TIMER_MS}`
|
|
44
46
|
|
|
45
47
|
/**
|
|
@@ -246,6 +248,11 @@ export default class BackgroundJobsMain {
|
|
|
246
248
|
this._beaconClient = undefined
|
|
247
249
|
/** @type {BackgroundJobsLifecycleControlServer | undefined} */
|
|
248
250
|
this.lifecycleControlServer = undefined
|
|
251
|
+
/**
|
|
252
|
+
* Dedicated advisory-lock connection holding this main's generation liveness
|
|
253
|
+
* lock for the whole lifetime of the main (generation mode only).
|
|
254
|
+
* @type {import("../database/drivers/base.js").default | undefined} */
|
|
255
|
+
this._livenessLockConnection = undefined
|
|
249
256
|
}
|
|
250
257
|
|
|
251
258
|
/**
|
|
@@ -296,6 +303,7 @@ export default class BackgroundJobsMain {
|
|
|
296
303
|
if (this.generationId && !this.adapter.supportsOwnedEnqueueFromHandoff()) {
|
|
297
304
|
throw new Error("The configured background jobs adapter does not support atomic owned-handoff enqueue")
|
|
298
305
|
}
|
|
306
|
+
if (this.generationId) await this._acquireGenerationLivenessLock()
|
|
299
307
|
|
|
300
308
|
if (!this.generationId || this.initialGenerationState !== "candidate") {
|
|
301
309
|
this.startupHandoffSnapshot = await this._generationOwnedHandoffSnapshot()
|
|
@@ -389,7 +397,11 @@ export default class BackgroundJobsMain {
|
|
|
389
397
|
try {
|
|
390
398
|
await this._drainStartupHandoffReclaim()
|
|
391
399
|
} finally {
|
|
392
|
-
|
|
400
|
+
try {
|
|
401
|
+
await this._releaseGenerationLivenessLock()
|
|
402
|
+
} finally {
|
|
403
|
+
await this._stopBeaconAndServer()
|
|
404
|
+
}
|
|
393
405
|
}
|
|
394
406
|
}
|
|
395
407
|
}
|
|
@@ -624,6 +636,7 @@ export default class BackgroundJobsMain {
|
|
|
624
636
|
}
|
|
625
637
|
this.lifecycleState = "active"
|
|
626
638
|
this._creditReadyWorkers()
|
|
639
|
+
await this._settleForeignGenerationHandoffs()
|
|
627
640
|
this.logger.info(() => ["Background jobs generation activation acknowledged", {generationId: this.generationId}])
|
|
628
641
|
void this._drain().catch((error) => {
|
|
629
642
|
this.logger.error(() => ["Background jobs generation post-activation drain failed", {error, generationId: this.generationId}])
|
|
@@ -751,6 +764,154 @@ export default class BackgroundJobsMain {
|
|
|
751
764
|
* retained solely to perform this cleanup.
|
|
752
765
|
* @returns {void}
|
|
753
766
|
*/
|
|
767
|
+
/**
|
|
768
|
+
* Builds the per-generation liveness advisory-lock name for one generation on
|
|
769
|
+
* this main's background-jobs database. A live main holds this lock for its
|
|
770
|
+
* whole lifetime; a foreign generation whose lock is free (or stale) has no
|
|
771
|
+
* live main supervising its durable handoffs.
|
|
772
|
+
* @param {string} generationId - Generation to name.
|
|
773
|
+
* @returns {string} - Lock name.
|
|
774
|
+
*/
|
|
775
|
+
_generationLivenessLockName(generationId) {
|
|
776
|
+
const databaseIdentifier = this.configuration.getBackgroundJobsConfig().databaseIdentifier || "default"
|
|
777
|
+
|
|
778
|
+
return `${GENERATION_LIVENESS_LOCK_PREFIX}:${databaseIdentifier}:${generationId}`
|
|
779
|
+
}
|
|
780
|
+
|
|
781
|
+
/**
|
|
782
|
+
* Acquires this main's own generation liveness lock on a dedicated connection
|
|
783
|
+
* it holds until stop. Contention means a still-live main of the same
|
|
784
|
+
* generation exists (a deploy anomaly); the main keeps running without the
|
|
785
|
+
* marker and cross-generation settlement conservatively leaves its rows to it.
|
|
786
|
+
* @returns {Promise<void>} - Resolves after the marker is held or degraded.
|
|
787
|
+
*/
|
|
788
|
+
async _acquireGenerationLivenessLock() {
|
|
789
|
+
if (!this.generationId || this._livenessLockConnection) return
|
|
790
|
+
const lockName = this._generationLivenessLockName(this.generationId)
|
|
791
|
+
|
|
792
|
+
try {
|
|
793
|
+
const connection = await this.configuration.getDatabasePool(this._livenessLockDatabaseIdentifier()).spawnConnection()
|
|
794
|
+
const acquired = await connection.tryAcquireAdvisoryLock(lockName)
|
|
795
|
+
|
|
796
|
+
if (!acquired) {
|
|
797
|
+
await connection.close()
|
|
798
|
+
this.logger.warn(() => ["Background jobs generation liveness lock is already held by another live main of this generation", {generationId: this.generationId}])
|
|
799
|
+
|
|
800
|
+
return
|
|
801
|
+
}
|
|
802
|
+
|
|
803
|
+
this.configuration.registerAdvisoryLockConnection(connection)
|
|
804
|
+
this._livenessLockConnection = connection
|
|
805
|
+
} catch (error) {
|
|
806
|
+
this.logger.warn(() => ["Failed to acquire background jobs generation liveness lock", {error, generationId: this.generationId}])
|
|
807
|
+
}
|
|
808
|
+
}
|
|
809
|
+
|
|
810
|
+
/**
|
|
811
|
+
* Releases this main's generation liveness marker during stop so a later
|
|
812
|
+
* generation activation can prove this generation is gone. Closing the
|
|
813
|
+
* dedicated connection releases the advisory lock on every tier.
|
|
814
|
+
* @returns {Promise<void>} - Resolves after the marker is released.
|
|
815
|
+
*/
|
|
816
|
+
async _releaseGenerationLivenessLock() {
|
|
817
|
+
const connection = this._livenessLockConnection
|
|
818
|
+
|
|
819
|
+
this._livenessLockConnection = undefined
|
|
820
|
+
if (!connection) return
|
|
821
|
+
|
|
822
|
+
this.configuration.unregisterAdvisoryLockConnection(connection)
|
|
823
|
+
|
|
824
|
+
try {
|
|
825
|
+
await connection.releaseHeldAdvisoryLocks()
|
|
826
|
+
} finally {
|
|
827
|
+
await connection.close()
|
|
828
|
+
}
|
|
829
|
+
}
|
|
830
|
+
|
|
831
|
+
/**
|
|
832
|
+
* Database identifier whose pools carry the generation liveness locks. This
|
|
833
|
+
* matches the built-in store's own resolution of the background-jobs config.
|
|
834
|
+
* @returns {string} - Database identifier.
|
|
835
|
+
*/
|
|
836
|
+
_livenessLockDatabaseIdentifier() {
|
|
837
|
+
return this.configuration.getBackgroundJobsConfig().databaseIdentifier || "default"
|
|
838
|
+
}
|
|
839
|
+
|
|
840
|
+
/**
|
|
841
|
+
* One activation event: snapshot all durable handoffs, group the rows whose
|
|
842
|
+
* worker identity belongs to a foreign generation, and settle each group only
|
|
843
|
+
* after proving that generation has no live main (its liveness lock is free).
|
|
844
|
+
* Own-generation rows, identity-free legacy rows, and rows of live foreign
|
|
845
|
+
* generations stay owned by their existing recovery paths. Failures are
|
|
846
|
+
* reported as framework errors and never fail activation.
|
|
847
|
+
* @returns {Promise<void>} - Resolves after settlement or its reported failure.
|
|
848
|
+
*/
|
|
849
|
+
async _settleForeignGenerationHandoffs() {
|
|
850
|
+
if (!this.generationId) return
|
|
851
|
+
|
|
852
|
+
try {
|
|
853
|
+
const handoffs = await this.store.snapshotHandedOffJobs()
|
|
854
|
+
/** @type {Map<string, import("./types.js").BackgroundJobHandoffSnapshot[]>} */
|
|
855
|
+
const handoffsByForeignGeneration = new Map()
|
|
856
|
+
|
|
857
|
+
for (const handoff of handoffs) {
|
|
858
|
+
const parsed = parseGenerationWorkerId(handoff.workerId)
|
|
859
|
+
if (!parsed || parsed.generationId === this.generationId) continue
|
|
860
|
+
|
|
861
|
+
const group = handoffsByForeignGeneration.get(parsed.generationId) || []
|
|
862
|
+
group.push(handoff)
|
|
863
|
+
handoffsByForeignGeneration.set(parsed.generationId, group)
|
|
864
|
+
}
|
|
865
|
+
|
|
866
|
+
for (const [foreignGenerationId, foreignHandoffs] of handoffsByForeignGeneration) {
|
|
867
|
+
const settledJobs = await this._settleForeignGenerationHandoffGroup({foreignGenerationId, handoffs: foreignHandoffs})
|
|
868
|
+
|
|
869
|
+
if (settledJobs.length > 0) await this._handleOrphanedJobs({jobs: settledJobs, warning: "Marked orphaned background jobs whose generation was retired before this generation activated"})
|
|
870
|
+
}
|
|
871
|
+
} catch (error) {
|
|
872
|
+
const normalizedError = error instanceof Error ? error : new Error(String(error))
|
|
873
|
+
const payload = {context: {generationId: this.generationId, stage: "background-job-activation-cross-generation-settlement"}, error: normalizedError}
|
|
874
|
+
const errorEvents = this.configuration.getErrorEvents()
|
|
875
|
+
|
|
876
|
+
this.logger.error(() => ["Failed to settle foreign-generation handoffs at activation:", normalizedError])
|
|
877
|
+
errorEvents.emit("framework-error", payload)
|
|
878
|
+
errorEvents.emit("all-error", {...payload, errorType: "framework-error"})
|
|
879
|
+
}
|
|
880
|
+
}
|
|
881
|
+
|
|
882
|
+
/**
|
|
883
|
+
* Settles one foreign generation's exact handoffs under a proof of absence:
|
|
884
|
+
* acquire that generation's liveness lock on a dedicated connection and hold
|
|
885
|
+
* it across the fenced store transition. A busy lock means a live main of
|
|
886
|
+
* that generation still supervises its rows (retired-and-draining included),
|
|
887
|
+
* so nothing here may touch them.
|
|
888
|
+
* @param {object} args - Options.
|
|
889
|
+
* @param {string} args.foreignGenerationId - Generation whose rows are candidates.
|
|
890
|
+
* @param {import("./types.js").BackgroundJobHandoffSnapshot[]} args.handoffs - Exact snapshot rows.
|
|
891
|
+
* @returns {Promise<import("./types.js").BackgroundJobRow[]>} - Accepted orphan transitions.
|
|
892
|
+
*/
|
|
893
|
+
async _settleForeignGenerationHandoffGroup({foreignGenerationId, handoffs}) {
|
|
894
|
+
const lockName = this._generationLivenessLockName(foreignGenerationId)
|
|
895
|
+
const connection = await this.configuration.getDatabasePool(this._livenessLockDatabaseIdentifier()).spawnConnection()
|
|
896
|
+
|
|
897
|
+
try {
|
|
898
|
+
const acquired = await connection.tryAcquireAdvisoryLock(lockName)
|
|
899
|
+
|
|
900
|
+
if (!acquired) return []
|
|
901
|
+
|
|
902
|
+
return await this.store.markOrphanedHandoffs({
|
|
903
|
+
error: "Job orphaned when a new generation activated while its owning generation was not live",
|
|
904
|
+
handoffs
|
|
905
|
+
})
|
|
906
|
+
} finally {
|
|
907
|
+
try {
|
|
908
|
+
await connection.releaseHeldAdvisoryLocks()
|
|
909
|
+
} finally {
|
|
910
|
+
await connection.close()
|
|
911
|
+
}
|
|
912
|
+
}
|
|
913
|
+
}
|
|
914
|
+
|
|
754
915
|
_setupStartupHandoffReclaim() {
|
|
755
916
|
if (this.startupHandoffSnapshot.length === 0) return
|
|
756
917
|
if (this._startupHandoffReclaimTimer || this._startupHandoffReclaimPromise || this._startupHandoffGraceElapsed) return
|
package/build/configuration.js
CHANGED
|
@@ -273,6 +273,7 @@ export default class VelociousConfiguration {
|
|
|
273
273
|
* served this lock" surface (design §3).
|
|
274
274
|
* @type {Map<string, {backend: import("./database/advisory-lock-backend-policy.js").AdvisoryLockTier, fencingSupported: boolean, servedAt: string}>} */
|
|
275
275
|
_advisoryLockAcquisitions = new Map()
|
|
276
|
+
_advisoryLockReentries = new Map()
|
|
276
277
|
|
|
277
278
|
/** @type {Map<string, number>} */
|
|
278
279
|
_schemaCacheGenerationsByReuseKey = new Map()
|
|
@@ -4355,19 +4356,36 @@ export default class VelociousConfiguration {
|
|
|
4355
4356
|
this._advisoryLockAcquisitions.set(lockName, {backend, fencingSupported, servedAt: new Date().toISOString()})
|
|
4356
4357
|
}
|
|
4357
4358
|
|
|
4359
|
+
/**
|
|
4360
|
+
* Counts one re-entry of an advisory lock the calling async context already
|
|
4361
|
+
* holds (`Monitor` semantics, see `advisory-lock-runner.js`).
|
|
4362
|
+
*
|
|
4363
|
+
* This is the measurement that decides whether a hand-passed `*LockHeld`
|
|
4364
|
+
* boolean is still load-bearing: a name whose reentries stay at zero over a
|
|
4365
|
+
* real workload is not nesting at all, while a name that reenters is one
|
|
4366
|
+
* where the framework now owns the depth and the application flag can go.
|
|
4367
|
+
* @param {object} args - Re-entry record.
|
|
4368
|
+
* @param {string} args.lockName - Lock name.
|
|
4369
|
+
* @returns {void}
|
|
4370
|
+
*/
|
|
4371
|
+
recordAdvisoryLockReentry({lockName}) {
|
|
4372
|
+
this._advisoryLockReentries.set(lockName, (this._advisoryLockReentries.get(lockName) ?? 0) + 1)
|
|
4373
|
+
}
|
|
4374
|
+
|
|
4358
4375
|
/**
|
|
4359
4376
|
* The "which backend served this lock" diagnostics surface (design §3):
|
|
4360
4377
|
* per lock name, the tier of the most recent acquisition, whether that tier
|
|
4361
4378
|
* carries fencing tokens, and the resolved policy (minimum/allowed/preferred
|
|
4362
4379
|
* tiers) for the name.
|
|
4363
4380
|
* @param {string} lockName - Lock name.
|
|
4364
|
-
* @returns {{acquisition: {backend: import("./database/advisory-lock-backend-policy.js").AdvisoryLockTier, fencingSupported: boolean, servedAt: string} | null, lockServer: {connected: boolean, lastError: string | null, lastProbeAt: string | null, leasedNames: string[]} | null, policy: import("./database/advisory-lock-backend-policy.js").ResolvedAdvisoryLockBackendPolicy}} - Backend status for the name.
|
|
4381
|
+
* @returns {{acquisition: {backend: import("./database/advisory-lock-backend-policy.js").AdvisoryLockTier, fencingSupported: boolean, servedAt: string} | null, lockServer: {connected: boolean, lastError: string | null, lastProbeAt: string | null, leasedNames: string[]} | null, policy: import("./database/advisory-lock-backend-policy.js").ResolvedAdvisoryLockBackendPolicy, reentries: number}} - Backend status for the name.
|
|
4365
4382
|
*/
|
|
4366
4383
|
getAdvisoryLockBackendStatus(lockName) {
|
|
4367
4384
|
return {
|
|
4368
4385
|
acquisition: this._advisoryLockAcquisitions.get(lockName) ?? null,
|
|
4369
4386
|
lockServer: this._advisoryLockLockServerBackend?.getStatus() ?? null,
|
|
4370
|
-
policy: this.resolveAdvisoryLockPolicy(lockName)
|
|
4387
|
+
policy: this.resolveAdvisoryLockPolicy(lockName),
|
|
4388
|
+
reentries: this._advisoryLockReentries.get(lockName) ?? 0
|
|
4371
4389
|
}
|
|
4372
4390
|
}
|
|
4373
4391
|
|
|
@@ -59,7 +59,7 @@ export default class AdvisoryLockFileBackend {
|
|
|
59
59
|
this.pathStyle = pathStyle
|
|
60
60
|
this.pollIntervalMs = pollIntervalMs
|
|
61
61
|
this.holderId = `${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2, 10)}`
|
|
62
|
-
/** @type {Map<string, {timer: ReturnType<typeof setInterval>}>} */
|
|
62
|
+
/** @type {Map<string, {depth: number, timer: ReturnType<typeof setInterval>}>} */
|
|
63
63
|
this.holdsByName = new Map()
|
|
64
64
|
}
|
|
65
65
|
|
|
@@ -113,7 +113,17 @@ export default class AdvisoryLockFileBackend {
|
|
|
113
113
|
* @returns {Promise<boolean>} - True when this backend became the lock owner.
|
|
114
114
|
*/
|
|
115
115
|
async tryAcquire(name) {
|
|
116
|
-
|
|
116
|
+
// Already held by this backend instance: a re-entrant acquisition
|
|
117
|
+
// (`Monitor` semantics) adds a level instead of contending with itself.
|
|
118
|
+
// Refusing it made a nested hold time out on this tier while the same
|
|
119
|
+
// nesting worked on the database tiers.
|
|
120
|
+
const existingHold = this.holdsByName.get(name)
|
|
121
|
+
|
|
122
|
+
if (existingHold) {
|
|
123
|
+
existingHold.depth += 1
|
|
124
|
+
|
|
125
|
+
return true
|
|
126
|
+
}
|
|
117
127
|
|
|
118
128
|
await this.ensureDirectory()
|
|
119
129
|
|
|
@@ -141,6 +151,14 @@ export default class AdvisoryLockFileBackend {
|
|
|
141
151
|
|
|
142
152
|
if (!hold) return false
|
|
143
153
|
|
|
154
|
+
// One release per acquisition: a re-entrant hold keeps the lock until its
|
|
155
|
+
// outermost level releases, exactly like the database tiers.
|
|
156
|
+
if (hold.depth > 1) {
|
|
157
|
+
hold.depth -= 1
|
|
158
|
+
|
|
159
|
+
return true
|
|
160
|
+
}
|
|
161
|
+
|
|
144
162
|
this.clearHold(name)
|
|
145
163
|
|
|
146
164
|
const owner = await this.readOwner(this.lockPathFor(name))
|
|
@@ -236,7 +254,7 @@ export default class AdvisoryLockFileBackend {
|
|
|
236
254
|
|
|
237
255
|
if (typeof timer.unref === "function") timer.unref()
|
|
238
256
|
|
|
239
|
-
this.holdsByName.set(name, {timer})
|
|
257
|
+
this.holdsByName.set(name, {depth: 1, timer})
|
|
240
258
|
|
|
241
259
|
return true
|
|
242
260
|
}
|
|
@@ -15,7 +15,7 @@ import {AdvisoryLockServiceUnavailableError} from "./advisory-lock-errors.js"
|
|
|
15
15
|
* @property {(type: string, args: {onMessage: (body: Record<string, unknown>) => void, params: Record<string, unknown>, timeoutMs: number}) => LockServerConnectionLike} openConnection - Opens the 1:1 connection.
|
|
16
16
|
* @property {() => Promise<void> | void} [close] - Closes the transport.
|
|
17
17
|
* @typedef {{epoch: number, name: string, timer: ReturnType<typeof setInterval>, leaseExpiresAt: number}} LockServerHold
|
|
18
|
-
* @typedef {{status: "acquired", epoch: number, leaseExpiresAt: number} | {status: "busy"} | {status: "timeout"}} LockServerAcquireResult
|
|
18
|
+
* @typedef {{status: "acquired", epoch: number, leaseExpiresAt: number} | {status: "already-held", epoch: number, leaseExpiresAt: number} | {status: "busy"} | {status: "timeout"}} LockServerAcquireResult
|
|
19
19
|
*/
|
|
20
20
|
|
|
21
21
|
/** Protocol version spoken by the centralized lock server (consens-lock v2). */
|
|
@@ -181,6 +181,16 @@ export default class AdvisoryLockLockServerBackend {
|
|
|
181
181
|
return true
|
|
182
182
|
}
|
|
183
183
|
|
|
184
|
+
// `already-held` means the server says *this connection* already owns the
|
|
185
|
+
// lease: a re-entrant acquisition (`Monitor` semantics), not contention.
|
|
186
|
+
// Reporting it as "not acquired" is what turned every nested hold into a
|
|
187
|
+
// timeout once a name moved to this tier.
|
|
188
|
+
if (result.status === "already-held") {
|
|
189
|
+
this.startRenewal(name, result.epoch)
|
|
190
|
+
|
|
191
|
+
return true
|
|
192
|
+
}
|
|
193
|
+
|
|
184
194
|
if (result.status === "busy") return false
|
|
185
195
|
|
|
186
196
|
return false
|
|
@@ -197,7 +207,7 @@ export default class AdvisoryLockLockServerBackend {
|
|
|
197
207
|
/** @type {LockServerAcquireResult} */
|
|
198
208
|
const result = /** @type {LockServerAcquireResult} */ (await this.request("try-acquire", {name}))
|
|
199
209
|
|
|
200
|
-
if (result.status !== "acquired") return false
|
|
210
|
+
if (result.status !== "acquired" && result.status !== "already-held") return false
|
|
201
211
|
|
|
202
212
|
this.startRenewal(name, result.epoch)
|
|
203
213
|
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
// @ts-check
|
|
2
2
|
|
|
3
|
+
import {AsyncLocalStorage} from "node:async_hooks"
|
|
3
4
|
import timeout, {TimeoutError} from "awaitery/build/timeout.js"
|
|
4
5
|
import {DEFAULT_ADVISORY_LOCK_ACQUIRE_TIMEOUT_MS} from "./advisory-lock-bound.js"
|
|
5
6
|
import {
|
|
@@ -59,6 +60,27 @@ class AdvisoryLockBusyError extends Error {
|
|
|
59
60
|
}
|
|
60
61
|
}
|
|
61
62
|
|
|
63
|
+
/**
|
|
64
|
+
* Advisory locks held by one async context, keyed by
|
|
65
|
+
* `<databaseIdentifier>\u0000<lock name>` and carrying the re-entry depth plus
|
|
66
|
+
* the provenance of the tier that actually holds the lock.
|
|
67
|
+
*
|
|
68
|
+
* Re-entry is scoped to the async context, not to the process: a nested call
|
|
69
|
+
* made inside a held callback belongs to the same logical operation and must
|
|
70
|
+
* not contend with itself, while an unrelated concurrent request in the same
|
|
71
|
+
* process is a different context and must still wait. That is Ruby's `Monitor`
|
|
72
|
+
* ("re-entrant for the holding thread") expressed with the async-context
|
|
73
|
+
* analogue of a thread.
|
|
74
|
+
*
|
|
75
|
+
* Without this, the four tiers disagreed on re-entry: a nested acquisition on
|
|
76
|
+
* a `dedicatedConnection` opened a second connection and self-deadlocked, the
|
|
77
|
+
* file tier refused it as held, and the lock-server tier received
|
|
78
|
+
* `already-held` and treated it as contention. Applications worked around it
|
|
79
|
+
* with hand-passed `*LockHeld` booleans, which is exactly the parameter a
|
|
80
|
+
* monitor makes redundant.
|
|
81
|
+
*/
|
|
82
|
+
const heldAdvisoryLocks = new AsyncLocalStorage()
|
|
83
|
+
|
|
62
84
|
/**
|
|
63
85
|
* Thrown when a callback holds an advisory lock longer than `holdTimeoutMs`.
|
|
64
86
|
*/
|
|
@@ -107,6 +129,10 @@ export default class AdvisoryLockRunner {
|
|
|
107
129
|
* @returns {Promise<T>} - Resolves with the callback result.
|
|
108
130
|
*/
|
|
109
131
|
async withAdvisoryLock(name, callback, args = {}) {
|
|
132
|
+
const heldEntry = this.heldEntryFor(name)
|
|
133
|
+
|
|
134
|
+
if (heldEntry) return await this.runReentrantCallback({callback, entry: heldEntry, name})
|
|
135
|
+
|
|
110
136
|
const lockServerResult = await this.attemptLockServer({args, callback, mode: "wait", name})
|
|
111
137
|
|
|
112
138
|
// The wait mode never returns a deferred outcome, so the value member is present.
|
|
@@ -159,6 +185,10 @@ export default class AdvisoryLockRunner {
|
|
|
159
185
|
* @returns {Promise<T>} - Resolves with the callback result.
|
|
160
186
|
*/
|
|
161
187
|
async withAdvisoryLockOrFail(name, callback, args = {}) {
|
|
188
|
+
const heldEntry = this.heldEntryFor(name)
|
|
189
|
+
|
|
190
|
+
if (heldEntry) return await this.runReentrantCallback({callback, entry: heldEntry, name})
|
|
191
|
+
|
|
162
192
|
const lockServerResult = await this.attemptLockServer({args, callback, mode: "or-fail", name})
|
|
163
193
|
|
|
164
194
|
// The or-fail mode throws on contention, so the value member is present.
|
|
@@ -218,6 +248,10 @@ export default class AdvisoryLockRunner {
|
|
|
218
248
|
* @returns {Promise<{acquired: true, result: T} | {acquired: false, reason: "contended", lockName: string}>} - Either the held-callback result or the defer outcome.
|
|
219
249
|
*/
|
|
220
250
|
async withAdvisoryLockOrDefer(name, callback, args = {}) {
|
|
251
|
+
const heldEntry = this.heldEntryFor(name)
|
|
252
|
+
|
|
253
|
+
if (heldEntry) return {acquired: true, result: await this.runReentrantCallback({callback, entry: heldEntry, name})}
|
|
254
|
+
|
|
221
255
|
const lockServerResult = await this.attemptLockServer({args, callback, mode: "defer", name})
|
|
222
256
|
|
|
223
257
|
if (lockServerResult) {
|
|
@@ -266,13 +300,69 @@ export default class AdvisoryLockRunner {
|
|
|
266
300
|
* @returns {Promise<T>} - Resolves with the callback result.
|
|
267
301
|
*/
|
|
268
302
|
async runLockedCallback({callback, holdTimeoutMs, lockBackend, name, release}) {
|
|
303
|
+
const store = new Map(heldAdvisoryLocks.getStore() ?? [])
|
|
304
|
+
|
|
305
|
+
store.set(this.lockStorageKey(name), {depth: 1, lockBackend})
|
|
306
|
+
|
|
269
307
|
try {
|
|
270
|
-
|
|
308
|
+
// The lock is released outside the context scope: a release path that
|
|
309
|
+
// touches the same name (a cleanup hook, an error reporter) must not look
|
|
310
|
+
// like a re-entry of a hold that is already over.
|
|
311
|
+
return await heldAdvisoryLocks.run(store, async () => await AdvisoryLockRunner.runWithAdvisoryLockHoldTimeout(name, callback, holdTimeoutMs, lockBackend))
|
|
271
312
|
} finally {
|
|
272
313
|
await release()
|
|
273
314
|
}
|
|
274
315
|
}
|
|
275
316
|
|
|
317
|
+
/**
|
|
318
|
+
* Returns the held entry when this async context already holds the lock.
|
|
319
|
+
* @param {string} name - Lock name.
|
|
320
|
+
* @returns {{depth: number, lockBackend: AdvisoryLockBackendProvenance} | null} - The held entry, or null when this context does not hold the lock.
|
|
321
|
+
*/
|
|
322
|
+
heldEntryFor(name) {
|
|
323
|
+
const store = heldAdvisoryLocks.getStore()
|
|
324
|
+
|
|
325
|
+
if (!store) return null
|
|
326
|
+
|
|
327
|
+
return store.get(this.lockStorageKey(name)) ?? null
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
/**
|
|
331
|
+
* Runs a callback as a re-entry of a lock this async context already holds.
|
|
332
|
+
*
|
|
333
|
+
* The re-entry adds no backend round trip and cannot fail on contention: the
|
|
334
|
+
* outer hold already owns the lock, so mutual exclusion is unchanged. The
|
|
335
|
+
* outer hold owns the hold timeout and the lease; a re-entry does not start a
|
|
336
|
+
* second one, and the callback therefore receives the outer provenance without
|
|
337
|
+
* a `control`.
|
|
338
|
+
* @template T
|
|
339
|
+
* @param {{callback: (args?: {control?: import("awaitery/build/timeout.js").TimeoutControl, lockBackend: AdvisoryLockBackendProvenance}) => Promise<T>, entry: {depth: number, lockBackend: AdvisoryLockBackendProvenance}, name: string}} args - Re-entry request.
|
|
340
|
+
* @returns {Promise<T>} - Callback result.
|
|
341
|
+
*/
|
|
342
|
+
async runReentrantCallback({callback, entry, name}) {
|
|
343
|
+
entry.depth += 1
|
|
344
|
+
this.configuration.recordAdvisoryLockReentry({lockName: name})
|
|
345
|
+
|
|
346
|
+
try {
|
|
347
|
+
return await callback({lockBackend: entry.lockBackend})
|
|
348
|
+
} finally {
|
|
349
|
+
entry.depth -= 1
|
|
350
|
+
}
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
/**
|
|
354
|
+
* Builds the async-context storage key for one lock name.
|
|
355
|
+
*
|
|
356
|
+
* The database identifier is part of the key because it is part of the lock's
|
|
357
|
+
* identity: two tenant databases reusing one name hold two different locks,
|
|
358
|
+
* and a nested call in the other database is not a re-entry.
|
|
359
|
+
* @param {string} name - Lock name.
|
|
360
|
+
* @returns {string} - Storage key for the held-entry map.
|
|
361
|
+
*/
|
|
362
|
+
lockStorageKey(name) {
|
|
363
|
+
return `${this.databaseIdentifier}\u0000${name}`
|
|
364
|
+
}
|
|
365
|
+
|
|
276
366
|
/**
|
|
277
367
|
* Serves one acquisition on the lock-server tier when the lock name prefers
|
|
278
368
|
* it (design §1/§9), falling *upward* to the driver path when the server is
|
|
@@ -109,10 +109,11 @@ export default class VelociousBaseRoute {
|
|
|
109
109
|
* Runs namespace.
|
|
110
110
|
* @abstract
|
|
111
111
|
* @param {string} name - Name.
|
|
112
|
-
* @param {(arg: import("./namespace-route.js").default) => void}
|
|
112
|
+
* @param {object | ((arg: import("./namespace-route.js").default) => void)} options - Namespace options or the draw callback (legacy form).
|
|
113
|
+
* @param {((arg: import("./namespace-route.js").default) => void) | undefined} [callback] - Callback function.
|
|
113
114
|
* @returns {void} - No return value.
|
|
114
115
|
*/
|
|
115
|
-
namespace(name, callback) { throw new Error("'namespace' not implemented") } // eslint-disable-line no-unused-vars
|
|
116
|
+
namespace(name, options, callback) { throw new Error("'namespace' not implemented") } // eslint-disable-line no-unused-vars
|
|
116
117
|
|
|
117
118
|
/**
|
|
118
119
|
* Runs post.
|
|
@@ -6,10 +6,11 @@ export default class VelociousBasicRoute extends BaseRoute {
|
|
|
6
6
|
/**
|
|
7
7
|
* Runs get.
|
|
8
8
|
* @param {string} name - Route name.
|
|
9
|
+
* @param {{on?: "member" | "collection"} | {skipControllerConnections?: boolean}} [options] - Route options; `skipControllerConnections: true` skips the request-scoped controller connection for this route.
|
|
9
10
|
*/
|
|
10
|
-
get(name) {
|
|
11
|
+
get(name, options = {}) {
|
|
11
12
|
const GetRoute = VelociousBasicRoute.GetRouteType
|
|
12
|
-
const route = new GetRoute({name})
|
|
13
|
+
const route = new GetRoute({name, ...options})
|
|
13
14
|
|
|
14
15
|
this.routes.push(route)
|
|
15
16
|
}
|
|
@@ -51,35 +52,59 @@ export default class VelociousBasicRoute extends BaseRoute {
|
|
|
51
52
|
|
|
52
53
|
/**
|
|
53
54
|
* Runs namespace.
|
|
54
|
-
* @param {string} name -
|
|
55
|
-
* @param {(arg: import("./namespace-route.js").default) => void}
|
|
55
|
+
* @param {string} name - Route name.
|
|
56
|
+
* @param {{skipControllerConnections?: boolean} | ((arg: import("./namespace-route.js").default) => void)} [options] - Namespace options (`skipControllerConnections: true` skips the request-scoped controller connection for every route declared inside this namespace; a per-route option still overrides it) or the draw callback (legacy form).
|
|
57
|
+
* @param {((arg: import("./namespace-route.js").default) => void) | undefined} [callback] - Callback function.
|
|
56
58
|
* @returns {void} - No return value.
|
|
57
59
|
*/
|
|
58
|
-
namespace(name, callback) {
|
|
60
|
+
namespace(name, options = {}, callback) {
|
|
59
61
|
const NamespaceRoute = VelociousBasicRoute.NameSpaceRouteType
|
|
60
62
|
|
|
61
63
|
if (!NamespaceRoute) throw new Error("No NamespaceRoute registered")
|
|
62
64
|
|
|
65
|
+
// Backward compatible: the second argument may be the draw callback.
|
|
66
|
+
if (typeof options === "function") {
|
|
67
|
+
callback = options
|
|
68
|
+
options = {}
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
const namespaceOptions = /** @type {{skipControllerConnections?: boolean}} */ (options)
|
|
72
|
+
const skipControllerConnections = namespaceOptions.skipControllerConnections === true
|
|
73
|
+
|
|
63
74
|
const route = new NamespaceRoute({name})
|
|
64
75
|
|
|
65
76
|
this.routes.push(route)
|
|
66
77
|
|
|
67
78
|
if (callback) {
|
|
79
|
+
const originalGet = route.get.bind(route)
|
|
80
|
+
const originalPost = route.post.bind(route)
|
|
81
|
+
|
|
82
|
+
route.get = (routeName, routeOptions = {}) => {
|
|
83
|
+
const subOptions = /** @type {{skipControllerConnections?: boolean}} */ (routeOptions)
|
|
84
|
+
return originalGet(routeName, {...routeOptions, skipControllerConnections: subOptions.skipControllerConnections ?? skipControllerConnections})
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
route.post = (routeName, routeOptions = {}) => {
|
|
88
|
+
const subOptions = /** @type {{skipControllerConnections?: boolean}} */ (routeOptions)
|
|
89
|
+
return originalPost(routeName, {...routeOptions, skipControllerConnections: subOptions.skipControllerConnections ?? skipControllerConnections})
|
|
90
|
+
}
|
|
91
|
+
|
|
68
92
|
callback(route)
|
|
69
93
|
}
|
|
70
94
|
}
|
|
71
95
|
|
|
72
96
|
/**
|
|
73
97
|
* Runs post.
|
|
74
|
-
* @param {string} name -
|
|
98
|
+
* @param {string} name - Route name.
|
|
99
|
+
* @param {{on?: "member" | "collection"} | {skipControllerConnections?: boolean}} [options] - Route options; `skipControllerConnections: true` skips the request-scoped controller connection for this route.
|
|
75
100
|
* @returns {void} - No return value.
|
|
76
101
|
*/
|
|
77
|
-
post(name) {
|
|
102
|
+
post(name, options = {}) {
|
|
78
103
|
const PostRoute = VelociousBasicRoute.PostRouteType
|
|
79
104
|
|
|
80
105
|
if (!PostRoute) throw new Error("No PostRoute registered")
|
|
81
106
|
|
|
82
|
-
const route = new PostRoute({name})
|
|
107
|
+
const route = new PostRoute({name, ...options})
|
|
83
108
|
|
|
84
109
|
this.routes.push(route)
|
|
85
110
|
}
|
|
@@ -28,16 +28,32 @@ function assignActionAndController(params, name) {
|
|
|
28
28
|
}
|
|
29
29
|
|
|
30
30
|
class VelociousRouteGetRoute extends BaseRoute {
|
|
31
|
+
/**
|
|
32
|
+
* When true the resolver skips the request-scoped controller connection for
|
|
33
|
+
* this route: the action runs without an ambient `ensureConnections` checkout,
|
|
34
|
+
* so a long-lived action (streaming response, long wait) does not pin a pool
|
|
35
|
+
* connection for its whole lifetime. Each explicit `ensureConnections` call
|
|
36
|
+
* inside the action still opens and returns its own short-lived connection.
|
|
37
|
+
* @type {boolean}
|
|
38
|
+
*/
|
|
39
|
+
skipControllerConnections = false
|
|
40
|
+
|
|
31
41
|
/**
|
|
32
42
|
* Runs constructor.
|
|
33
43
|
* @param {object} args - Options object.
|
|
34
44
|
* @param {string} args.name - Name.
|
|
45
|
+
* @param {boolean} [args.skipControllerConnections] - Skip the request-scoped controller connection for this route.
|
|
35
46
|
*/
|
|
36
|
-
constructor({name, ...restArgs}) {
|
|
47
|
+
constructor({name, skipControllerConnections = false, ...restArgs}) {
|
|
37
48
|
super()
|
|
38
49
|
restArgsError(restArgs)
|
|
39
50
|
this.name = name
|
|
40
51
|
this.regExp = new RegExp(`^(${escapeStringRegexp(name)})(.*)$`)
|
|
52
|
+
if (skipControllerConnections !== true && skipControllerConnections !== false) {
|
|
53
|
+
throw new Error(`Expected get route 'skipControllerConnections' to be a boolean, got: ${String(skipControllerConnections)}`)
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
this.skipControllerConnections = skipControllerConnections
|
|
41
57
|
}
|
|
42
58
|
|
|
43
59
|
getHumanPaths() {
|
|
@@ -27,16 +27,32 @@ function assignActionAndController(params, name) {
|
|
|
27
27
|
}
|
|
28
28
|
|
|
29
29
|
class VelociousRoutePostRoute extends BaseRoute {
|
|
30
|
+
/**
|
|
31
|
+
* When true the resolver skips the request-scoped controller connection for
|
|
32
|
+
* this route: the action runs without an ambient `ensureConnections` checkout,
|
|
33
|
+
* so a long-lived action (streaming response, long wait) does not pin a pool
|
|
34
|
+
* connection for its whole lifetime. Each explicit `ensureConnections` call
|
|
35
|
+
* inside the action still opens and returns its own short-lived connection.
|
|
36
|
+
* @type {boolean}
|
|
37
|
+
*/
|
|
38
|
+
skipControllerConnections = false
|
|
39
|
+
|
|
30
40
|
/**
|
|
31
41
|
* Runs constructor.
|
|
32
42
|
* @param {object} args - Options object.
|
|
33
43
|
* @param {string} args.name - Name.
|
|
44
|
+
* @param {boolean} [args.skipControllerConnections] - Skip the request-scoped controller connection for this route.
|
|
34
45
|
*/
|
|
35
|
-
constructor({name, ...restArgs}) {
|
|
46
|
+
constructor({name, skipControllerConnections = false, ...restArgs}) {
|
|
36
47
|
super()
|
|
37
48
|
restArgsError(restArgs)
|
|
38
49
|
this.name = name
|
|
39
50
|
this.regExp = new RegExp(`^(${escapeStringRegexp(name)})(.*)$`)
|
|
51
|
+
if (skipControllerConnections !== true && skipControllerConnections !== false) {
|
|
52
|
+
throw new Error(`Expected post route 'skipControllerConnections' to be a boolean, got: ${String(skipControllerConnections)}`)
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
this.skipControllerConnections = skipControllerConnections
|
|
40
56
|
}
|
|
41
57
|
|
|
42
58
|
getHumanPaths() {
|