velocious 1.0.720 → 1.0.722
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/background-jobs/main.js +173 -4
- package/build/configuration-types.js +1 -1
- package/build/database/advisory-lock-reentrancy-node.js +9 -0
- package/build/database/advisory-lock-reentrancy.js +66 -0
- package/build/database/advisory-lock-runner.js +9 -12
- package/build/environment-handlers/node.js +1 -0
- package/build/routes/base-route.js +3 -2
- package/build/routes/basic-route.js +33 -8
- package/build/routes/get-route.js +17 -1
- package/build/routes/post-route.js +17 -1
- package/build/routes/resolver.js +5 -3
- package/build/src/background-jobs/main.d.ts +60 -0
- package/build/src/background-jobs/main.d.ts.map +1 -1
- package/build/src/background-jobs/main.js +164 -6
- package/build/src/configuration-types.d.ts +2 -2
- package/build/src/configuration-types.js +2 -2
- package/build/src/database/advisory-lock-reentrancy-node.d.ts +2 -0
- package/build/src/database/advisory-lock-reentrancy-node.d.ts.map +1 -0
- package/build/src/database/advisory-lock-reentrancy-node.js +7 -0
- package/build/src/database/advisory-lock-reentrancy.d.ts +67 -0
- package/build/src/database/advisory-lock-reentrancy.d.ts.map +1 -0
- package/build/src/database/advisory-lock-reentrancy.js +60 -0
- package/build/src/database/advisory-lock-runner.d.ts +25 -33
- package/build/src/database/advisory-lock-runner.d.ts.map +1 -1
- package/build/src/database/advisory-lock-runner.js +10 -13
- package/build/src/environment-handlers/node.d.ts +1 -0
- package/build/src/environment-handlers/node.d.ts.map +1 -1
- package/build/src/environment-handlers/node.js +2 -1
- package/build/src/routes/base-route.d.ts +3 -2
- package/build/src/routes/base-route.d.ts.map +1 -1
- package/build/src/routes/base-route.js +4 -3
- package/build/src/routes/basic-route.d.ts +19 -6
- package/build/src/routes/basic-route.d.ts.map +1 -1
- package/build/src/routes/basic-route.js +29 -9
- package/build/src/routes/get-route.d.ts +12 -1
- package/build/src/routes/get-route.d.ts.map +1 -1
- package/build/src/routes/get-route.js +16 -2
- package/build/src/routes/post-route.d.ts +12 -1
- package/build/src/routes/post-route.d.ts.map +1 -1
- package/build/src/routes/post-route.js +16 -2
- package/build/src/routes/resolver.d.ts +2 -1
- package/build/src/routes/resolver.d.ts.map +1 -1
- package/build/src/routes/resolver.js +6 -4
- package/build/src/sync/sync-cursor.d.ts +30 -0
- package/build/src/sync/sync-cursor.d.ts.map +1 -0
- package/build/src/sync/sync-cursor.js +54 -0
- package/build/src/sync/sync-realtime-bridge.d.ts +127 -24
- package/build/src/sync/sync-realtime-bridge.d.ts.map +1 -1
- package/build/src/sync/sync-realtime-bridge.js +92 -56
- package/build/src/sync/sync-websocket-channel.d.ts +49 -0
- package/build/src/sync/sync-websocket-channel.d.ts.map +1 -1
- package/build/src/sync/sync-websocket-channel.js +138 -2
- package/build/sync/sync-cursor.js +56 -0
- package/build/sync/sync-realtime-bridge.js +97 -51
- package/build/sync/sync-websocket-channel.js +153 -1
- package/package.json +1 -1
- package/src/background-jobs/main.js +173 -4
- package/src/configuration-types.js +1 -1
- package/src/database/advisory-lock-reentrancy-node.js +9 -0
- package/src/database/advisory-lock-reentrancy.js +66 -0
- package/src/database/advisory-lock-runner.js +9 -12
- package/src/environment-handlers/node.js +1 -0
- package/src/routes/base-route.js +3 -2
- package/src/routes/basic-route.js +33 -8
- package/src/routes/get-route.js +17 -1
- package/src/routes/post-route.js +17 -1
- package/src/routes/resolver.js +5 -3
- package/src/sync/sync-cursor.js +56 -0
- package/src/sync/sync-realtime-bridge.js +97 -51
- package/src/sync/sync-websocket-channel.js +153 -1
|
@@ -2,10 +2,53 @@
|
|
|
2
2
|
|
|
3
3
|
import VelociousWebsocketChannel from "../http-server/websocket-channel.js"
|
|
4
4
|
import {VELOCIOUS_SYNC_CHANNEL} from "./sync-channel-name.js"
|
|
5
|
+
import {cursorFromSyncEntry, isCursorAfter} from "./sync-cursor.js"
|
|
6
|
+
import SyncModelChangeFeedService from "./sync-model-change-feed-service.js"
|
|
5
7
|
|
|
6
8
|
/** Configurations whose framework sync channel has already been registered. */
|
|
7
9
|
const registeredConfigurations = new WeakSet()
|
|
8
10
|
|
|
11
|
+
/** Default catch-up replay page size when the subscriber sends no `limit` param. */
|
|
12
|
+
const DEFAULT_REPLAY_LIMIT = 1000
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* Serializes a cursor into the change-feed request params under the given prefix.
|
|
16
|
+
* @param {"after" | "upTo"} prefix - Request field prefix.
|
|
17
|
+
* @param {import("./sync-api-client-types.js").SyncCursor} cursor - Cursor to serialize.
|
|
18
|
+
* @returns {Record<string, string | number | null>} Serialized cursor params.
|
|
19
|
+
*/
|
|
20
|
+
function cursorParams(prefix, cursor) {
|
|
21
|
+
if (!cursor) return {}
|
|
22
|
+
|
|
23
|
+
return {
|
|
24
|
+
[`${prefix}Id`]: cursor.id,
|
|
25
|
+
...(cursor.serverSequence !== null ? {[`${prefix}ServerSequence`]: cursor.serverSequence} : {}),
|
|
26
|
+
[`${prefix}UpdatedAt`]: cursor.updatedAt
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Reads the after-cursor the subscriber declared in its subscribe params, or null
|
|
32
|
+
* when it sent none (a fresh subscription replays the whole scope).
|
|
33
|
+
* @param {Record<string, ReturnType<typeof JSON.parse>>} params - Subscribe params.
|
|
34
|
+
* @returns {import("./sync-api-client-types.js").SyncCursor} Declared after-cursor, or null.
|
|
35
|
+
*/
|
|
36
|
+
function cursorFromParams(params) {
|
|
37
|
+
const updatedAt = typeof params.afterUpdatedAt === "string" ? params.afterUpdatedAt : null
|
|
38
|
+
|
|
39
|
+
if (!updatedAt) return null
|
|
40
|
+
|
|
41
|
+
const serverSequence = params.afterServerSequence === null || params.afterServerSequence === undefined
|
|
42
|
+
? null
|
|
43
|
+
: Number(params.afterServerSequence)
|
|
44
|
+
|
|
45
|
+
return {
|
|
46
|
+
id: params.afterId === null || params.afterId === undefined ? null : String(params.afterId),
|
|
47
|
+
serverSequence,
|
|
48
|
+
updatedAt
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
|
|
9
52
|
/**
|
|
10
53
|
* Framework-owned websocket channel for synced resources
|
|
11
54
|
* ({@link VELOCIOUS_SYNC_CHANNEL}).
|
|
@@ -25,6 +68,14 @@ export default class SyncWebsocketChannel extends VelociousWebsocketChannel {
|
|
|
25
68
|
*/
|
|
26
69
|
_scope = null
|
|
27
70
|
|
|
71
|
+
/**
|
|
72
|
+
* Cursor of the last sync entry delivered to this subscription, advanced by
|
|
73
|
+
* live broadcasts and catch-up replay so a resume replays only the rows missed
|
|
74
|
+
* while the socket was down.
|
|
75
|
+
* @type {import("./sync-api-client-types.js").SyncCursor}
|
|
76
|
+
*/
|
|
77
|
+
_deliveredCursor = null
|
|
78
|
+
|
|
28
79
|
/**
|
|
29
80
|
* Registers the framework sync channel on a configuration declaring a
|
|
30
81
|
* `sync.api` block (guarded so repeated server boots with the same
|
|
@@ -169,13 +220,17 @@ export default class SyncWebsocketChannel extends VelociousWebsocketChannel {
|
|
|
169
220
|
async deliverBroadcast(body, meta) {
|
|
170
221
|
if (!this._isUserScope()) {
|
|
171
222
|
this.sendMessage(body, meta)
|
|
223
|
+
this._trackDeliveredCursor(body)
|
|
172
224
|
|
|
173
225
|
return
|
|
174
226
|
}
|
|
175
227
|
|
|
176
228
|
const deliverableBody = await this._userScopeDeliverableBody(body)
|
|
177
229
|
|
|
178
|
-
if (deliverableBody !== null)
|
|
230
|
+
if (deliverableBody !== null) {
|
|
231
|
+
this.sendMessage(deliverableBody, meta)
|
|
232
|
+
this._trackDeliveredCursor(deliverableBody)
|
|
233
|
+
}
|
|
179
234
|
}
|
|
180
235
|
|
|
181
236
|
/**
|
|
@@ -245,6 +300,103 @@ export default class SyncWebsocketChannel extends VelociousWebsocketChannel {
|
|
|
245
300
|
return deliverableSyncs[0]
|
|
246
301
|
}
|
|
247
302
|
|
|
303
|
+
/**
|
|
304
|
+
* Replays the catch-up backlog to this subscriber once its subscription is
|
|
305
|
+
* authorized: pages the scoped change feed from the cursor the client declared
|
|
306
|
+
* in its subscribe params and sends each page as a standard sync envelope, then
|
|
307
|
+
* live broadcasts continue from there. A fresh subscription with no cursor
|
|
308
|
+
* replays the whole authorized scope.
|
|
309
|
+
* @returns {Promise<void>}
|
|
310
|
+
*/
|
|
311
|
+
async subscribed() {
|
|
312
|
+
if (!this._scope) return
|
|
313
|
+
|
|
314
|
+
await this._replayFrom(cursorFromParams(this.params))
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
/**
|
|
318
|
+
* Replays rows missed while the socket was down, resuming from the cursor of
|
|
319
|
+
* the last entry this subscription delivered instead of re-reading the whole
|
|
320
|
+
* scope from the subscribe-time cursor.
|
|
321
|
+
* @returns {Promise<void>}
|
|
322
|
+
*/
|
|
323
|
+
async onResume() {
|
|
324
|
+
if (!this._scope) return
|
|
325
|
+
|
|
326
|
+
await this._replayFrom(this._deliveredCursor ?? cursorFromParams(this.params))
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
/**
|
|
330
|
+
* Pages the scoped change feed from a cursor and delivers each page to this
|
|
331
|
+
* subscriber as a `{echoOrigin, syncs}` envelope. The snapshot high-water is
|
|
332
|
+
* resolved once and pinned across pages so rows arriving mid-replay are left to
|
|
333
|
+
* the live broadcast path instead of looping the replay. The delivered cursor
|
|
334
|
+
* advances so a later resume resumes from here.
|
|
335
|
+
* @param {import("./sync-api-client-types.js").SyncCursor} afterCursor - Cursor to replay after.
|
|
336
|
+
* @returns {Promise<void>}
|
|
337
|
+
*/
|
|
338
|
+
async _replayFrom(afterCursor) {
|
|
339
|
+
const configuration = this.session.configuration
|
|
340
|
+
const scope = this._scope
|
|
341
|
+
|
|
342
|
+
await configuration.ensureConnections({name: `${VELOCIOUS_SYNC_CHANNEL} catch-up replay`}, async () => {
|
|
343
|
+
const resource = await this.buildSyncResource()
|
|
344
|
+
const limit = this._replayLimit(resource)
|
|
345
|
+
/** @type {import("./sync-api-client-types.js").SyncCursor} */
|
|
346
|
+
let upToCursor = null
|
|
347
|
+
|
|
348
|
+
while (true) {
|
|
349
|
+
const page = await resource.changeFeedService({
|
|
350
|
+
params: {...this.params, ...cursorParams("after", afterCursor), ...(upToCursor ? cursorParams("upTo", upToCursor) : {})},
|
|
351
|
+
scope
|
|
352
|
+
}).changes()
|
|
353
|
+
|
|
354
|
+
if (!upToCursor) upToCursor = page.upToCursor
|
|
355
|
+
|
|
356
|
+
if (page.syncs.length === 0) break
|
|
357
|
+
|
|
358
|
+
await this.deliverBroadcast({echoOrigin: null, syncs: page.syncs})
|
|
359
|
+
|
|
360
|
+
afterCursor = page.nextCursor
|
|
361
|
+
if (afterCursor) this._deliveredCursor = afterCursor
|
|
362
|
+
|
|
363
|
+
if (page.syncs.length < limit) break
|
|
364
|
+
}
|
|
365
|
+
})
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
/**
|
|
369
|
+
* Resolves the catch-up replay page size from the subscriber's `limit` param
|
|
370
|
+
* (defaulting to {@link DEFAULT_REPLAY_LIMIT}).
|
|
371
|
+
* @param {import("./sync-resource-base.js").default} resource - Authorizing sync resource.
|
|
372
|
+
* @returns {number} Replay page size.
|
|
373
|
+
*/
|
|
374
|
+
_replayLimit(resource) {
|
|
375
|
+
return new SyncModelChangeFeedService({modelClass: resource.syncModelClass(), params: this.params}).normalizedLimit(this.params.limit)
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
/**
|
|
379
|
+
* Advances the delivered-cursor watermark from a delivered envelope so a resume
|
|
380
|
+
* resumes after the newest row actually sent to this subscriber. Broadcasts may
|
|
381
|
+
* arrive out of order across resources, so each entry only advances the
|
|
382
|
+
* watermark when it is later than the current one.
|
|
383
|
+
* @param {import("../http-server/websocket-channel.js").WebsocketJsonValue} body - Delivered envelope.
|
|
384
|
+
* @returns {void}
|
|
385
|
+
*/
|
|
386
|
+
_trackDeliveredCursor(body) {
|
|
387
|
+
if (!body || typeof body !== "object" || Array.isArray(body)) return
|
|
388
|
+
|
|
389
|
+
const envelope = /** @type {Record<string, ReturnType<typeof JSON.parse>>} */ (body)
|
|
390
|
+
const syncs = Array.isArray(envelope.syncs) ? envelope.syncs : [envelope]
|
|
391
|
+
|
|
392
|
+
for (const entry of syncs) {
|
|
393
|
+
const syncEntry = /** @type {Record<string, ReturnType<typeof JSON.parse>>} */ (entry)
|
|
394
|
+
const cursor = cursorFromSyncEntry(syncEntry)
|
|
395
|
+
|
|
396
|
+
if (cursor && isCursorAfter(cursor, this._deliveredCursor)) this._deliveredCursor = cursor
|
|
397
|
+
}
|
|
398
|
+
}
|
|
399
|
+
|
|
248
400
|
/**
|
|
249
401
|
* Returns the authorized scope for debug snapshots.
|
|
250
402
|
* @returns {Record<string, ReturnType<typeof JSON.parse>>} Debug-safe subscription details.
|
package/package.json
CHANGED
|
@@ -9,7 +9,7 @@ import Logger from "../logger.js"
|
|
|
9
9
|
import PruneTerminalBackgroundJobsJob from "../jobs/prune-terminal-background-jobs.js"
|
|
10
10
|
import VelociousError from "../velocious-error.js"
|
|
11
11
|
import shutdownLifecycle, { runShutdownSteps } from "../utils/shutdown-lifecycle.js"
|
|
12
|
-
import { validateGenerationId, workerIdBelongsToGeneration } from "./generation-identity.js"
|
|
12
|
+
import { parseGenerationWorkerId, validateGenerationId, workerIdBelongsToGeneration } from "./generation-identity.js"
|
|
13
13
|
import BackgroundJobsLifecycleControlServer from "./lifecycle-control-server.js"
|
|
14
14
|
|
|
15
15
|
/**
|
|
@@ -40,6 +40,8 @@ const WORKER_LIVENESS_SWEEP_MS = 15000
|
|
|
40
40
|
/** Grace for workers from the previous main generation to reconnect and adopt leases. */
|
|
41
41
|
const WORKER_RECONNECT_GRACE_MS = 30000
|
|
42
42
|
const GENERATION_ORPHANED_AFTER_MS = 60 * 60 * 1000
|
|
43
|
+
/** Prefix of the per-generation liveness advisory lock one live main holds for its whole generation. */
|
|
44
|
+
const GENERATION_LIVENESS_LOCK_PREFIX = "background-jobs:generation-liveness"
|
|
43
45
|
const WORKER_RECONNECT_GRACE_VALIDATION_MESSAGE = `workerReconnectGraceMs must be an integer between 0 and ${MAX_TIMER_MS}`
|
|
44
46
|
|
|
45
47
|
/**
|
|
@@ -246,6 +248,11 @@ export default class BackgroundJobsMain {
|
|
|
246
248
|
this._beaconClient = undefined
|
|
247
249
|
/** @type {BackgroundJobsLifecycleControlServer | undefined} */
|
|
248
250
|
this.lifecycleControlServer = undefined
|
|
251
|
+
/**
|
|
252
|
+
* Dedicated advisory-lock connection holding this main's generation liveness
|
|
253
|
+
* lock for the whole lifetime of the main (generation mode only).
|
|
254
|
+
* @type {import("../database/drivers/base.js").default | undefined} */
|
|
255
|
+
this._livenessLockConnection = undefined
|
|
249
256
|
}
|
|
250
257
|
|
|
251
258
|
/**
|
|
@@ -296,6 +303,7 @@ export default class BackgroundJobsMain {
|
|
|
296
303
|
if (this.generationId && !this.adapter.supportsOwnedEnqueueFromHandoff()) {
|
|
297
304
|
throw new Error("The configured background jobs adapter does not support atomic owned-handoff enqueue")
|
|
298
305
|
}
|
|
306
|
+
if (this.generationId) await this._acquireGenerationLivenessLock()
|
|
299
307
|
|
|
300
308
|
if (!this.generationId || this.initialGenerationState !== "candidate") {
|
|
301
309
|
this.startupHandoffSnapshot = await this._generationOwnedHandoffSnapshot()
|
|
@@ -389,7 +397,11 @@ export default class BackgroundJobsMain {
|
|
|
389
397
|
try {
|
|
390
398
|
await this._drainStartupHandoffReclaim()
|
|
391
399
|
} finally {
|
|
392
|
-
|
|
400
|
+
try {
|
|
401
|
+
await this._releaseGenerationLivenessLock()
|
|
402
|
+
} finally {
|
|
403
|
+
await this._stopBeaconAndServer()
|
|
404
|
+
}
|
|
393
405
|
}
|
|
394
406
|
}
|
|
395
407
|
}
|
|
@@ -624,6 +636,7 @@ export default class BackgroundJobsMain {
|
|
|
624
636
|
}
|
|
625
637
|
this.lifecycleState = "active"
|
|
626
638
|
this._creditReadyWorkers()
|
|
639
|
+
await this._settleForeignGenerationHandoffs()
|
|
627
640
|
this.logger.info(() => ["Background jobs generation activation acknowledged", {generationId: this.generationId}])
|
|
628
641
|
void this._drain().catch((error) => {
|
|
629
642
|
this.logger.error(() => ["Background jobs generation post-activation drain failed", {error, generationId: this.generationId}])
|
|
@@ -751,6 +764,154 @@ export default class BackgroundJobsMain {
|
|
|
751
764
|
* retained solely to perform this cleanup.
|
|
752
765
|
* @returns {void}
|
|
753
766
|
*/
|
|
767
|
+
/**
|
|
768
|
+
* Builds the per-generation liveness advisory-lock name for one generation on
|
|
769
|
+
* this main's background-jobs database. A live main holds this lock for its
|
|
770
|
+
* whole lifetime; a foreign generation whose lock is free (or stale) has no
|
|
771
|
+
* live main supervising its durable handoffs.
|
|
772
|
+
* @param {string} generationId - Generation to name.
|
|
773
|
+
* @returns {string} - Lock name.
|
|
774
|
+
*/
|
|
775
|
+
_generationLivenessLockName(generationId) {
|
|
776
|
+
const databaseIdentifier = this.configuration.getBackgroundJobsConfig().databaseIdentifier || "default"
|
|
777
|
+
|
|
778
|
+
return `${GENERATION_LIVENESS_LOCK_PREFIX}:${databaseIdentifier}:${generationId}`
|
|
779
|
+
}
|
|
780
|
+
|
|
781
|
+
/**
|
|
782
|
+
* Acquires this main's own generation liveness lock on a dedicated connection
|
|
783
|
+
* it holds until stop. Contention means a still-live main of the same
|
|
784
|
+
* generation exists (a deploy anomaly); the main keeps running without the
|
|
785
|
+
* marker and cross-generation settlement conservatively leaves its rows to it.
|
|
786
|
+
* @returns {Promise<void>} - Resolves after the marker is held or degraded.
|
|
787
|
+
*/
|
|
788
|
+
async _acquireGenerationLivenessLock() {
|
|
789
|
+
if (!this.generationId || this._livenessLockConnection) return
|
|
790
|
+
const lockName = this._generationLivenessLockName(this.generationId)
|
|
791
|
+
|
|
792
|
+
try {
|
|
793
|
+
const connection = await this.configuration.getDatabasePool(this._livenessLockDatabaseIdentifier()).spawnConnection()
|
|
794
|
+
const acquired = await connection.tryAcquireAdvisoryLock(lockName)
|
|
795
|
+
|
|
796
|
+
if (!acquired) {
|
|
797
|
+
await connection.close()
|
|
798
|
+
this.logger.warn(() => ["Background jobs generation liveness lock is already held by another live main of this generation", {generationId: this.generationId}])
|
|
799
|
+
|
|
800
|
+
return
|
|
801
|
+
}
|
|
802
|
+
|
|
803
|
+
this.configuration.registerAdvisoryLockConnection(connection)
|
|
804
|
+
this._livenessLockConnection = connection
|
|
805
|
+
} catch (error) {
|
|
806
|
+
this.logger.warn(() => ["Failed to acquire background jobs generation liveness lock", {error, generationId: this.generationId}])
|
|
807
|
+
}
|
|
808
|
+
}
|
|
809
|
+
|
|
810
|
+
/**
|
|
811
|
+
* Releases this main's generation liveness marker during stop so a later
|
|
812
|
+
* generation activation can prove this generation is gone. Closing the
|
|
813
|
+
* dedicated connection releases the advisory lock on every tier.
|
|
814
|
+
* @returns {Promise<void>} - Resolves after the marker is released.
|
|
815
|
+
*/
|
|
816
|
+
async _releaseGenerationLivenessLock() {
|
|
817
|
+
const connection = this._livenessLockConnection
|
|
818
|
+
|
|
819
|
+
this._livenessLockConnection = undefined
|
|
820
|
+
if (!connection) return
|
|
821
|
+
|
|
822
|
+
this.configuration.unregisterAdvisoryLockConnection(connection)
|
|
823
|
+
|
|
824
|
+
try {
|
|
825
|
+
await connection.releaseHeldAdvisoryLocks()
|
|
826
|
+
} finally {
|
|
827
|
+
await connection.close()
|
|
828
|
+
}
|
|
829
|
+
}
|
|
830
|
+
|
|
831
|
+
/**
|
|
832
|
+
* Database identifier whose pools carry the generation liveness locks. This
|
|
833
|
+
* matches the built-in store's own resolution of the background-jobs config.
|
|
834
|
+
* @returns {string} - Database identifier.
|
|
835
|
+
*/
|
|
836
|
+
_livenessLockDatabaseIdentifier() {
|
|
837
|
+
return this.configuration.getBackgroundJobsConfig().databaseIdentifier || "default"
|
|
838
|
+
}
|
|
839
|
+
|
|
840
|
+
/**
|
|
841
|
+
* One activation event: snapshot all durable handoffs, group the rows whose
|
|
842
|
+
* worker identity belongs to a foreign generation, and settle each group only
|
|
843
|
+
* after proving that generation has no live main (its liveness lock is free).
|
|
844
|
+
* Own-generation rows, identity-free legacy rows, and rows of live foreign
|
|
845
|
+
* generations stay owned by their existing recovery paths. Failures are
|
|
846
|
+
* reported as framework errors and never fail activation.
|
|
847
|
+
* @returns {Promise<void>} - Resolves after settlement or its reported failure.
|
|
848
|
+
*/
|
|
849
|
+
async _settleForeignGenerationHandoffs() {
|
|
850
|
+
if (!this.generationId) return
|
|
851
|
+
|
|
852
|
+
try {
|
|
853
|
+
const handoffs = await this.store.snapshotHandedOffJobs()
|
|
854
|
+
/** @type {Map<string, import("./types.js").BackgroundJobHandoffSnapshot[]>} */
|
|
855
|
+
const handoffsByForeignGeneration = new Map()
|
|
856
|
+
|
|
857
|
+
for (const handoff of handoffs) {
|
|
858
|
+
const parsed = parseGenerationWorkerId(handoff.workerId)
|
|
859
|
+
if (!parsed || parsed.generationId === this.generationId) continue
|
|
860
|
+
|
|
861
|
+
const group = handoffsByForeignGeneration.get(parsed.generationId) || []
|
|
862
|
+
group.push(handoff)
|
|
863
|
+
handoffsByForeignGeneration.set(parsed.generationId, group)
|
|
864
|
+
}
|
|
865
|
+
|
|
866
|
+
for (const [foreignGenerationId, foreignHandoffs] of handoffsByForeignGeneration) {
|
|
867
|
+
const settledJobs = await this._settleForeignGenerationHandoffGroup({foreignGenerationId, handoffs: foreignHandoffs})
|
|
868
|
+
|
|
869
|
+
if (settledJobs.length > 0) await this._handleOrphanedJobs({jobs: settledJobs, warning: "Marked orphaned background jobs whose generation was retired before this generation activated"})
|
|
870
|
+
}
|
|
871
|
+
} catch (error) {
|
|
872
|
+
const normalizedError = error instanceof Error ? error : new Error(String(error))
|
|
873
|
+
const payload = {context: {generationId: this.generationId, stage: "background-job-activation-cross-generation-settlement"}, error: normalizedError}
|
|
874
|
+
const errorEvents = this.configuration.getErrorEvents()
|
|
875
|
+
|
|
876
|
+
this.logger.error(() => ["Failed to settle foreign-generation handoffs at activation:", normalizedError])
|
|
877
|
+
errorEvents.emit("framework-error", payload)
|
|
878
|
+
errorEvents.emit("all-error", {...payload, errorType: "framework-error"})
|
|
879
|
+
}
|
|
880
|
+
}
|
|
881
|
+
|
|
882
|
+
/**
|
|
883
|
+
* Settles one foreign generation's exact handoffs under a proof of absence:
|
|
884
|
+
* acquire that generation's liveness lock on a dedicated connection and hold
|
|
885
|
+
* it across the fenced store transition. A busy lock means a live main of
|
|
886
|
+
* that generation still supervises its rows (retired-and-draining included),
|
|
887
|
+
* so nothing here may touch them.
|
|
888
|
+
* @param {object} args - Options.
|
|
889
|
+
* @param {string} args.foreignGenerationId - Generation whose rows are candidates.
|
|
890
|
+
* @param {import("./types.js").BackgroundJobHandoffSnapshot[]} args.handoffs - Exact snapshot rows.
|
|
891
|
+
* @returns {Promise<import("./types.js").BackgroundJobRow[]>} - Accepted orphan transitions.
|
|
892
|
+
*/
|
|
893
|
+
async _settleForeignGenerationHandoffGroup({foreignGenerationId, handoffs}) {
|
|
894
|
+
const lockName = this._generationLivenessLockName(foreignGenerationId)
|
|
895
|
+
const connection = await this.configuration.getDatabasePool(this._livenessLockDatabaseIdentifier()).spawnConnection()
|
|
896
|
+
|
|
897
|
+
try {
|
|
898
|
+
const acquired = await connection.tryAcquireAdvisoryLock(lockName)
|
|
899
|
+
|
|
900
|
+
if (!acquired) return []
|
|
901
|
+
|
|
902
|
+
return await this.store.markOrphanedHandoffs({
|
|
903
|
+
error: "Job orphaned when a new generation activated while its owning generation was not live",
|
|
904
|
+
handoffs
|
|
905
|
+
})
|
|
906
|
+
} finally {
|
|
907
|
+
try {
|
|
908
|
+
await connection.releaseHeldAdvisoryLocks()
|
|
909
|
+
} finally {
|
|
910
|
+
await connection.close()
|
|
911
|
+
}
|
|
912
|
+
}
|
|
913
|
+
}
|
|
914
|
+
|
|
754
915
|
_setupStartupHandoffReclaim() {
|
|
755
916
|
if (this.startupHandoffSnapshot.length === 0) return
|
|
756
917
|
if (this._startupHandoffReclaimTimer || this._startupHandoffReclaimPromise || this._startupHandoffGraceElapsed) return
|
|
@@ -1156,12 +1317,20 @@ export default class BackgroundJobsMain {
|
|
|
1156
1317
|
*/
|
|
1157
1318
|
async _handleClientSocketMessage({jsonSocket, message}) {
|
|
1158
1319
|
if (this.generationId && (this.lifecycleState === "retiring" || this.lifecycleState === "retired")) {
|
|
1159
|
-
|
|
1320
|
+
// A retiring generation stays a durable enqueue target. Queued rows are
|
|
1321
|
+
// generation-neutral in the shared store and a retiring main never
|
|
1322
|
+
// dispatches them (`_handleEnqueue` only drains while active), so accepted
|
|
1323
|
+
// work stays queued for the active generation - the release lifecycle
|
|
1324
|
+
// contract holds that returned/retried queue work is eligible for the
|
|
1325
|
+
// active generation. Refusing these enqueues strands producers with no
|
|
1326
|
+
// successor endpoint to retry against, such as this main's own recovery
|
|
1327
|
+
// listeners. Schedule ownership is not generation-neutral, so those
|
|
1328
|
+
// operations stay fenced here.
|
|
1160
1329
|
if (message?.type === "replace-scheduled") jsonSocket.send({type: "replace-scheduled-error", error: "Background jobs generation is retired"})
|
|
1161
1330
|
if (message?.type === "cancel-scheduled") jsonSocket.send({type: "cancel-scheduled-error", error: "Background jobs generation is retired"})
|
|
1162
1331
|
if (message?.type === "get-scheduled-job") jsonSocket.send({type: "get-scheduled-job-error", error: "Background jobs generation is retired"})
|
|
1163
1332
|
if (message?.type === "wake-scheduled") jsonSocket.send({type: "wake-scheduled-error", error: "Background jobs generation is retired"})
|
|
1164
|
-
if (message?.type !== "enqueue"
|
|
1333
|
+
if (message?.type !== "enqueue") return
|
|
1165
1334
|
}
|
|
1166
1335
|
|
|
1167
1336
|
if (message?.type === "enqueue") {
|
|
@@ -602,7 +602,7 @@
|
|
|
602
602
|
* @property {() => VelociousSyncRealtimeWebsocketClient | Promise<VelociousSyncRealtimeWebsocketClient>} createClient - Builds the (unconnected) websocket client; the framework owns connect/disconnect.
|
|
603
603
|
* @property {(context: ReturnType<typeof JSON.parse>) => Array<VelociousSyncRealtimeChannelDescriptor> | Promise<Array<VelociousSyncRealtimeChannelDescriptor>>} [channels] - Deprecated legacy escape hatch: resolves extra app-channel descriptors from the `subscribeRealtime(context)` context. Declared pull scopes subscribe the framework sync channel automatically.
|
|
604
604
|
* @property {() => string | Promise<string>} [localOrigin] - Resolves this device's echo origin; pushed messages with a matching `echoOrigin` are dropped.
|
|
605
|
-
* @property {boolean} [pullOnReconnect] -
|
|
605
|
+
* @property {boolean} [pullOnReconnect] - Deprecated no-op: realtime catch-up now replays the server-side sync cursor on subscribe and resume, so the former pull-on-reconnect no longer fires. Kept so existing app configs still typecheck.
|
|
606
606
|
*/
|
|
607
607
|
|
|
608
608
|
/**
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
// @ts-check
|
|
2
|
+
|
|
3
|
+
import {AsyncLocalStorage} from "node:async_hooks"
|
|
4
|
+
|
|
5
|
+
import {installAdvisoryLockReentrancyStorage} from "./advisory-lock-reentrancy.js"
|
|
6
|
+
|
|
7
|
+
// Imported for its side effect by the Node environment handler: only Node has
|
|
8
|
+
// async-context storage, and browser/Expo bundles must never reach this module.
|
|
9
|
+
installAdvisoryLockReentrancyStorage(new AsyncLocalStorage())
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
// @ts-check
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Backend provenance of one advisory-lock hold (STC-210 design §3).
|
|
5
|
+
* @typedef {object} AdvisoryLockBackendProvenance
|
|
6
|
+
* @property {import("./advisory-lock-backend-policy.js").AdvisoryLockTier} backend - Tier that served the acquisition.
|
|
7
|
+
* @property {string} [driverType] - Database driver type when the driver tier served it.
|
|
8
|
+
* @property {number} [epoch] - Fencing epoch when the serving tier issues one.
|
|
9
|
+
* @property {boolean} fencingSupported - Whether this tier carries fencing tokens (design §8); the file tier reports false rather than pretending.
|
|
10
|
+
* @property {string} [holderId] - Backend holder id for the file tier.
|
|
11
|
+
* @property {string} [lockName] - Lock name this provenance belongs to.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* One advisory-lock hold tracked in the current async context.
|
|
16
|
+
* @typedef {object} HeldAdvisoryLock
|
|
17
|
+
* @property {number} depth - Nesting depth of the hold.
|
|
18
|
+
* @property {AdvisoryLockBackendProvenance} lockBackend - Backend that served the outer hold.
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* Async-context storage contract for advisory-lock re-entrancy.
|
|
23
|
+
* @typedef {object} AdvisoryLockReentrancyStorage
|
|
24
|
+
* @property {() => Map<string, HeldAdvisoryLock> | undefined} getStore - Store for the current async context.
|
|
25
|
+
* @property {(store: Map<string, HeldAdvisoryLock>, callback: () => ReturnType<typeof JSON.parse>) => ReturnType<typeof JSON.parse>} run - Runs the callback with the store installed.
|
|
26
|
+
*/
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* This module is imported by browser and Expo bundles, so it must stay free of
|
|
30
|
+
* Node-only imports. The Node handler installs the real AsyncLocalStorage through
|
|
31
|
+
* `advisory-lock-reentrancy-node.js`; other runtimes leave it uninstalled and
|
|
32
|
+
* holds then run without re-entry tracking.
|
|
33
|
+
* @type {AdvisoryLockReentrancyStorage | undefined}
|
|
34
|
+
*/
|
|
35
|
+
let installedStorage
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Installs the runtime's async-context storage. Called once by the Node handler.
|
|
39
|
+
* @param {AdvisoryLockReentrancyStorage} storage - Storage owned by the runtime.
|
|
40
|
+
* @returns {void}
|
|
41
|
+
*/
|
|
42
|
+
export function installAdvisoryLockReentrancyStorage(storage) {
|
|
43
|
+
installedStorage = storage
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* Returns the locks the current async context holds, when this runtime has
|
|
48
|
+
* async-context storage.
|
|
49
|
+
* @returns {Map<string, HeldAdvisoryLock> | undefined} - Current context store.
|
|
50
|
+
*/
|
|
51
|
+
export function advisoryLockReentrancyStore() {
|
|
52
|
+
return installedStorage?.getStore()
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* Runs a callback with the store installed as the current context. Runtimes
|
|
57
|
+
* without async-context storage run the callback directly.
|
|
58
|
+
* @param {Map<string, HeldAdvisoryLock>} store - Store for the hold being executed.
|
|
59
|
+
* @param {() => ReturnType<typeof JSON.parse>} callback - Hold work.
|
|
60
|
+
* @returns {ReturnType<typeof JSON.parse>} - Callback result.
|
|
61
|
+
*/
|
|
62
|
+
export function runWithAdvisoryLockReentrancyStore(store, callback) {
|
|
63
|
+
if (!installedStorage) return callback()
|
|
64
|
+
|
|
65
|
+
return installedStorage.run(store, callback)
|
|
66
|
+
}
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
// @ts-check
|
|
2
2
|
|
|
3
|
-
import {AsyncLocalStorage} from "node:async_hooks"
|
|
4
3
|
import timeout, {TimeoutError} from "awaitery/build/timeout.js"
|
|
5
4
|
import {DEFAULT_ADVISORY_LOCK_ACQUIRE_TIMEOUT_MS} from "./advisory-lock-bound.js"
|
|
6
5
|
import {
|
|
@@ -13,19 +12,14 @@ import {
|
|
|
13
12
|
AdvisoryLockServiceUnavailableError,
|
|
14
13
|
advisoryLockBackendUnavailable
|
|
15
14
|
} from "./advisory-lock-errors.js"
|
|
15
|
+
import {advisoryLockReentrancyStore, runWithAdvisoryLockReentrancyStore} from "./advisory-lock-reentrancy.js"
|
|
16
16
|
import {emitAdvisoryLockFallbackReport} from "./advisory-lock-telemetry.js"
|
|
17
17
|
|
|
18
18
|
/**
|
|
19
19
|
* Provenance of one advisory-lock acquisition (STC-210 design §3): every
|
|
20
20
|
* held lock reports the tier that actually served it, so a fallback is
|
|
21
21
|
* visible to the callback, in logs, and in `getAdvisoryLockBackendStatus()`.
|
|
22
|
-
* @typedef {
|
|
23
|
-
* @property {import("./advisory-lock-backend-policy.js").AdvisoryLockTier} backend - Tier that served the acquisition.
|
|
24
|
-
* @property {string} [driverType] - Database driver type when the driver tier served it.
|
|
25
|
-
* @property {number} [epoch] - Fencing epoch when the serving tier issues one.
|
|
26
|
-
* @property {boolean} fencingSupported - Whether this tier carries fencing tokens (design §8); the file tier reports false rather than pretending.
|
|
27
|
-
* @property {string} [holderId] - Backend holder id for the file tier.
|
|
28
|
-
* @property {string} [lockName] - Lock name this provenance belongs to.
|
|
22
|
+
* @typedef {import("./advisory-lock-reentrancy.js").AdvisoryLockBackendProvenance} AdvisoryLockBackendProvenance
|
|
29
23
|
*/
|
|
30
24
|
|
|
31
25
|
/**
|
|
@@ -78,8 +72,11 @@ class AdvisoryLockBusyError extends Error {
|
|
|
78
72
|
* `already-held` and treated it as contention. Applications worked around it
|
|
79
73
|
* with hand-passed `*LockHeld` booleans, which is exactly the parameter a
|
|
80
74
|
* monitor makes redundant.
|
|
75
|
+
*
|
|
76
|
+
* The storage itself belongs to the environment handler: Node owns async-context
|
|
77
|
+
* storage, while browser and Expo runtimes have none and therefore hold without
|
|
78
|
+
* re-entry tracking instead of pulling `node:async_hooks` into their bundles.
|
|
81
79
|
*/
|
|
82
|
-
const heldAdvisoryLocks = new AsyncLocalStorage()
|
|
83
80
|
|
|
84
81
|
/**
|
|
85
82
|
* Thrown when a callback holds an advisory lock longer than `holdTimeoutMs`.
|
|
@@ -300,7 +297,7 @@ export default class AdvisoryLockRunner {
|
|
|
300
297
|
* @returns {Promise<T>} - Resolves with the callback result.
|
|
301
298
|
*/
|
|
302
299
|
async runLockedCallback({callback, holdTimeoutMs, lockBackend, name, release}) {
|
|
303
|
-
const store = new Map(
|
|
300
|
+
const store = new Map(advisoryLockReentrancyStore() ?? [])
|
|
304
301
|
|
|
305
302
|
store.set(this.lockStorageKey(name), {depth: 1, lockBackend})
|
|
306
303
|
|
|
@@ -308,7 +305,7 @@ export default class AdvisoryLockRunner {
|
|
|
308
305
|
// The lock is released outside the context scope: a release path that
|
|
309
306
|
// touches the same name (a cleanup hook, an error reporter) must not look
|
|
310
307
|
// like a re-entry of a hold that is already over.
|
|
311
|
-
return await
|
|
308
|
+
return await runWithAdvisoryLockReentrancyStore(store, async () => await AdvisoryLockRunner.runWithAdvisoryLockHoldTimeout(name, callback, holdTimeoutMs, lockBackend))
|
|
312
309
|
} finally {
|
|
313
310
|
await release()
|
|
314
311
|
}
|
|
@@ -320,7 +317,7 @@ export default class AdvisoryLockRunner {
|
|
|
320
317
|
* @returns {{depth: number, lockBackend: AdvisoryLockBackendProvenance} | null} - The held entry, or null when this context does not hold the lock.
|
|
321
318
|
*/
|
|
322
319
|
heldEntryFor(name) {
|
|
323
|
-
const store =
|
|
320
|
+
const store = advisoryLockReentrancyStore()
|
|
324
321
|
|
|
325
322
|
if (!store) return null
|
|
326
323
|
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
// @ts-check
|
|
2
2
|
|
|
3
3
|
import "../database/annotations-async-hooks.js"
|
|
4
|
+
import "../database/advisory-lock-reentrancy-node.js"
|
|
4
5
|
import AdvisoryLockFileBackend from "../database/advisory-lock-file-backend.js"
|
|
5
6
|
import AdvisoryLockLockServerBackend from "../database/advisory-lock-lock-server-backend.js"
|
|
6
7
|
import Base from "./base.js"
|
package/src/routes/base-route.js
CHANGED
|
@@ -109,10 +109,11 @@ export default class VelociousBaseRoute {
|
|
|
109
109
|
* Runs namespace.
|
|
110
110
|
* @abstract
|
|
111
111
|
* @param {string} name - Name.
|
|
112
|
-
* @param {(arg: import("./namespace-route.js").default) => void}
|
|
112
|
+
* @param {object | ((arg: import("./namespace-route.js").default) => void)} options - Namespace options or the draw callback (legacy form).
|
|
113
|
+
* @param {((arg: import("./namespace-route.js").default) => void) | undefined} [callback] - Callback function.
|
|
113
114
|
* @returns {void} - No return value.
|
|
114
115
|
*/
|
|
115
|
-
namespace(name, callback) { throw new Error("'namespace' not implemented") } // eslint-disable-line no-unused-vars
|
|
116
|
+
namespace(name, options, callback) { throw new Error("'namespace' not implemented") } // eslint-disable-line no-unused-vars
|
|
116
117
|
|
|
117
118
|
/**
|
|
118
119
|
* Runs post.
|