velocious 1.0.669 → 1.0.671
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -1
- package/build/background-jobs/store.js +42 -8
- package/build/src/background-jobs/store.d.ts +30 -2
- package/build/src/background-jobs/store.d.ts.map +1 -1
- package/build/src/background-jobs/store.js +45 -16
- package/build/src/sync/server-sequence-allocator.d.ts +1 -0
- package/build/src/sync/server-sequence-allocator.d.ts.map +1 -1
- package/build/src/sync/server-sequence-allocator.js +23 -5
- package/build/src/sync/sync-publisher.d.ts +0 -17
- package/build/src/sync/sync-publisher.d.ts.map +1 -1
- package/build/src/sync/sync-publisher.js +10 -79
- package/build/src/sync/upsert-server-origin-sync-row.d.ts +29 -0
- package/build/src/sync/upsert-server-origin-sync-row.d.ts.map +1 -0
- package/build/src/sync/upsert-server-origin-sync-row.js +128 -0
- package/build/sync/server-sequence-allocator.js +23 -4
- package/build/sync/sync-publisher.js +9 -88
- package/build/sync/upsert-server-origin-sync-row.js +158 -0
- package/package.json +1 -1
- package/src/background-jobs/store.js +42 -8
- package/src/sync/server-sequence-allocator.js +23 -4
- package/src/sync/sync-publisher.js +9 -88
- package/src/sync/upsert-server-origin-sync-row.js +158 -0
|
@@ -38,6 +38,7 @@ export default class ServerSequenceAllocator {
|
|
|
38
38
|
this.tableName = tableName
|
|
39
39
|
this._memorySequence = 0
|
|
40
40
|
this._isReady = false
|
|
41
|
+
this._transactionalReadinessPending = false
|
|
41
42
|
/** @type {Promise<void> | null} */
|
|
42
43
|
this._readyPromise = null
|
|
43
44
|
}
|
|
@@ -110,10 +111,28 @@ export default class ServerSequenceAllocator {
|
|
|
110
111
|
// DDL joins any transaction already open on this connection (the mixin's
|
|
111
112
|
// beforeCreate allocation always runs inside the record save transaction),
|
|
112
113
|
// and on transactional-DDL databases (MSSQL, PostgreSQL, SQLite) a rollback
|
|
113
|
-
// of that outer transaction removes the just-created table again.
|
|
114
|
-
//
|
|
115
|
-
//
|
|
116
|
-
|
|
114
|
+
// of that outer transaction removes the just-created table again. Keep
|
|
115
|
+
// readiness pending for every later allocation in that same transaction;
|
|
116
|
+
// otherwise its table-exists check would incorrectly cache readiness before
|
|
117
|
+
// the outer transaction commits.
|
|
118
|
+
if (!created) {
|
|
119
|
+
if (!this._transactionalReadinessPending) this._isReady = true
|
|
120
|
+
return
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
if (db.insideTransaction()) {
|
|
124
|
+
this._transactionalReadinessPending = true
|
|
125
|
+
|
|
126
|
+
void db.transactionCompletion().then(() => {
|
|
127
|
+
// The transaction may have committed or rolled back. Re-check once
|
|
128
|
+
// on the next allocation, then cache the durable outcome normally.
|
|
129
|
+
this._isReady = false
|
|
130
|
+
this._transactionalReadinessPending = false
|
|
131
|
+
})
|
|
132
|
+
} else {
|
|
133
|
+
this._isReady = true
|
|
134
|
+
this._transactionalReadinessPending = false
|
|
135
|
+
}
|
|
117
136
|
}, connection)
|
|
118
137
|
|
|
119
138
|
try {
|
|
@@ -4,12 +4,11 @@ import Configuration from "../configuration.js"
|
|
|
4
4
|
import Logger from "../logger.js"
|
|
5
5
|
import {scalarModelPrimaryKeyValue} from "../utils/model-primary-key.js"
|
|
6
6
|
import restArgsError from "../utils/rest-args-error.js"
|
|
7
|
-
import sha256Hex from "../utils/sha256-hex.js"
|
|
8
|
-
import stableJsonStringify from "../utils/stable-json.js"
|
|
9
7
|
|
|
10
8
|
import {declaredSyncScopeAttributes} from "./sync-scope-attributes.js"
|
|
11
|
-
import {deliverDeclaredBroadcasts
|
|
9
|
+
import {deliverDeclaredBroadcasts} from "./sync-change-fanout.js"
|
|
12
10
|
import {isPublishingSuppressed} from "./sync-publish-suppression.js"
|
|
11
|
+
import {upsertServerOriginSyncRow} from "./upsert-server-origin-sync-row.js"
|
|
13
12
|
import {VELOCIOUS_SYNC_CHANNEL} from "./sync-channel-name.js"
|
|
14
13
|
|
|
15
14
|
/** @type {{create: "afterCreate", update: "afterUpdate", destroy: "afterDestroy"}} */
|
|
@@ -213,8 +212,13 @@ export default class SyncPublisher {
|
|
|
213
212
|
|
|
214
213
|
await record.connection().afterCommit(async () => {
|
|
215
214
|
try {
|
|
216
|
-
const
|
|
217
|
-
|
|
215
|
+
const syncRow = await upsertServerOriginSyncRow({
|
|
216
|
+
actorForeignKeyColumn: this.config.actorForeignKeyColumn,
|
|
217
|
+
attributes,
|
|
218
|
+
persistenceModel: operationScope,
|
|
219
|
+
scopeColumnNames: resourceConfig.scopePlan.map(({columnName}) => columnName),
|
|
220
|
+
syncModel: this.config.syncModel
|
|
221
|
+
})
|
|
218
222
|
|
|
219
223
|
await this.broadcaster()({
|
|
220
224
|
body: {
|
|
@@ -362,55 +366,6 @@ export default class SyncPublisher {
|
|
|
362
366
|
return entry
|
|
363
367
|
}
|
|
364
368
|
|
|
365
|
-
/**
|
|
366
|
-
* Upserts the published server-origin sync row for a resource identity:
|
|
367
|
-
* server-origin rows carry a null actor column (no device to echo the
|
|
368
|
-
* change back to), so repeated server changes to one complete resource and
|
|
369
|
-
* scope identity reuse and re-sequence one feed row. A database advisory
|
|
370
|
-
* lock serializes reconciliation plus the shared upsert because unique
|
|
371
|
-
* constraints containing nullable actor/scope columns do not enforce this
|
|
372
|
-
* identity portably across supported databases. Reconciliation retains the
|
|
373
|
-
* row with the newest feed sequence (then lowest id for a deterministic tie)
|
|
374
|
-
* and removes older matching server-origin rows before applying the current
|
|
375
|
-
* mutation.
|
|
376
|
-
* @param {Record<string, ReturnType<typeof JSON.parse>>} attributes - Snapshotted sync row attributes.
|
|
377
|
-
* @param {ReturnType<typeof JSON.parse>} syncModel - Operation-bound or static Sync model interface.
|
|
378
|
-
* @param {string[]} scopeColumnNames - Persisted scope columns participating in the complete identity.
|
|
379
|
-
* @returns {Promise<ReturnType<typeof JSON.parse>>} Upserted sync row.
|
|
380
|
-
*/
|
|
381
|
-
async upsertPublishedSyncRow(attributes, syncModel = this.config.syncModel, scopeColumnNames = []) {
|
|
382
|
-
/** @type {Record<string, ReturnType<typeof JSON.parse>>} */
|
|
383
|
-
const identity = {
|
|
384
|
-
[this.config.actorForeignKeyColumn]: null,
|
|
385
|
-
resource_id: attributes.resource_id,
|
|
386
|
-
resource_type: attributes.resource_type
|
|
387
|
-
}
|
|
388
|
-
|
|
389
|
-
for (const columnName of scopeColumnNames) {
|
|
390
|
-
if (!Object.hasOwn(attributes, columnName)) {
|
|
391
|
-
throw new Error(`Published sync row identity is missing the scope column ${columnName}`)
|
|
392
|
-
}
|
|
393
|
-
|
|
394
|
-
identity[columnName] = attributes[columnName]
|
|
395
|
-
}
|
|
396
|
-
|
|
397
|
-
return await this.config.syncModel.withAdvisoryLock(syncPublisherIdentityLockName(identity), async () => {
|
|
398
|
-
const matchingSyncs = await syncModel
|
|
399
|
-
.where(identity)
|
|
400
|
-
.toArray()
|
|
401
|
-
|
|
402
|
-
matchingSyncs.sort(comparePublishedSyncRowsByRecency)
|
|
403
|
-
|
|
404
|
-
const [existingSync, ...duplicateSyncs] = matchingSyncs
|
|
405
|
-
|
|
406
|
-
for (const duplicateSync of duplicateSyncs) {
|
|
407
|
-
await duplicateSync.destroy()
|
|
408
|
-
}
|
|
409
|
-
|
|
410
|
-
return await upsertSyncRow({attributes, existingSync, syncModel})
|
|
411
|
-
}, {dedicatedConnection: true})
|
|
412
|
-
}
|
|
413
|
-
|
|
414
369
|
/**
|
|
415
370
|
* Returns the broadcaster delivering declared broadcasts: the injected one,
|
|
416
371
|
* or the configuration's channel broadcast awaited through the pending
|
|
@@ -465,40 +420,6 @@ export default class SyncPublisher {
|
|
|
465
420
|
}
|
|
466
421
|
}
|
|
467
422
|
|
|
468
|
-
/**
|
|
469
|
-
* Returns a deterministic, MySQL-safe advisory-lock name for one complete
|
|
470
|
-
* server-origin publisher identity. Stable JSON preserves null actor/scope
|
|
471
|
-
* components distinctly from strings, and the truncated SHA-256 digest keeps
|
|
472
|
-
* the final name below MySQL/MariaDB's 64-character `GET_LOCK` limit.
|
|
473
|
-
* @param {Record<string, ReturnType<typeof JSON.parse>>} identity - Complete sync-row identity in column form.
|
|
474
|
-
* @returns {string} Advisory-lock name.
|
|
475
|
-
*/
|
|
476
|
-
function syncPublisherIdentityLockName(identity) {
|
|
477
|
-
const hash = sha256Hex(stableJsonStringify(identity)).slice(0, 32)
|
|
478
|
-
|
|
479
|
-
return `vsp:${hash}`
|
|
480
|
-
}
|
|
481
|
-
|
|
482
|
-
/**
|
|
483
|
-
* Orders matching published rows by the feed's public recency contract so
|
|
484
|
-
* legacy duplicates have one deterministic survivor. Server sequences are
|
|
485
|
-
* positive and monotonic; a legacy null sequence is older than any assigned
|
|
486
|
-
* sequence, and the immutable row id breaks otherwise-equal ties.
|
|
487
|
-
* @param {ReturnType<typeof JSON.parse>} left - First matching sync row.
|
|
488
|
-
* @param {ReturnType<typeof JSON.parse>} right - Second matching sync row.
|
|
489
|
-
* @returns {number} Sort comparison with the canonical survivor first.
|
|
490
|
-
*/
|
|
491
|
-
function comparePublishedSyncRowsByRecency(left, right) {
|
|
492
|
-
const leftSequence = left.serverSequence()
|
|
493
|
-
const rightSequence = right.serverSequence()
|
|
494
|
-
|
|
495
|
-
if (leftSequence === null && rightSequence !== null) return 1
|
|
496
|
-
if (leftSequence !== null && rightSequence === null) return -1
|
|
497
|
-
if (leftSequence !== rightSequence) return rightSequence - leftSequence
|
|
498
|
-
|
|
499
|
-
return left.id().localeCompare(right.id())
|
|
500
|
-
}
|
|
501
|
-
|
|
502
423
|
/**
|
|
503
424
|
* Resolves a model class's active publish declaration from `static sync`.
|
|
504
425
|
* Opted-out (`publish: false`) and undeclared models resolve to null; every
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
// @ts-check
|
|
2
|
+
|
|
3
|
+
import restArgsError from "../utils/rest-args-error.js"
|
|
4
|
+
import sha256Hex from "../utils/sha256-hex.js"
|
|
5
|
+
import stableJsonStringify from "../utils/stable-json.js"
|
|
6
|
+
|
|
7
|
+
import {upsertSyncRow} from "./sync-change-fanout.js"
|
|
8
|
+
import {declaredSyncScopeAttributes} from "./sync-scope-attributes.js"
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Atomically reconciles and upserts one server-origin sync row by its complete,
|
|
12
|
+
* null-safe logical identity. The identity contains the null actor, resource
|
|
13
|
+
* type/id, and either every scope column declared by the sync model or the
|
|
14
|
+
* caller's explicit effective scope columns for a supported legacy publisher.
|
|
15
|
+
* A stable portable advisory lock serializes runtime publication and maintenance/backfill calls;
|
|
16
|
+
* legacy duplicates converge to the newest server sequence, then the lowest
|
|
17
|
+
* immutable id, before the survivor is updated and re-sequenced.
|
|
18
|
+
*
|
|
19
|
+
* The lock is acquired through the static sync model on a dedicated connection.
|
|
20
|
+
* `persistenceModel` may be an operation-bound model facade so reads/writes keep
|
|
21
|
+
* their owning operation while sharing the same identity lock as ordinary
|
|
22
|
+
* publisher calls.
|
|
23
|
+
* @param {object} args - Persistence arguments.
|
|
24
|
+
* @param {string} [args.actorForeignKeyColumn] - Persisted actor foreign-key column.
|
|
25
|
+
* @param {Record<string, ReturnType<typeof JSON.parse>>} args.attributes - Complete sync-row mutation attributes.
|
|
26
|
+
* @param {ReturnType<typeof JSON.parse>} [args.persistenceModel] - Optional operation-bound model used for row reads/writes.
|
|
27
|
+
* @param {string[]} [args.scopeColumnNames] - Exact persisted scope columns overriding model declarations for a supported legacy publisher identity.
|
|
28
|
+
* @param {ReturnType<typeof JSON.parse>} args.syncModel - Static sync model owning scope metadata and the advisory lock.
|
|
29
|
+
* @returns {Promise<ReturnType<typeof JSON.parse>>} Created or reconciled sync row.
|
|
30
|
+
*/
|
|
31
|
+
export async function upsertServerOriginSyncRow({
|
|
32
|
+
actorForeignKeyColumn = "authentication_token_id",
|
|
33
|
+
attributes,
|
|
34
|
+
persistenceModel,
|
|
35
|
+
scopeColumnNames,
|
|
36
|
+
syncModel,
|
|
37
|
+
...restArgs
|
|
38
|
+
}) {
|
|
39
|
+
restArgsError(restArgs)
|
|
40
|
+
|
|
41
|
+
if (typeof actorForeignKeyColumn !== "string" || !actorForeignKeyColumn) {
|
|
42
|
+
throw new Error("Server-origin sync row actorForeignKeyColumn must be a non-empty string")
|
|
43
|
+
}
|
|
44
|
+
if (!attributes || typeof attributes !== "object" || Array.isArray(attributes)) {
|
|
45
|
+
throw new Error("Server-origin sync row attributes must be a plain object")
|
|
46
|
+
}
|
|
47
|
+
if (!Object.hasOwn(attributes, actorForeignKeyColumn) || attributes[actorForeignKeyColumn] === undefined) {
|
|
48
|
+
throw new Error(`Server-origin sync row identity must include the actor column ${actorForeignKeyColumn}`)
|
|
49
|
+
}
|
|
50
|
+
if (attributes[actorForeignKeyColumn] !== null) {
|
|
51
|
+
throw new Error(`Server-origin sync row actor column ${actorForeignKeyColumn} must be null`)
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
validateRequiredIdentityString({attributes, columnName: "resource_id"})
|
|
55
|
+
validateRequiredIdentityString({attributes, columnName: "resource_type"})
|
|
56
|
+
|
|
57
|
+
/** @type {Record<string, ReturnType<typeof JSON.parse>>} */
|
|
58
|
+
const identity = {
|
|
59
|
+
[actorForeignKeyColumn]: null,
|
|
60
|
+
resource_id: attributes.resource_id,
|
|
61
|
+
resource_type: attributes.resource_type
|
|
62
|
+
}
|
|
63
|
+
const attributeToColumnName = syncModel.getAttributeNameToColumnNameMap()
|
|
64
|
+
const scopeColumns = new Set(scopeColumnNames || [])
|
|
65
|
+
|
|
66
|
+
if (scopeColumnNames === undefined) {
|
|
67
|
+
for (const scopeAttribute of declaredSyncScopeAttributes(syncModel) || []) {
|
|
68
|
+
const columnName = attributeToColumnName[scopeAttribute]
|
|
69
|
+
|
|
70
|
+
if (!columnName) {
|
|
71
|
+
throw new Error(`${syncModel.name} declares the sync scope attribute ${scopeAttribute} but has no matching column for it`)
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
scopeColumns.add(columnName)
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
for (const columnName of scopeColumns) {
|
|
79
|
+
if (typeof columnName !== "string" || !columnName || !Object.values(attributeToColumnName).includes(columnName)) {
|
|
80
|
+
throw new Error(`Server-origin sync row received an unknown scope column: ${String(columnName)}`)
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
if (!Object.hasOwn(attributes, columnName) || attributes[columnName] === undefined) {
|
|
84
|
+
throw new Error(`Server-origin sync row identity must include the declared scope column ${columnName}`)
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
const value = attributes[columnName]
|
|
88
|
+
|
|
89
|
+
if (value !== null && typeof value !== "string" && (typeof value !== "number" || !Number.isFinite(value))) {
|
|
90
|
+
throw new Error(`Server-origin sync row scope column ${columnName} must be a string, finite number, or null`)
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
identity[columnName] = value
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
const rowModel = persistenceModel || syncModel
|
|
97
|
+
|
|
98
|
+
return await syncModel.withAdvisoryLock(serverOriginSyncRowLockName(identity), async () => {
|
|
99
|
+
const matchingSyncs = await rowModel
|
|
100
|
+
.where(identity)
|
|
101
|
+
.toArray()
|
|
102
|
+
|
|
103
|
+
matchingSyncs.sort(compareServerOriginSyncRowsByRecency)
|
|
104
|
+
|
|
105
|
+
const [existingSync, ...duplicateSyncs] = matchingSyncs
|
|
106
|
+
|
|
107
|
+
for (const duplicateSync of duplicateSyncs) {
|
|
108
|
+
await duplicateSync.destroy()
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
return await upsertSyncRow({attributes, existingSync, syncModel: rowModel})
|
|
112
|
+
}, {dedicatedConnection: true})
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/**
|
|
116
|
+
* Validates one required string component of the server-origin identity.
|
|
117
|
+
* @param {{attributes: Record<string, ReturnType<typeof JSON.parse>>, columnName: "resource_id" | "resource_type"}} args - Attributes and required column.
|
|
118
|
+
* @returns {void}
|
|
119
|
+
*/
|
|
120
|
+
function validateRequiredIdentityString({attributes, columnName}) {
|
|
121
|
+
const value = attributes[columnName]
|
|
122
|
+
|
|
123
|
+
if (typeof value !== "string" || !value) {
|
|
124
|
+
throw new Error(`Server-origin sync row ${columnName} must be a non-empty string`)
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/**
|
|
129
|
+
* Returns the stable MySQL-safe advisory-lock name shared by every caller of a
|
|
130
|
+
* complete server-origin identity. Stable JSON preserves null identity values,
|
|
131
|
+
* while the truncated SHA-256 digest keeps the name under GET_LOCK's limit.
|
|
132
|
+
* @param {Record<string, ReturnType<typeof JSON.parse>>} identity - Complete sync-row identity in column form.
|
|
133
|
+
* @returns {string} Advisory-lock name.
|
|
134
|
+
*/
|
|
135
|
+
function serverOriginSyncRowLockName(identity) {
|
|
136
|
+
const hash = sha256Hex(stableJsonStringify(identity)).slice(0, 32)
|
|
137
|
+
|
|
138
|
+
return `vsp:${hash}`
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
/**
|
|
142
|
+
* Orders matching rows with the canonical deterministic survivor first:
|
|
143
|
+
* newest server sequence, then lowest immutable id. A null legacy sequence is
|
|
144
|
+
* older than every assigned sequence.
|
|
145
|
+
* @param {ReturnType<typeof JSON.parse>} left - First matching sync row.
|
|
146
|
+
* @param {ReturnType<typeof JSON.parse>} right - Second matching sync row.
|
|
147
|
+
* @returns {number} Sort comparison.
|
|
148
|
+
*/
|
|
149
|
+
function compareServerOriginSyncRowsByRecency(left, right) {
|
|
150
|
+
const leftSequence = left.serverSequence()
|
|
151
|
+
const rightSequence = right.serverSequence()
|
|
152
|
+
|
|
153
|
+
if (leftSequence === null && rightSequence !== null) return 1
|
|
154
|
+
if (leftSequence !== null && rightSequence === null) return -1
|
|
155
|
+
if (leftSequence !== rightSequence) return rightSequence - leftSequence
|
|
156
|
+
|
|
157
|
+
return left.id().localeCompare(right.id())
|
|
158
|
+
}
|
package/package.json
CHANGED
|
@@ -59,6 +59,16 @@ import {
|
|
|
59
59
|
* @property {{failureMessage: string, name: string}} [advisoryLock] - Session lock held around the transaction.
|
|
60
60
|
*/
|
|
61
61
|
|
|
62
|
+
/**
|
|
63
|
+
* BackgroundJobPruneCandidateMetadata type. Immutable metadata for one selected
|
|
64
|
+
* retention candidate batch, exposed to the optional prune barrier hook.
|
|
65
|
+
* @typedef {object} BackgroundJobPruneCandidateMetadata
|
|
66
|
+
* @property {Readonly<Array<string>>} candidates - Job ids selected for this batch.
|
|
67
|
+
* @property {string} status - Terminal status the candidates were selected by.
|
|
68
|
+
* @property {string} column - Terminal timestamp column compared against the cutoff.
|
|
69
|
+
* @property {number} cutoff - Cutoff timestamp the candidates were selected against.
|
|
70
|
+
*/
|
|
71
|
+
|
|
62
72
|
/**
|
|
63
73
|
* BackgroundJobConcurrencyCountRow type.
|
|
64
74
|
* @typedef {object} BackgroundJobConcurrencyCountRow
|
|
@@ -158,13 +168,15 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
|
|
|
158
168
|
* @param {string} [args.databaseIdentifier] - Database identifier.
|
|
159
169
|
* @param {{now: () => number}} [args.clock] - Injectable persistence clock.
|
|
160
170
|
* @param {(producerProof: import("./types.js").BackgroundJobProducerProof) => void | Promise<void>} [args.afterOwnedProducerValidation] - Exact owned-enqueue validation hook.
|
|
171
|
+
* @param {(metadata: BackgroundJobPruneCandidateMetadata) => void | Promise<void>} [args.afterPruneCandidatesSelected] - Optional barrier invoked after one retention batch's candidate discovery and before its serialized delete transaction.
|
|
161
172
|
*/
|
|
162
|
-
constructor({configuration, databaseIdentifier, clock, afterOwnedProducerValidation}) {
|
|
173
|
+
constructor({configuration, databaseIdentifier, clock, afterOwnedProducerValidation, afterPruneCandidatesSelected}) {
|
|
163
174
|
super()
|
|
164
175
|
this.configuration = configuration
|
|
165
176
|
this.databaseIdentifier = databaseIdentifier
|
|
166
177
|
this.clock = clock || {now: () => Date.now()}
|
|
167
178
|
this.afterOwnedProducerValidation = afterOwnedProducerValidation
|
|
179
|
+
this.afterPruneCandidatesSelected = afterPruneCandidatesSelected
|
|
168
180
|
this.logger = new Logger(this)
|
|
169
181
|
this._readyPromise = null
|
|
170
182
|
this._queueConcurrencyReconciled = false
|
|
@@ -1790,7 +1802,14 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
|
|
|
1790
1802
|
|
|
1791
1803
|
/**
|
|
1792
1804
|
* Deletes rows of one terminal status older than a cutoff, batch by batch,
|
|
1793
|
-
* until a page returns fewer than `batchSize` rows.
|
|
1805
|
+
* until a candidate page returns fewer than `batchSize` rows. Candidate
|
|
1806
|
+
* discovery runs on a plain connection — never inside the serialized count
|
|
1807
|
+
* mutation — so a long scan cannot hold the count-revision lock and starve
|
|
1808
|
+
* enqueue acknowledgements; only the short delete transaction is serialized.
|
|
1809
|
+
* The delete revalidates status and cutoff for the selected ids, publishes
|
|
1810
|
+
* the delta from the actual affected-row count, and a page whose candidates
|
|
1811
|
+
* were already removed by a concurrent pruner still ends the pass only when
|
|
1812
|
+
* the page itself is short.
|
|
1794
1813
|
* @param {object} args - Options.
|
|
1795
1814
|
* @param {string} args.status - Terminal status to prune.
|
|
1796
1815
|
* @param {string} args.column - Timestamp column compared against the cutoff.
|
|
@@ -1802,22 +1821,37 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
|
|
|
1802
1821
|
let deleted = 0
|
|
1803
1822
|
|
|
1804
1823
|
for (;;) {
|
|
1805
|
-
const
|
|
1806
|
-
|
|
1824
|
+
const candidates = await this._withDb(async (db) =>
|
|
1825
|
+
await db
|
|
1807
1826
|
.newQuery()
|
|
1808
1827
|
.from(JOBS_TABLE)
|
|
1809
1828
|
.select("id")
|
|
1810
1829
|
.where({status})
|
|
1811
1830
|
.where(`${db.quoteColumn(column)} <= ${db.quote(cutoff)}`)
|
|
1831
|
+
.order({column, direction: "ASC"})
|
|
1832
|
+
.order({column: "id", direction: "ASC"})
|
|
1812
1833
|
.limit(batchSize)
|
|
1813
1834
|
.results()
|
|
1835
|
+
)
|
|
1814
1836
|
|
|
1815
|
-
|
|
1837
|
+
if (candidates.length === 0) break
|
|
1816
1838
|
|
|
1817
|
-
|
|
1839
|
+
const candidateIds = Object.freeze(candidates.map((/** @type {Record<string, ReturnType<typeof JSON.parse>>} */ row) => String(row.id)))
|
|
1840
|
+
|
|
1841
|
+
if (this.afterPruneCandidatesSelected) {
|
|
1842
|
+
await this.afterPruneCandidatesSelected(Object.freeze({
|
|
1843
|
+
candidates: candidateIds,
|
|
1844
|
+
column,
|
|
1845
|
+
cutoff,
|
|
1846
|
+
status
|
|
1847
|
+
}))
|
|
1848
|
+
}
|
|
1849
|
+
|
|
1850
|
+
const removed = await this._serializedCountMutation(async (db) => {
|
|
1851
|
+
const ids = candidateIds.map((id) => db.quote(id)).join(", ")
|
|
1818
1852
|
|
|
1819
1853
|
const removed = await db.affectedRows(
|
|
1820
|
-
`DELETE FROM ${db.quoteTable(JOBS_TABLE)} WHERE ${db.quoteColumn("id")} IN (${ids})`
|
|
1854
|
+
`DELETE FROM ${db.quoteTable(JOBS_TABLE)} WHERE ${db.quoteColumn("id")} IN (${ids}) AND ${db.quoteColumn("status")} = ${db.quote(status)} AND ${db.quoteColumn(column)} <= ${db.quote(cutoff)}`
|
|
1821
1855
|
)
|
|
1822
1856
|
|
|
1823
1857
|
await this._recordCountDelta(db, {all: -removed, [status]: -removed})
|
|
@@ -1826,7 +1860,7 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
|
|
|
1826
1860
|
})
|
|
1827
1861
|
|
|
1828
1862
|
deleted += removed
|
|
1829
|
-
if (
|
|
1863
|
+
if (candidates.length < batchSize) break
|
|
1830
1864
|
}
|
|
1831
1865
|
|
|
1832
1866
|
return deleted
|
|
@@ -38,6 +38,7 @@ export default class ServerSequenceAllocator {
|
|
|
38
38
|
this.tableName = tableName
|
|
39
39
|
this._memorySequence = 0
|
|
40
40
|
this._isReady = false
|
|
41
|
+
this._transactionalReadinessPending = false
|
|
41
42
|
/** @type {Promise<void> | null} */
|
|
42
43
|
this._readyPromise = null
|
|
43
44
|
}
|
|
@@ -110,10 +111,28 @@ export default class ServerSequenceAllocator {
|
|
|
110
111
|
// DDL joins any transaction already open on this connection (the mixin's
|
|
111
112
|
// beforeCreate allocation always runs inside the record save transaction),
|
|
112
113
|
// and on transactional-DDL databases (MSSQL, PostgreSQL, SQLite) a rollback
|
|
113
|
-
// of that outer transaction removes the just-created table again.
|
|
114
|
-
//
|
|
115
|
-
//
|
|
116
|
-
|
|
114
|
+
// of that outer transaction removes the just-created table again. Keep
|
|
115
|
+
// readiness pending for every later allocation in that same transaction;
|
|
116
|
+
// otherwise its table-exists check would incorrectly cache readiness before
|
|
117
|
+
// the outer transaction commits.
|
|
118
|
+
if (!created) {
|
|
119
|
+
if (!this._transactionalReadinessPending) this._isReady = true
|
|
120
|
+
return
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
if (db.insideTransaction()) {
|
|
124
|
+
this._transactionalReadinessPending = true
|
|
125
|
+
|
|
126
|
+
void db.transactionCompletion().then(() => {
|
|
127
|
+
// The transaction may have committed or rolled back. Re-check once
|
|
128
|
+
// on the next allocation, then cache the durable outcome normally.
|
|
129
|
+
this._isReady = false
|
|
130
|
+
this._transactionalReadinessPending = false
|
|
131
|
+
})
|
|
132
|
+
} else {
|
|
133
|
+
this._isReady = true
|
|
134
|
+
this._transactionalReadinessPending = false
|
|
135
|
+
}
|
|
117
136
|
}, connection)
|
|
118
137
|
|
|
119
138
|
try {
|
|
@@ -4,12 +4,11 @@ import Configuration from "../configuration.js"
|
|
|
4
4
|
import Logger from "../logger.js"
|
|
5
5
|
import {scalarModelPrimaryKeyValue} from "../utils/model-primary-key.js"
|
|
6
6
|
import restArgsError from "../utils/rest-args-error.js"
|
|
7
|
-
import sha256Hex from "../utils/sha256-hex.js"
|
|
8
|
-
import stableJsonStringify from "../utils/stable-json.js"
|
|
9
7
|
|
|
10
8
|
import {declaredSyncScopeAttributes} from "./sync-scope-attributes.js"
|
|
11
|
-
import {deliverDeclaredBroadcasts
|
|
9
|
+
import {deliverDeclaredBroadcasts} from "./sync-change-fanout.js"
|
|
12
10
|
import {isPublishingSuppressed} from "./sync-publish-suppression.js"
|
|
11
|
+
import {upsertServerOriginSyncRow} from "./upsert-server-origin-sync-row.js"
|
|
13
12
|
import {VELOCIOUS_SYNC_CHANNEL} from "./sync-channel-name.js"
|
|
14
13
|
|
|
15
14
|
/** @type {{create: "afterCreate", update: "afterUpdate", destroy: "afterDestroy"}} */
|
|
@@ -213,8 +212,13 @@ export default class SyncPublisher {
|
|
|
213
212
|
|
|
214
213
|
await record.connection().afterCommit(async () => {
|
|
215
214
|
try {
|
|
216
|
-
const
|
|
217
|
-
|
|
215
|
+
const syncRow = await upsertServerOriginSyncRow({
|
|
216
|
+
actorForeignKeyColumn: this.config.actorForeignKeyColumn,
|
|
217
|
+
attributes,
|
|
218
|
+
persistenceModel: operationScope,
|
|
219
|
+
scopeColumnNames: resourceConfig.scopePlan.map(({columnName}) => columnName),
|
|
220
|
+
syncModel: this.config.syncModel
|
|
221
|
+
})
|
|
218
222
|
|
|
219
223
|
await this.broadcaster()({
|
|
220
224
|
body: {
|
|
@@ -362,55 +366,6 @@ export default class SyncPublisher {
|
|
|
362
366
|
return entry
|
|
363
367
|
}
|
|
364
368
|
|
|
365
|
-
/**
|
|
366
|
-
* Upserts the published server-origin sync row for a resource identity:
|
|
367
|
-
* server-origin rows carry a null actor column (no device to echo the
|
|
368
|
-
* change back to), so repeated server changes to one complete resource and
|
|
369
|
-
* scope identity reuse and re-sequence one feed row. A database advisory
|
|
370
|
-
* lock serializes reconciliation plus the shared upsert because unique
|
|
371
|
-
* constraints containing nullable actor/scope columns do not enforce this
|
|
372
|
-
* identity portably across supported databases. Reconciliation retains the
|
|
373
|
-
* row with the newest feed sequence (then lowest id for a deterministic tie)
|
|
374
|
-
* and removes older matching server-origin rows before applying the current
|
|
375
|
-
* mutation.
|
|
376
|
-
* @param {Record<string, ReturnType<typeof JSON.parse>>} attributes - Snapshotted sync row attributes.
|
|
377
|
-
* @param {ReturnType<typeof JSON.parse>} syncModel - Operation-bound or static Sync model interface.
|
|
378
|
-
* @param {string[]} scopeColumnNames - Persisted scope columns participating in the complete identity.
|
|
379
|
-
* @returns {Promise<ReturnType<typeof JSON.parse>>} Upserted sync row.
|
|
380
|
-
*/
|
|
381
|
-
async upsertPublishedSyncRow(attributes, syncModel = this.config.syncModel, scopeColumnNames = []) {
|
|
382
|
-
/** @type {Record<string, ReturnType<typeof JSON.parse>>} */
|
|
383
|
-
const identity = {
|
|
384
|
-
[this.config.actorForeignKeyColumn]: null,
|
|
385
|
-
resource_id: attributes.resource_id,
|
|
386
|
-
resource_type: attributes.resource_type
|
|
387
|
-
}
|
|
388
|
-
|
|
389
|
-
for (const columnName of scopeColumnNames) {
|
|
390
|
-
if (!Object.hasOwn(attributes, columnName)) {
|
|
391
|
-
throw new Error(`Published sync row identity is missing the scope column ${columnName}`)
|
|
392
|
-
}
|
|
393
|
-
|
|
394
|
-
identity[columnName] = attributes[columnName]
|
|
395
|
-
}
|
|
396
|
-
|
|
397
|
-
return await this.config.syncModel.withAdvisoryLock(syncPublisherIdentityLockName(identity), async () => {
|
|
398
|
-
const matchingSyncs = await syncModel
|
|
399
|
-
.where(identity)
|
|
400
|
-
.toArray()
|
|
401
|
-
|
|
402
|
-
matchingSyncs.sort(comparePublishedSyncRowsByRecency)
|
|
403
|
-
|
|
404
|
-
const [existingSync, ...duplicateSyncs] = matchingSyncs
|
|
405
|
-
|
|
406
|
-
for (const duplicateSync of duplicateSyncs) {
|
|
407
|
-
await duplicateSync.destroy()
|
|
408
|
-
}
|
|
409
|
-
|
|
410
|
-
return await upsertSyncRow({attributes, existingSync, syncModel})
|
|
411
|
-
}, {dedicatedConnection: true})
|
|
412
|
-
}
|
|
413
|
-
|
|
414
369
|
/**
|
|
415
370
|
* Returns the broadcaster delivering declared broadcasts: the injected one,
|
|
416
371
|
* or the configuration's channel broadcast awaited through the pending
|
|
@@ -465,40 +420,6 @@ export default class SyncPublisher {
|
|
|
465
420
|
}
|
|
466
421
|
}
|
|
467
422
|
|
|
468
|
-
/**
|
|
469
|
-
* Returns a deterministic, MySQL-safe advisory-lock name for one complete
|
|
470
|
-
* server-origin publisher identity. Stable JSON preserves null actor/scope
|
|
471
|
-
* components distinctly from strings, and the truncated SHA-256 digest keeps
|
|
472
|
-
* the final name below MySQL/MariaDB's 64-character `GET_LOCK` limit.
|
|
473
|
-
* @param {Record<string, ReturnType<typeof JSON.parse>>} identity - Complete sync-row identity in column form.
|
|
474
|
-
* @returns {string} Advisory-lock name.
|
|
475
|
-
*/
|
|
476
|
-
function syncPublisherIdentityLockName(identity) {
|
|
477
|
-
const hash = sha256Hex(stableJsonStringify(identity)).slice(0, 32)
|
|
478
|
-
|
|
479
|
-
return `vsp:${hash}`
|
|
480
|
-
}
|
|
481
|
-
|
|
482
|
-
/**
|
|
483
|
-
* Orders matching published rows by the feed's public recency contract so
|
|
484
|
-
* legacy duplicates have one deterministic survivor. Server sequences are
|
|
485
|
-
* positive and monotonic; a legacy null sequence is older than any assigned
|
|
486
|
-
* sequence, and the immutable row id breaks otherwise-equal ties.
|
|
487
|
-
* @param {ReturnType<typeof JSON.parse>} left - First matching sync row.
|
|
488
|
-
* @param {ReturnType<typeof JSON.parse>} right - Second matching sync row.
|
|
489
|
-
* @returns {number} Sort comparison with the canonical survivor first.
|
|
490
|
-
*/
|
|
491
|
-
function comparePublishedSyncRowsByRecency(left, right) {
|
|
492
|
-
const leftSequence = left.serverSequence()
|
|
493
|
-
const rightSequence = right.serverSequence()
|
|
494
|
-
|
|
495
|
-
if (leftSequence === null && rightSequence !== null) return 1
|
|
496
|
-
if (leftSequence !== null && rightSequence === null) return -1
|
|
497
|
-
if (leftSequence !== rightSequence) return rightSequence - leftSequence
|
|
498
|
-
|
|
499
|
-
return left.id().localeCompare(right.id())
|
|
500
|
-
}
|
|
501
|
-
|
|
502
423
|
/**
|
|
503
424
|
* Resolves a model class's active publish declaration from `static sync`.
|
|
504
425
|
* Opted-out (`publish: false`) and undeclared models resolve to null; every
|