s3db.js 10.0.17 → 10.0.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/s3db.cjs.js +48 -40
- package/dist/s3db.cjs.js.map +1 -1
- package/dist/s3db.d.ts +198 -2
- package/dist/s3db.es.js +48 -40
- package/dist/s3db.es.js.map +1 -1
- package/package.json +4 -2
- package/src/plugins/eventual-consistency/config.js +65 -29
- package/src/plugins/eventual-consistency/utils.js +3 -13
- package/src/s3db.d.ts +198 -2
- package/src/plugins/eventual-consistency.plugin.js +0 -2559
|
@@ -1,2559 +0,0 @@
|
|
|
1
|
-
import Plugin from "./plugin.class.js";
|
|
2
|
-
import tryFn from "../concerns/try-fn.js";
|
|
3
|
-
import { idGenerator } from "../concerns/id.js";
|
|
4
|
-
import { PromisePool } from "@supercharge/promise-pool";
|
|
5
|
-
|
|
6
|
-
export class EventualConsistencyPlugin extends Plugin {
|
|
7
|
-
constructor(options = {}) {
|
|
8
|
-
super(options);
|
|
9
|
-
|
|
10
|
-
// Validate resources structure
|
|
11
|
-
if (!options.resources || typeof options.resources !== 'object') {
|
|
12
|
-
throw new Error(
|
|
13
|
-
"EventualConsistencyPlugin requires 'resources' option.\n" +
|
|
14
|
-
"Example: { resources: { urls: ['clicks', 'views'], posts: ['likes'] } }"
|
|
15
|
-
);
|
|
16
|
-
}
|
|
17
|
-
|
|
18
|
-
// Auto-detect timezone from environment or system
|
|
19
|
-
const detectedTimezone = this._detectTimezone();
|
|
20
|
-
|
|
21
|
-
// Create shared configuration
|
|
22
|
-
this.config = {
|
|
23
|
-
cohort: {
|
|
24
|
-
timezone: options.cohort?.timezone || detectedTimezone
|
|
25
|
-
},
|
|
26
|
-
reducer: options.reducer || ((transactions) => {
|
|
27
|
-
let baseValue = 0;
|
|
28
|
-
for (const t of transactions) {
|
|
29
|
-
if (t.operation === 'set') {
|
|
30
|
-
baseValue = t.value;
|
|
31
|
-
} else if (t.operation === 'add') {
|
|
32
|
-
baseValue += t.value;
|
|
33
|
-
} else if (t.operation === 'sub') {
|
|
34
|
-
baseValue -= t.value;
|
|
35
|
-
}
|
|
36
|
-
}
|
|
37
|
-
return baseValue;
|
|
38
|
-
}),
|
|
39
|
-
consolidationInterval: options.consolidationInterval ?? 300,
|
|
40
|
-
consolidationConcurrency: options.consolidationConcurrency || 5,
|
|
41
|
-
consolidationWindow: options.consolidationWindow || 24,
|
|
42
|
-
autoConsolidate: options.autoConsolidate !== false,
|
|
43
|
-
lateArrivalStrategy: options.lateArrivalStrategy || 'warn',
|
|
44
|
-
batchTransactions: options.batchTransactions || false,
|
|
45
|
-
batchSize: options.batchSize || 100,
|
|
46
|
-
mode: options.mode || 'async',
|
|
47
|
-
lockTimeout: options.lockTimeout || 300,
|
|
48
|
-
transactionRetention: options.transactionRetention || 30,
|
|
49
|
-
gcInterval: options.gcInterval || 86400,
|
|
50
|
-
verbose: options.verbose || false,
|
|
51
|
-
enableAnalytics: options.enableAnalytics || false,
|
|
52
|
-
analyticsConfig: {
|
|
53
|
-
periods: options.analyticsConfig?.periods || ['hour', 'day', 'month'],
|
|
54
|
-
metrics: options.analyticsConfig?.metrics || ['count', 'sum', 'avg', 'min', 'max'],
|
|
55
|
-
rollupStrategy: options.analyticsConfig?.rollupStrategy || 'incremental',
|
|
56
|
-
retentionDays: options.analyticsConfig?.retentionDays || 365
|
|
57
|
-
},
|
|
58
|
-
// Checkpoint configuration for high-volume scenarios
|
|
59
|
-
enableCheckpoints: options.enableCheckpoints !== false, // Default: true
|
|
60
|
-
checkpointStrategy: options.checkpointStrategy || 'hourly', // 'hourly', 'daily', 'manual', 'disabled'
|
|
61
|
-
checkpointRetention: options.checkpointRetention || 90, // Days to keep checkpoints
|
|
62
|
-
checkpointThreshold: options.checkpointThreshold || 1000, // Min transactions before creating checkpoint
|
|
63
|
-
deleteConsolidatedTransactions: options.deleteConsolidatedTransactions !== false, // Delete transactions after checkpoint
|
|
64
|
-
autoCheckpoint: options.autoCheckpoint !== false // Auto-create checkpoints for old cohorts
|
|
65
|
-
};
|
|
66
|
-
|
|
67
|
-
// Create field handlers map
|
|
68
|
-
this.fieldHandlers = new Map(); // Map<resourceName, Map<fieldName, handler>>
|
|
69
|
-
|
|
70
|
-
// Parse resources configuration
|
|
71
|
-
for (const [resourceName, fields] of Object.entries(options.resources)) {
|
|
72
|
-
if (!Array.isArray(fields)) {
|
|
73
|
-
throw new Error(
|
|
74
|
-
`EventualConsistencyPlugin resources.${resourceName} must be an array of field names`
|
|
75
|
-
);
|
|
76
|
-
}
|
|
77
|
-
|
|
78
|
-
const resourceHandlers = new Map();
|
|
79
|
-
for (const fieldName of fields) {
|
|
80
|
-
// Create a field handler for each resource/field combination
|
|
81
|
-
resourceHandlers.set(fieldName, this._createFieldHandler(resourceName, fieldName));
|
|
82
|
-
}
|
|
83
|
-
this.fieldHandlers.set(resourceName, resourceHandlers);
|
|
84
|
-
}
|
|
85
|
-
|
|
86
|
-
// Warn about batching in distributed environments
|
|
87
|
-
if (this.config.batchTransactions && !this.config.verbose) {
|
|
88
|
-
console.warn(
|
|
89
|
-
`[EventualConsistency] WARNING: batchTransactions is enabled. ` +
|
|
90
|
-
`This stores transactions in memory and will lose data if container crashes. ` +
|
|
91
|
-
`Not recommended for distributed/production environments.`
|
|
92
|
-
);
|
|
93
|
-
}
|
|
94
|
-
|
|
95
|
-
// Log initialization if verbose
|
|
96
|
-
if (this.config.verbose) {
|
|
97
|
-
const totalFields = Array.from(this.fieldHandlers.values())
|
|
98
|
-
.reduce((sum, handlers) => sum + handlers.size, 0);
|
|
99
|
-
console.log(
|
|
100
|
-
`[EventualConsistency] Initialized with ${this.fieldHandlers.size} resource(s), ` +
|
|
101
|
-
`${totalFields} field(s) total`
|
|
102
|
-
);
|
|
103
|
-
|
|
104
|
-
// Log detected timezone if not explicitly set
|
|
105
|
-
if (!options.cohort?.timezone) {
|
|
106
|
-
console.log(
|
|
107
|
-
`[EventualConsistency] Auto-detected timezone: ${this.config.cohort.timezone} ` +
|
|
108
|
-
`(from ${process.env.TZ ? 'TZ env var' : 'system Intl API'})`
|
|
109
|
-
);
|
|
110
|
-
}
|
|
111
|
-
}
|
|
112
|
-
}
|
|
113
|
-
|
|
114
|
-
/**
|
|
115
|
-
* Create a field handler for a specific resource/field combination
|
|
116
|
-
* @private
|
|
117
|
-
*/
|
|
118
|
-
_createFieldHandler(resourceName, fieldName) {
|
|
119
|
-
return {
|
|
120
|
-
resource: resourceName,
|
|
121
|
-
field: fieldName,
|
|
122
|
-
transactionResource: null,
|
|
123
|
-
targetResource: null,
|
|
124
|
-
analyticsResource: null,
|
|
125
|
-
lockResource: null,
|
|
126
|
-
checkpointResource: null, // NEW: Checkpoint resource for high-volume optimization
|
|
127
|
-
consolidationTimer: null,
|
|
128
|
-
gcTimer: null,
|
|
129
|
-
pendingTransactions: new Map(),
|
|
130
|
-
deferredSetup: false
|
|
131
|
-
};
|
|
132
|
-
}
|
|
133
|
-
|
|
134
|
-
async onSetup() {
|
|
135
|
-
// Iterate over all resource/field combinations
|
|
136
|
-
for (const [resourceName, fieldHandlers] of this.fieldHandlers) {
|
|
137
|
-
const targetResource = this.database.resources[resourceName];
|
|
138
|
-
|
|
139
|
-
if (!targetResource) {
|
|
140
|
-
// Resource doesn't exist yet - mark for deferred setup
|
|
141
|
-
for (const handler of fieldHandlers.values()) {
|
|
142
|
-
handler.deferredSetup = true;
|
|
143
|
-
}
|
|
144
|
-
// Watch for this resource to be created
|
|
145
|
-
this._watchForResource(resourceName);
|
|
146
|
-
continue;
|
|
147
|
-
}
|
|
148
|
-
|
|
149
|
-
// Resource exists - setup all fields for this resource
|
|
150
|
-
for (const [fieldName, handler] of fieldHandlers) {
|
|
151
|
-
handler.targetResource = targetResource;
|
|
152
|
-
await this._completeFieldSetup(handler);
|
|
153
|
-
}
|
|
154
|
-
}
|
|
155
|
-
}
|
|
156
|
-
|
|
157
|
-
/**
|
|
158
|
-
* Watch for a specific resource creation
|
|
159
|
-
* @private
|
|
160
|
-
*/
|
|
161
|
-
_watchForResource(resourceName) {
|
|
162
|
-
const hookCallback = async ({ resource, config }) => {
|
|
163
|
-
if (config.name === resourceName) {
|
|
164
|
-
const fieldHandlers = this.fieldHandlers.get(resourceName);
|
|
165
|
-
if (!fieldHandlers) return;
|
|
166
|
-
|
|
167
|
-
// Setup all fields for this resource
|
|
168
|
-
for (const [fieldName, handler] of fieldHandlers) {
|
|
169
|
-
if (handler.deferredSetup) {
|
|
170
|
-
handler.targetResource = resource;
|
|
171
|
-
handler.deferredSetup = false;
|
|
172
|
-
await this._completeFieldSetup(handler);
|
|
173
|
-
}
|
|
174
|
-
}
|
|
175
|
-
}
|
|
176
|
-
};
|
|
177
|
-
|
|
178
|
-
this.database.addHook('afterCreateResource', hookCallback);
|
|
179
|
-
}
|
|
180
|
-
|
|
181
|
-
/**
|
|
182
|
-
* Complete setup for a single field handler
|
|
183
|
-
* @private
|
|
184
|
-
*/
|
|
185
|
-
async _completeFieldSetup(handler) {
|
|
186
|
-
if (!handler.targetResource) return;
|
|
187
|
-
|
|
188
|
-
const config = this.config;
|
|
189
|
-
const resourceName = handler.resource;
|
|
190
|
-
const fieldName = handler.field;
|
|
191
|
-
|
|
192
|
-
// Create transaction resource with partitions
|
|
193
|
-
const transactionResourceName = `${resourceName}_transactions_${fieldName}`;
|
|
194
|
-
const partitionConfig = this.createPartitionConfig();
|
|
195
|
-
|
|
196
|
-
const [ok, err, transactionResource] = await tryFn(() =>
|
|
197
|
-
this.database.createResource({
|
|
198
|
-
name: transactionResourceName,
|
|
199
|
-
attributes: {
|
|
200
|
-
id: 'string|required',
|
|
201
|
-
originalId: 'string|required',
|
|
202
|
-
field: 'string|required',
|
|
203
|
-
value: 'number|required',
|
|
204
|
-
operation: 'string|required',
|
|
205
|
-
timestamp: 'string|required',
|
|
206
|
-
cohortDate: 'string|required',
|
|
207
|
-
cohortHour: 'string|required',
|
|
208
|
-
cohortMonth: 'string|optional',
|
|
209
|
-
source: 'string|optional',
|
|
210
|
-
applied: 'boolean|optional'
|
|
211
|
-
},
|
|
212
|
-
behavior: 'body-overflow',
|
|
213
|
-
timestamps: true,
|
|
214
|
-
partitions: partitionConfig,
|
|
215
|
-
asyncPartitions: true,
|
|
216
|
-
createdBy: 'EventualConsistencyPlugin'
|
|
217
|
-
})
|
|
218
|
-
);
|
|
219
|
-
|
|
220
|
-
if (!ok && !this.database.resources[transactionResourceName]) {
|
|
221
|
-
throw new Error(`Failed to create transaction resource for ${resourceName}.${fieldName}: ${err?.message}`);
|
|
222
|
-
}
|
|
223
|
-
|
|
224
|
-
handler.transactionResource = ok ? transactionResource : this.database.resources[transactionResourceName];
|
|
225
|
-
|
|
226
|
-
// Create lock resource
|
|
227
|
-
const lockResourceName = `${resourceName}_consolidation_locks_${fieldName}`;
|
|
228
|
-
const [lockOk, lockErr, lockResource] = await tryFn(() =>
|
|
229
|
-
this.database.createResource({
|
|
230
|
-
name: lockResourceName,
|
|
231
|
-
attributes: {
|
|
232
|
-
id: 'string|required',
|
|
233
|
-
lockedAt: 'number|required',
|
|
234
|
-
workerId: 'string|optional'
|
|
235
|
-
},
|
|
236
|
-
behavior: 'body-only',
|
|
237
|
-
timestamps: false,
|
|
238
|
-
createdBy: 'EventualConsistencyPlugin'
|
|
239
|
-
})
|
|
240
|
-
);
|
|
241
|
-
|
|
242
|
-
if (!lockOk && !this.database.resources[lockResourceName]) {
|
|
243
|
-
throw new Error(`Failed to create lock resource for ${resourceName}.${fieldName}: ${lockErr?.message}`);
|
|
244
|
-
}
|
|
245
|
-
|
|
246
|
-
handler.lockResource = lockOk ? lockResource : this.database.resources[lockResourceName];
|
|
247
|
-
|
|
248
|
-
// Create analytics resource if enabled
|
|
249
|
-
if (config.enableAnalytics) {
|
|
250
|
-
await this._createAnalyticsResourceForHandler(handler);
|
|
251
|
-
}
|
|
252
|
-
|
|
253
|
-
// Add helper methods to the target resource
|
|
254
|
-
this._addHelperMethodsForHandler(handler);
|
|
255
|
-
|
|
256
|
-
// Setup timers (TODO: implement timer management for handlers)
|
|
257
|
-
// For now, we'll skip auto-consolidation in multi-resource mode
|
|
258
|
-
|
|
259
|
-
if (config.verbose) {
|
|
260
|
-
console.log(
|
|
261
|
-
`[EventualConsistency] ${resourceName}.${fieldName} - ` +
|
|
262
|
-
`Setup complete. Resources: ${transactionResourceName}, ${lockResourceName}` +
|
|
263
|
-
`${config.enableAnalytics ? `, ${resourceName}_analytics_${fieldName}` : ''}`
|
|
264
|
-
);
|
|
265
|
-
}
|
|
266
|
-
}
|
|
267
|
-
|
|
268
|
-
/**
|
|
269
|
-
* Create analytics resource for a field handler
|
|
270
|
-
* @private
|
|
271
|
-
*/
|
|
272
|
-
async _createAnalyticsResourceForHandler(handler) {
|
|
273
|
-
const resourceName = handler.resource;
|
|
274
|
-
const fieldName = handler.field;
|
|
275
|
-
const analyticsResourceName = `${resourceName}_analytics_${fieldName}`;
|
|
276
|
-
|
|
277
|
-
const [ok, err, analyticsResource] = await tryFn(() =>
|
|
278
|
-
this.database.createResource({
|
|
279
|
-
name: analyticsResourceName,
|
|
280
|
-
attributes: {
|
|
281
|
-
id: 'string|required',
|
|
282
|
-
period: 'string|required',
|
|
283
|
-
cohort: 'string|required',
|
|
284
|
-
transactionCount: 'number|required',
|
|
285
|
-
totalValue: 'number|required',
|
|
286
|
-
avgValue: 'number|required',
|
|
287
|
-
minValue: 'number|required',
|
|
288
|
-
maxValue: 'number|required',
|
|
289
|
-
operations: 'object|optional',
|
|
290
|
-
recordCount: 'number|required',
|
|
291
|
-
consolidatedAt: 'string|required',
|
|
292
|
-
updatedAt: 'string|required'
|
|
293
|
-
},
|
|
294
|
-
behavior: 'body-overflow',
|
|
295
|
-
timestamps: false,
|
|
296
|
-
createdBy: 'EventualConsistencyPlugin'
|
|
297
|
-
})
|
|
298
|
-
);
|
|
299
|
-
|
|
300
|
-
if (!ok && !this.database.resources[analyticsResourceName]) {
|
|
301
|
-
throw new Error(`Failed to create analytics resource for ${resourceName}.${fieldName}: ${err?.message}`);
|
|
302
|
-
}
|
|
303
|
-
|
|
304
|
-
handler.analyticsResource = ok ? analyticsResource : this.database.resources[analyticsResourceName];
|
|
305
|
-
}
|
|
306
|
-
|
|
307
|
-
/**
|
|
308
|
-
* Add helper methods to the target resource for a field handler
|
|
309
|
-
* @private
|
|
310
|
-
*/
|
|
311
|
-
_addHelperMethodsForHandler(handler) {
|
|
312
|
-
const resource = handler.targetResource;
|
|
313
|
-
const fieldName = handler.field;
|
|
314
|
-
|
|
315
|
-
// Store handler reference on the resource for later access
|
|
316
|
-
if (!resource._eventualConsistencyPlugins) {
|
|
317
|
-
resource._eventualConsistencyPlugins = {};
|
|
318
|
-
}
|
|
319
|
-
resource._eventualConsistencyPlugins[fieldName] = handler;
|
|
320
|
-
|
|
321
|
-
// Add helper methods if not already added
|
|
322
|
-
if (!resource.add) {
|
|
323
|
-
this.addHelperMethods(); // Add all helper methods once
|
|
324
|
-
}
|
|
325
|
-
}
|
|
326
|
-
|
|
327
|
-
async onStart() {
|
|
328
|
-
// Start timers and emit events for all field handlers
|
|
329
|
-
for (const [resourceName, fieldHandlers] of this.fieldHandlers) {
|
|
330
|
-
for (const [fieldName, handler] of fieldHandlers) {
|
|
331
|
-
if (!handler.deferredSetup) {
|
|
332
|
-
// Start auto-consolidation timer if enabled
|
|
333
|
-
if (this.config.autoConsolidate && this.config.mode === 'async') {
|
|
334
|
-
this.startConsolidationTimerForHandler(handler, resourceName, fieldName);
|
|
335
|
-
}
|
|
336
|
-
|
|
337
|
-
// Start garbage collection timer
|
|
338
|
-
if (this.config.transactionRetention && this.config.transactionRetention > 0) {
|
|
339
|
-
this.startGarbageCollectionTimerForHandler(handler, resourceName, fieldName);
|
|
340
|
-
}
|
|
341
|
-
|
|
342
|
-
this.emit('eventual-consistency.started', {
|
|
343
|
-
resource: resourceName,
|
|
344
|
-
field: fieldName,
|
|
345
|
-
cohort: this.config.cohort
|
|
346
|
-
});
|
|
347
|
-
}
|
|
348
|
-
}
|
|
349
|
-
}
|
|
350
|
-
}
|
|
351
|
-
|
|
352
|
-
async onStop() {
|
|
353
|
-
// Stop all timers for all handlers
|
|
354
|
-
for (const [resourceName, fieldHandlers] of this.fieldHandlers) {
|
|
355
|
-
for (const [fieldName, handler] of fieldHandlers) {
|
|
356
|
-
// Stop consolidation timer
|
|
357
|
-
if (handler.consolidationTimer) {
|
|
358
|
-
clearInterval(handler.consolidationTimer);
|
|
359
|
-
handler.consolidationTimer = null;
|
|
360
|
-
}
|
|
361
|
-
|
|
362
|
-
// Stop garbage collection timer
|
|
363
|
-
if (handler.gcTimer) {
|
|
364
|
-
clearInterval(handler.gcTimer);
|
|
365
|
-
handler.gcTimer = null;
|
|
366
|
-
}
|
|
367
|
-
|
|
368
|
-
// Flush pending transactions
|
|
369
|
-
if (handler.pendingTransactions && handler.pendingTransactions.size > 0) {
|
|
370
|
-
await this._flushPendingTransactions(handler);
|
|
371
|
-
}
|
|
372
|
-
|
|
373
|
-
this.emit('eventual-consistency.stopped', {
|
|
374
|
-
resource: resourceName,
|
|
375
|
-
field: fieldName
|
|
376
|
-
});
|
|
377
|
-
}
|
|
378
|
-
}
|
|
379
|
-
}
|
|
380
|
-
|
|
381
|
-
createPartitionConfig() {
|
|
382
|
-
// Create partitions for transactions
|
|
383
|
-
const partitions = {
|
|
384
|
-
// Composite partition by originalId + applied status
|
|
385
|
-
// This is THE MOST CRITICAL optimization for consolidation!
|
|
386
|
-
// Why: Consolidation always queries { originalId, applied: false }
|
|
387
|
-
// Without this: Reads ALL transactions (applied + pending) and filters manually
|
|
388
|
-
// With this: Reads ONLY pending transactions - can be 1000x faster!
|
|
389
|
-
byOriginalIdAndApplied: {
|
|
390
|
-
fields: {
|
|
391
|
-
originalId: 'string',
|
|
392
|
-
applied: 'boolean'
|
|
393
|
-
}
|
|
394
|
-
},
|
|
395
|
-
// Partition by time cohorts for batch consolidation across many records
|
|
396
|
-
byHour: {
|
|
397
|
-
fields: {
|
|
398
|
-
cohortHour: 'string'
|
|
399
|
-
}
|
|
400
|
-
},
|
|
401
|
-
byDay: {
|
|
402
|
-
fields: {
|
|
403
|
-
cohortDate: 'string'
|
|
404
|
-
}
|
|
405
|
-
},
|
|
406
|
-
byMonth: {
|
|
407
|
-
fields: {
|
|
408
|
-
cohortMonth: 'string'
|
|
409
|
-
}
|
|
410
|
-
}
|
|
411
|
-
};
|
|
412
|
-
|
|
413
|
-
return partitions;
|
|
414
|
-
}
|
|
415
|
-
|
|
416
|
-
/**
|
|
417
|
-
* Auto-detect timezone from environment or system
|
|
418
|
-
* @private
|
|
419
|
-
*/
|
|
420
|
-
_detectTimezone() {
|
|
421
|
-
// 1. Try TZ environment variable (common in Docker/K8s)
|
|
422
|
-
if (process.env.TZ) {
|
|
423
|
-
return process.env.TZ;
|
|
424
|
-
}
|
|
425
|
-
|
|
426
|
-
// 2. Try Intl API (works in Node.js and browsers)
|
|
427
|
-
try {
|
|
428
|
-
const systemTimezone = Intl.DateTimeFormat().resolvedOptions().timeZone;
|
|
429
|
-
if (systemTimezone) {
|
|
430
|
-
return systemTimezone;
|
|
431
|
-
}
|
|
432
|
-
} catch (err) {
|
|
433
|
-
// Intl API not available or failed
|
|
434
|
-
}
|
|
435
|
-
|
|
436
|
-
// 3. Fallback to UTC
|
|
437
|
-
return 'UTC';
|
|
438
|
-
}
|
|
439
|
-
|
|
440
|
-
/**
|
|
441
|
-
* Helper method to resolve field and plugin from arguments
|
|
442
|
-
* @private
|
|
443
|
-
*/
|
|
444
|
-
_resolveFieldAndPlugin(resource, field, value) {
|
|
445
|
-
if (!resource._eventualConsistencyPlugins) {
|
|
446
|
-
throw new Error(`No eventual consistency plugins configured for this resource`);
|
|
447
|
-
}
|
|
448
|
-
|
|
449
|
-
const fieldPlugin = resource._eventualConsistencyPlugins[field];
|
|
450
|
-
|
|
451
|
-
if (!fieldPlugin) {
|
|
452
|
-
const availableFields = Object.keys(resource._eventualConsistencyPlugins).join(', ');
|
|
453
|
-
throw new Error(
|
|
454
|
-
`No eventual consistency plugin found for field "${field}". ` +
|
|
455
|
-
`Available fields: ${availableFields}`
|
|
456
|
-
);
|
|
457
|
-
}
|
|
458
|
-
|
|
459
|
-
return { field, value, plugin: fieldPlugin };
|
|
460
|
-
}
|
|
461
|
-
|
|
462
|
-
/**
|
|
463
|
-
* Helper method to perform atomic consolidation in sync mode
|
|
464
|
-
* @private
|
|
465
|
-
*/
|
|
466
|
-
async _syncModeConsolidate(id, field) {
|
|
467
|
-
// consolidateRecord already has distributed locking and handles persistence (upsert)
|
|
468
|
-
const consolidatedValue = await this.consolidateRecord(id);
|
|
469
|
-
return consolidatedValue;
|
|
470
|
-
}
|
|
471
|
-
|
|
472
|
-
/**
|
|
473
|
-
* Create synthetic 'set' transaction from current value
|
|
474
|
-
* @private
|
|
475
|
-
*/
|
|
476
|
-
_createSyntheticSetTransaction(currentValue) {
|
|
477
|
-
return {
|
|
478
|
-
id: '__synthetic__',
|
|
479
|
-
operation: 'set',
|
|
480
|
-
value: currentValue,
|
|
481
|
-
timestamp: new Date(0).toISOString(),
|
|
482
|
-
synthetic: true
|
|
483
|
-
};
|
|
484
|
-
}
|
|
485
|
-
|
|
486
|
-
addHelperMethods() {
|
|
487
|
-
// Get any handler from the first resource to access the resource instance
|
|
488
|
-
const firstResource = this.fieldHandlers.values().next().value;
|
|
489
|
-
if (!firstResource) return;
|
|
490
|
-
|
|
491
|
-
const firstHandler = firstResource.values().next().value;
|
|
492
|
-
if (!firstHandler || !firstHandler.targetResource) return;
|
|
493
|
-
|
|
494
|
-
const resource = firstHandler.targetResource;
|
|
495
|
-
const plugin = this;
|
|
496
|
-
|
|
497
|
-
// Add method to set value (replaces current value)
|
|
498
|
-
// Signature: set(id, field, value)
|
|
499
|
-
resource.set = async (id, field, value) => {
|
|
500
|
-
const { plugin: handler } =
|
|
501
|
-
plugin._resolveFieldAndPlugin(resource, field, value);
|
|
502
|
-
|
|
503
|
-
// Create transaction inline
|
|
504
|
-
const now = new Date();
|
|
505
|
-
const cohortInfo = plugin.getCohortInfo(now);
|
|
506
|
-
|
|
507
|
-
const transaction = {
|
|
508
|
-
id: idGenerator(),
|
|
509
|
-
originalId: id,
|
|
510
|
-
field: handler.field,
|
|
511
|
-
value: value,
|
|
512
|
-
operation: 'set',
|
|
513
|
-
timestamp: now.toISOString(),
|
|
514
|
-
cohortDate: cohortInfo.date,
|
|
515
|
-
cohortHour: cohortInfo.hour,
|
|
516
|
-
cohortMonth: cohortInfo.month,
|
|
517
|
-
source: 'set',
|
|
518
|
-
applied: false
|
|
519
|
-
};
|
|
520
|
-
|
|
521
|
-
await handler.transactionResource.insert(transaction);
|
|
522
|
-
|
|
523
|
-
// In sync mode, immediately consolidate
|
|
524
|
-
if (plugin.config.mode === 'sync') {
|
|
525
|
-
// Temporarily set config for legacy methods
|
|
526
|
-
const oldResource = plugin.config.resource;
|
|
527
|
-
const oldField = plugin.config.field;
|
|
528
|
-
const oldTransactionResource = plugin.transactionResource;
|
|
529
|
-
const oldTargetResource = plugin.targetResource;
|
|
530
|
-
const oldLockResource = plugin.lockResource;
|
|
531
|
-
const oldAnalyticsResource = plugin.analyticsResource;
|
|
532
|
-
|
|
533
|
-
plugin.config.resource = handler.resource;
|
|
534
|
-
plugin.config.field = handler.field;
|
|
535
|
-
plugin.transactionResource = handler.transactionResource;
|
|
536
|
-
plugin.targetResource = handler.targetResource;
|
|
537
|
-
plugin.lockResource = handler.lockResource;
|
|
538
|
-
plugin.analyticsResource = handler.analyticsResource;
|
|
539
|
-
|
|
540
|
-
const result = await plugin._syncModeConsolidate(id, field);
|
|
541
|
-
|
|
542
|
-
// Restore
|
|
543
|
-
plugin.config.resource = oldResource;
|
|
544
|
-
plugin.config.field = oldField;
|
|
545
|
-
plugin.transactionResource = oldTransactionResource;
|
|
546
|
-
plugin.targetResource = oldTargetResource;
|
|
547
|
-
plugin.lockResource = oldLockResource;
|
|
548
|
-
plugin.analyticsResource = oldAnalyticsResource;
|
|
549
|
-
|
|
550
|
-
return result;
|
|
551
|
-
}
|
|
552
|
-
|
|
553
|
-
return value;
|
|
554
|
-
};
|
|
555
|
-
|
|
556
|
-
// Add method to increment value
|
|
557
|
-
// Signature: add(id, field, amount)
|
|
558
|
-
resource.add = async (id, field, amount) => {
|
|
559
|
-
const { plugin: handler } =
|
|
560
|
-
plugin._resolveFieldAndPlugin(resource, field, amount);
|
|
561
|
-
|
|
562
|
-
// Create transaction inline
|
|
563
|
-
const now = new Date();
|
|
564
|
-
const cohortInfo = plugin.getCohortInfo(now);
|
|
565
|
-
|
|
566
|
-
const transaction = {
|
|
567
|
-
id: idGenerator(),
|
|
568
|
-
originalId: id,
|
|
569
|
-
field: handler.field,
|
|
570
|
-
value: amount,
|
|
571
|
-
operation: 'add',
|
|
572
|
-
timestamp: now.toISOString(),
|
|
573
|
-
cohortDate: cohortInfo.date,
|
|
574
|
-
cohortHour: cohortInfo.hour,
|
|
575
|
-
cohortMonth: cohortInfo.month,
|
|
576
|
-
source: 'add',
|
|
577
|
-
applied: false
|
|
578
|
-
};
|
|
579
|
-
|
|
580
|
-
await handler.transactionResource.insert(transaction);
|
|
581
|
-
|
|
582
|
-
// In sync mode, immediately consolidate
|
|
583
|
-
if (plugin.config.mode === 'sync') {
|
|
584
|
-
const oldResource = plugin.config.resource;
|
|
585
|
-
const oldField = plugin.config.field;
|
|
586
|
-
const oldTransactionResource = plugin.transactionResource;
|
|
587
|
-
const oldTargetResource = plugin.targetResource;
|
|
588
|
-
const oldLockResource = plugin.lockResource;
|
|
589
|
-
const oldAnalyticsResource = plugin.analyticsResource;
|
|
590
|
-
|
|
591
|
-
plugin.config.resource = handler.resource;
|
|
592
|
-
plugin.config.field = handler.field;
|
|
593
|
-
plugin.transactionResource = handler.transactionResource;
|
|
594
|
-
plugin.targetResource = handler.targetResource;
|
|
595
|
-
plugin.lockResource = handler.lockResource;
|
|
596
|
-
plugin.analyticsResource = handler.analyticsResource;
|
|
597
|
-
|
|
598
|
-
const result = await plugin._syncModeConsolidate(id, field);
|
|
599
|
-
|
|
600
|
-
plugin.config.resource = oldResource;
|
|
601
|
-
plugin.config.field = oldField;
|
|
602
|
-
plugin.transactionResource = oldTransactionResource;
|
|
603
|
-
plugin.targetResource = oldTargetResource;
|
|
604
|
-
plugin.lockResource = oldLockResource;
|
|
605
|
-
plugin.analyticsResource = oldAnalyticsResource;
|
|
606
|
-
|
|
607
|
-
return result;
|
|
608
|
-
}
|
|
609
|
-
|
|
610
|
-
// Async mode - return current value (optimistic)
|
|
611
|
-
const [ok, err, record] = await tryFn(() => handler.targetResource.get(id));
|
|
612
|
-
const currentValue = (ok && record) ? (record[field] || 0) : 0;
|
|
613
|
-
return currentValue + amount;
|
|
614
|
-
};
|
|
615
|
-
|
|
616
|
-
// Add method to decrement value
|
|
617
|
-
// Signature: sub(id, field, amount)
|
|
618
|
-
resource.sub = async (id, field, amount) => {
|
|
619
|
-
const { plugin: handler } =
|
|
620
|
-
plugin._resolveFieldAndPlugin(resource, field, amount);
|
|
621
|
-
|
|
622
|
-
// Create transaction inline
|
|
623
|
-
const now = new Date();
|
|
624
|
-
const cohortInfo = plugin.getCohortInfo(now);
|
|
625
|
-
|
|
626
|
-
const transaction = {
|
|
627
|
-
id: idGenerator(),
|
|
628
|
-
originalId: id,
|
|
629
|
-
field: handler.field,
|
|
630
|
-
value: amount,
|
|
631
|
-
operation: 'sub',
|
|
632
|
-
timestamp: now.toISOString(),
|
|
633
|
-
cohortDate: cohortInfo.date,
|
|
634
|
-
cohortHour: cohortInfo.hour,
|
|
635
|
-
cohortMonth: cohortInfo.month,
|
|
636
|
-
source: 'sub',
|
|
637
|
-
applied: false
|
|
638
|
-
};
|
|
639
|
-
|
|
640
|
-
await handler.transactionResource.insert(transaction);
|
|
641
|
-
|
|
642
|
-
// In sync mode, immediately consolidate
|
|
643
|
-
if (plugin.config.mode === 'sync') {
|
|
644
|
-
const oldResource = plugin.config.resource;
|
|
645
|
-
const oldField = plugin.config.field;
|
|
646
|
-
const oldTransactionResource = plugin.transactionResource;
|
|
647
|
-
const oldTargetResource = plugin.targetResource;
|
|
648
|
-
const oldLockResource = plugin.lockResource;
|
|
649
|
-
const oldAnalyticsResource = plugin.analyticsResource;
|
|
650
|
-
|
|
651
|
-
plugin.config.resource = handler.resource;
|
|
652
|
-
plugin.config.field = handler.field;
|
|
653
|
-
plugin.transactionResource = handler.transactionResource;
|
|
654
|
-
plugin.targetResource = handler.targetResource;
|
|
655
|
-
plugin.lockResource = handler.lockResource;
|
|
656
|
-
plugin.analyticsResource = handler.analyticsResource;
|
|
657
|
-
|
|
658
|
-
const result = await plugin._syncModeConsolidate(id, field);
|
|
659
|
-
|
|
660
|
-
plugin.config.resource = oldResource;
|
|
661
|
-
plugin.config.field = oldField;
|
|
662
|
-
plugin.transactionResource = oldTransactionResource;
|
|
663
|
-
plugin.targetResource = oldTargetResource;
|
|
664
|
-
plugin.lockResource = oldLockResource;
|
|
665
|
-
plugin.analyticsResource = oldAnalyticsResource;
|
|
666
|
-
|
|
667
|
-
return result;
|
|
668
|
-
}
|
|
669
|
-
|
|
670
|
-
// Async mode - return current value (optimistic)
|
|
671
|
-
const [ok, err, record] = await tryFn(() => handler.targetResource.get(id));
|
|
672
|
-
const currentValue = (ok && record) ? (record[field] || 0) : 0;
|
|
673
|
-
return currentValue - amount;
|
|
674
|
-
};
|
|
675
|
-
|
|
676
|
-
// Add method to manually trigger consolidation
|
|
677
|
-
// Signature: consolidate(id, field)
|
|
678
|
-
resource.consolidate = async (id, field) => {
|
|
679
|
-
if (!field) {
|
|
680
|
-
throw new Error(`Field parameter is required: consolidate(id, field)`);
|
|
681
|
-
}
|
|
682
|
-
|
|
683
|
-
const handler = resource._eventualConsistencyPlugins[field];
|
|
684
|
-
|
|
685
|
-
if (!handler) {
|
|
686
|
-
const availableFields = Object.keys(resource._eventualConsistencyPlugins).join(', ');
|
|
687
|
-
throw new Error(
|
|
688
|
-
`No eventual consistency plugin found for field "${field}". ` +
|
|
689
|
-
`Available fields: ${availableFields}`
|
|
690
|
-
);
|
|
691
|
-
}
|
|
692
|
-
|
|
693
|
-
// Temporarily set config for legacy methods
|
|
694
|
-
const oldResource = plugin.config.resource;
|
|
695
|
-
const oldField = plugin.config.field;
|
|
696
|
-
const oldTransactionResource = plugin.transactionResource;
|
|
697
|
-
const oldTargetResource = plugin.targetResource;
|
|
698
|
-
const oldLockResource = plugin.lockResource;
|
|
699
|
-
const oldAnalyticsResource = plugin.analyticsResource;
|
|
700
|
-
|
|
701
|
-
plugin.config.resource = handler.resource;
|
|
702
|
-
plugin.config.field = handler.field;
|
|
703
|
-
plugin.transactionResource = handler.transactionResource;
|
|
704
|
-
plugin.targetResource = handler.targetResource;
|
|
705
|
-
plugin.lockResource = handler.lockResource;
|
|
706
|
-
plugin.analyticsResource = handler.analyticsResource;
|
|
707
|
-
|
|
708
|
-
const result = await plugin.consolidateRecord(id);
|
|
709
|
-
|
|
710
|
-
plugin.config.resource = oldResource;
|
|
711
|
-
plugin.config.field = oldField;
|
|
712
|
-
plugin.transactionResource = oldTransactionResource;
|
|
713
|
-
plugin.targetResource = oldTargetResource;
|
|
714
|
-
plugin.lockResource = oldLockResource;
|
|
715
|
-
plugin.analyticsResource = oldAnalyticsResource;
|
|
716
|
-
|
|
717
|
-
return result;
|
|
718
|
-
};
|
|
719
|
-
|
|
720
|
-
// Add method to get consolidated value without applying
|
|
721
|
-
// Signature: getConsolidatedValue(id, field, options)
|
|
722
|
-
resource.getConsolidatedValue = async (id, field, options = {}) => {
|
|
723
|
-
const handler = resource._eventualConsistencyPlugins[field];
|
|
724
|
-
|
|
725
|
-
if (!handler) {
|
|
726
|
-
const availableFields = Object.keys(resource._eventualConsistencyPlugins).join(', ');
|
|
727
|
-
throw new Error(
|
|
728
|
-
`No eventual consistency plugin found for field "${field}". ` +
|
|
729
|
-
`Available fields: ${availableFields}`
|
|
730
|
-
);
|
|
731
|
-
}
|
|
732
|
-
|
|
733
|
-
// Temporarily set config for legacy methods
|
|
734
|
-
const oldResource = plugin.config.resource;
|
|
735
|
-
const oldField = plugin.config.field;
|
|
736
|
-
const oldTransactionResource = plugin.transactionResource;
|
|
737
|
-
const oldTargetResource = plugin.targetResource;
|
|
738
|
-
|
|
739
|
-
plugin.config.resource = handler.resource;
|
|
740
|
-
plugin.config.field = handler.field;
|
|
741
|
-
plugin.transactionResource = handler.transactionResource;
|
|
742
|
-
plugin.targetResource = handler.targetResource;
|
|
743
|
-
|
|
744
|
-
const result = await plugin.getConsolidatedValue(id, options);
|
|
745
|
-
|
|
746
|
-
plugin.config.resource = oldResource;
|
|
747
|
-
plugin.config.field = oldField;
|
|
748
|
-
plugin.transactionResource = oldTransactionResource;
|
|
749
|
-
plugin.targetResource = oldTargetResource;
|
|
750
|
-
|
|
751
|
-
return result;
|
|
752
|
-
};
|
|
753
|
-
|
|
754
|
-
// Add method to recalculate from scratch
|
|
755
|
-
// Signature: recalculate(id, field)
|
|
756
|
-
resource.recalculate = async (id, field) => {
|
|
757
|
-
if (!field) {
|
|
758
|
-
throw new Error(`Field parameter is required: recalculate(id, field)`);
|
|
759
|
-
}
|
|
760
|
-
|
|
761
|
-
const handler = resource._eventualConsistencyPlugins[field];
|
|
762
|
-
|
|
763
|
-
if (!handler) {
|
|
764
|
-
const availableFields = Object.keys(resource._eventualConsistencyPlugins).join(', ');
|
|
765
|
-
throw new Error(
|
|
766
|
-
`No eventual consistency plugin found for field "${field}". ` +
|
|
767
|
-
`Available fields: ${availableFields}`
|
|
768
|
-
);
|
|
769
|
-
}
|
|
770
|
-
|
|
771
|
-
// Temporarily set config for legacy methods
|
|
772
|
-
const oldResource = plugin.config.resource;
|
|
773
|
-
const oldField = plugin.config.field;
|
|
774
|
-
const oldTransactionResource = plugin.transactionResource;
|
|
775
|
-
const oldTargetResource = plugin.targetResource;
|
|
776
|
-
const oldLockResource = plugin.lockResource;
|
|
777
|
-
const oldAnalyticsResource = plugin.analyticsResource;
|
|
778
|
-
|
|
779
|
-
plugin.config.resource = handler.resource;
|
|
780
|
-
plugin.config.field = handler.field;
|
|
781
|
-
plugin.transactionResource = handler.transactionResource;
|
|
782
|
-
plugin.targetResource = handler.targetResource;
|
|
783
|
-
plugin.lockResource = handler.lockResource;
|
|
784
|
-
plugin.analyticsResource = handler.analyticsResource;
|
|
785
|
-
|
|
786
|
-
const result = await plugin.recalculateRecord(id);
|
|
787
|
-
|
|
788
|
-
plugin.config.resource = oldResource;
|
|
789
|
-
plugin.config.field = oldField;
|
|
790
|
-
plugin.transactionResource = oldTransactionResource;
|
|
791
|
-
plugin.targetResource = oldTargetResource;
|
|
792
|
-
plugin.lockResource = oldLockResource;
|
|
793
|
-
plugin.analyticsResource = oldAnalyticsResource;
|
|
794
|
-
|
|
795
|
-
return result;
|
|
796
|
-
};
|
|
797
|
-
}
|
|
798
|
-
|
|
799
|
-
async createTransaction(handler, data) {
|
|
800
|
-
const now = new Date();
|
|
801
|
-
const cohortInfo = this.getCohortInfo(now);
|
|
802
|
-
|
|
803
|
-
// Check for late arrivals (transaction older than watermark)
|
|
804
|
-
const watermarkMs = this.config.consolidationWindow * 60 * 60 * 1000;
|
|
805
|
-
const watermarkTime = now.getTime() - watermarkMs;
|
|
806
|
-
const cohortHourDate = new Date(cohortInfo.hour + ':00:00Z');
|
|
807
|
-
|
|
808
|
-
if (cohortHourDate.getTime() < watermarkTime) {
|
|
809
|
-
// Late arrival detected!
|
|
810
|
-
const hoursLate = Math.floor((now.getTime() - cohortHourDate.getTime()) / (60 * 60 * 1000));
|
|
811
|
-
|
|
812
|
-
if (this.config.lateArrivalStrategy === 'ignore') {
|
|
813
|
-
if (this.config.verbose) {
|
|
814
|
-
console.warn(
|
|
815
|
-
`[EventualConsistency] Late arrival ignored: transaction for ${cohortInfo.hour} ` +
|
|
816
|
-
`is ${hoursLate}h late (watermark: ${this.config.consolidationWindow}h)`
|
|
817
|
-
);
|
|
818
|
-
}
|
|
819
|
-
return null;
|
|
820
|
-
} else if (this.config.lateArrivalStrategy === 'warn') {
|
|
821
|
-
console.warn(
|
|
822
|
-
`[EventualConsistency] Late arrival detected: transaction for ${cohortInfo.hour} ` +
|
|
823
|
-
`is ${hoursLate}h late (watermark: ${this.config.consolidationWindow}h). ` +
|
|
824
|
-
`Processing anyway, but consolidation may not pick it up.`
|
|
825
|
-
);
|
|
826
|
-
}
|
|
827
|
-
// 'process' strategy: continue normally
|
|
828
|
-
}
|
|
829
|
-
|
|
830
|
-
const transaction = {
|
|
831
|
-
id: idGenerator(),
|
|
832
|
-
originalId: data.originalId,
|
|
833
|
-
field: handler.field,
|
|
834
|
-
value: data.value || 0,
|
|
835
|
-
operation: data.operation || 'set',
|
|
836
|
-
timestamp: now.toISOString(),
|
|
837
|
-
cohortDate: cohortInfo.date,
|
|
838
|
-
cohortHour: cohortInfo.hour,
|
|
839
|
-
cohortMonth: cohortInfo.month,
|
|
840
|
-
source: data.source || 'unknown',
|
|
841
|
-
applied: false
|
|
842
|
-
};
|
|
843
|
-
|
|
844
|
-
// Batch transactions if configured
|
|
845
|
-
if (this.config.batchTransactions) {
|
|
846
|
-
handler.pendingTransactions.set(transaction.id, transaction);
|
|
847
|
-
|
|
848
|
-
if (this.config.verbose) {
|
|
849
|
-
console.log(
|
|
850
|
-
`[EventualConsistency] ${handler.resource}.${handler.field} - ` +
|
|
851
|
-
`Transaction batched: ${data.operation} ${data.value} for ${data.originalId} ` +
|
|
852
|
-
`(batch: ${handler.pendingTransactions.size}/${this.config.batchSize})`
|
|
853
|
-
);
|
|
854
|
-
}
|
|
855
|
-
|
|
856
|
-
// Flush if batch size reached
|
|
857
|
-
if (handler.pendingTransactions.size >= this.config.batchSize) {
|
|
858
|
-
await this._flushPendingTransactions(handler);
|
|
859
|
-
}
|
|
860
|
-
} else {
|
|
861
|
-
await handler.transactionResource.insert(transaction);
|
|
862
|
-
|
|
863
|
-
if (this.config.verbose) {
|
|
864
|
-
console.log(
|
|
865
|
-
`[EventualConsistency] ${handler.resource}.${handler.field} - ` +
|
|
866
|
-
`Transaction created: ${data.operation} ${data.value} for ${data.originalId} ` +
|
|
867
|
-
`(cohort: ${cohortInfo.hour}, applied: false)`
|
|
868
|
-
);
|
|
869
|
-
}
|
|
870
|
-
}
|
|
871
|
-
|
|
872
|
-
return transaction;
|
|
873
|
-
}
|
|
874
|
-
|
|
875
|
-
async flushPendingTransactions() {
|
|
876
|
-
if (this.pendingTransactions.size === 0) return;
|
|
877
|
-
|
|
878
|
-
const transactions = Array.from(this.pendingTransactions.values());
|
|
879
|
-
|
|
880
|
-
try {
|
|
881
|
-
// Insert all pending transactions in parallel
|
|
882
|
-
await Promise.all(
|
|
883
|
-
transactions.map(transaction =>
|
|
884
|
-
this.transactionResource.insert(transaction)
|
|
885
|
-
)
|
|
886
|
-
);
|
|
887
|
-
|
|
888
|
-
// Only clear after successful inserts (prevents data loss on crashes)
|
|
889
|
-
this.pendingTransactions.clear();
|
|
890
|
-
} catch (error) {
|
|
891
|
-
// Keep pending transactions for retry on next flush
|
|
892
|
-
console.error('Failed to flush pending transactions:', error);
|
|
893
|
-
throw error;
|
|
894
|
-
}
|
|
895
|
-
}
|
|
896
|
-
|
|
897
|
-
getCohortInfo(date) {
|
|
898
|
-
const tz = this.config.cohort.timezone;
|
|
899
|
-
|
|
900
|
-
// Simple timezone offset calculation (can be enhanced with a library)
|
|
901
|
-
const offset = this.getTimezoneOffset(tz);
|
|
902
|
-
const localDate = new Date(date.getTime() + offset);
|
|
903
|
-
|
|
904
|
-
const year = localDate.getFullYear();
|
|
905
|
-
const month = String(localDate.getMonth() + 1).padStart(2, '0');
|
|
906
|
-
const day = String(localDate.getDate()).padStart(2, '0');
|
|
907
|
-
const hour = String(localDate.getHours()).padStart(2, '0');
|
|
908
|
-
|
|
909
|
-
return {
|
|
910
|
-
date: `${year}-${month}-${day}`,
|
|
911
|
-
hour: `${year}-${month}-${day}T${hour}`, // ISO-like format for hour partition
|
|
912
|
-
month: `${year}-${month}`
|
|
913
|
-
};
|
|
914
|
-
}
|
|
915
|
-
|
|
916
|
-
getTimezoneOffset(timezone) {
|
|
917
|
-
// Try to calculate offset using Intl API (handles DST automatically)
|
|
918
|
-
try {
|
|
919
|
-
const now = new Date();
|
|
920
|
-
|
|
921
|
-
// Get UTC time
|
|
922
|
-
const utcDate = new Date(now.toLocaleString('en-US', { timeZone: 'UTC' }));
|
|
923
|
-
|
|
924
|
-
// Get time in target timezone
|
|
925
|
-
const tzDate = new Date(now.toLocaleString('en-US', { timeZone: timezone }));
|
|
926
|
-
|
|
927
|
-
// Calculate offset in milliseconds
|
|
928
|
-
return tzDate.getTime() - utcDate.getTime();
|
|
929
|
-
} catch (err) {
|
|
930
|
-
// Intl API failed, fallback to manual offsets (without DST support)
|
|
931
|
-
const offsets = {
|
|
932
|
-
'UTC': 0,
|
|
933
|
-
'America/New_York': -5 * 3600000,
|
|
934
|
-
'America/Chicago': -6 * 3600000,
|
|
935
|
-
'America/Denver': -7 * 3600000,
|
|
936
|
-
'America/Los_Angeles': -8 * 3600000,
|
|
937
|
-
'America/Sao_Paulo': -3 * 3600000,
|
|
938
|
-
'Europe/London': 0,
|
|
939
|
-
'Europe/Paris': 1 * 3600000,
|
|
940
|
-
'Europe/Berlin': 1 * 3600000,
|
|
941
|
-
'Asia/Tokyo': 9 * 3600000,
|
|
942
|
-
'Asia/Shanghai': 8 * 3600000,
|
|
943
|
-
'Australia/Sydney': 10 * 3600000
|
|
944
|
-
};
|
|
945
|
-
|
|
946
|
-
if (this.config.verbose && !offsets[timezone]) {
|
|
947
|
-
console.warn(
|
|
948
|
-
`[EventualConsistency] Unknown timezone '${timezone}', using UTC. ` +
|
|
949
|
-
`Consider using a valid IANA timezone (e.g., 'America/New_York')`
|
|
950
|
-
);
|
|
951
|
-
}
|
|
952
|
-
|
|
953
|
-
return offsets[timezone] || 0;
|
|
954
|
-
}
|
|
955
|
-
}
|
|
956
|
-
|
|
957
|
-
startConsolidationTimer() {
|
|
958
|
-
const intervalMs = this.config.consolidationInterval * 1000; // Convert seconds to ms
|
|
959
|
-
|
|
960
|
-
if (this.config.verbose) {
|
|
961
|
-
const nextRun = new Date(Date.now() + intervalMs);
|
|
962
|
-
console.log(
|
|
963
|
-
`[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
|
|
964
|
-
`Consolidation timer started. Next run at ${nextRun.toISOString()} ` +
|
|
965
|
-
`(every ${this.config.consolidationInterval}s)`
|
|
966
|
-
);
|
|
967
|
-
}
|
|
968
|
-
|
|
969
|
-
this.consolidationTimer = setInterval(async () => {
|
|
970
|
-
await this.runConsolidation();
|
|
971
|
-
}, intervalMs);
|
|
972
|
-
}
|
|
973
|
-
|
|
974
|
-
startConsolidationTimerForHandler(handler, resourceName, fieldName) {
|
|
975
|
-
const intervalMs = this.config.consolidationInterval * 1000; // Convert seconds to ms
|
|
976
|
-
|
|
977
|
-
if (this.config.verbose) {
|
|
978
|
-
const nextRun = new Date(Date.now() + intervalMs);
|
|
979
|
-
console.log(
|
|
980
|
-
`[EventualConsistency] ${resourceName}.${fieldName} - ` +
|
|
981
|
-
`Consolidation timer started. Next run at ${nextRun.toISOString()} ` +
|
|
982
|
-
`(every ${this.config.consolidationInterval}s)`
|
|
983
|
-
);
|
|
984
|
-
}
|
|
985
|
-
|
|
986
|
-
handler.consolidationTimer = setInterval(async () => {
|
|
987
|
-
await this.runConsolidationForHandler(handler, resourceName, fieldName);
|
|
988
|
-
}, intervalMs);
|
|
989
|
-
}
|
|
990
|
-
|
|
991
|
-
async runConsolidationForHandler(handler, resourceName, fieldName) {
|
|
992
|
-
// Temporarily swap config to use this handler
|
|
993
|
-
const oldResource = this.config.resource;
|
|
994
|
-
const oldField = this.config.field;
|
|
995
|
-
const oldTransactionResource = this.transactionResource;
|
|
996
|
-
const oldTargetResource = this.targetResource;
|
|
997
|
-
const oldLockResource = this.lockResource;
|
|
998
|
-
const oldAnalyticsResource = this.analyticsResource;
|
|
999
|
-
|
|
1000
|
-
this.config.resource = resourceName;
|
|
1001
|
-
this.config.field = fieldName;
|
|
1002
|
-
this.transactionResource = handler.transactionResource;
|
|
1003
|
-
this.targetResource = handler.targetResource;
|
|
1004
|
-
this.lockResource = handler.lockResource;
|
|
1005
|
-
this.analyticsResource = handler.analyticsResource;
|
|
1006
|
-
|
|
1007
|
-
try {
|
|
1008
|
-
await this.runConsolidation();
|
|
1009
|
-
} finally {
|
|
1010
|
-
// Restore
|
|
1011
|
-
this.config.resource = oldResource;
|
|
1012
|
-
this.config.field = oldField;
|
|
1013
|
-
this.transactionResource = oldTransactionResource;
|
|
1014
|
-
this.targetResource = oldTargetResource;
|
|
1015
|
-
this.lockResource = oldLockResource;
|
|
1016
|
-
this.analyticsResource = oldAnalyticsResource;
|
|
1017
|
-
}
|
|
1018
|
-
}
|
|
1019
|
-
|
|
1020
|
-
async runConsolidation() {
|
|
1021
|
-
const startTime = Date.now();
|
|
1022
|
-
|
|
1023
|
-
if (this.config.verbose) {
|
|
1024
|
-
console.log(
|
|
1025
|
-
`[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
|
|
1026
|
-
`Starting consolidation run at ${new Date().toISOString()}`
|
|
1027
|
-
);
|
|
1028
|
-
}
|
|
1029
|
-
|
|
1030
|
-
try {
|
|
1031
|
-
// Query unapplied transactions from recent cohorts (last 24 hours by default)
|
|
1032
|
-
// This uses hourly partition for O(1) performance instead of full scan
|
|
1033
|
-
const now = new Date();
|
|
1034
|
-
const hoursToCheck = this.config.consolidationWindow || 24; // Configurable lookback window (in hours)
|
|
1035
|
-
const cohortHours = [];
|
|
1036
|
-
|
|
1037
|
-
for (let i = 0; i < hoursToCheck; i++) {
|
|
1038
|
-
const date = new Date(now.getTime() - (i * 60 * 60 * 1000)); // Subtract hours
|
|
1039
|
-
const cohortInfo = this.getCohortInfo(date);
|
|
1040
|
-
cohortHours.push(cohortInfo.hour);
|
|
1041
|
-
}
|
|
1042
|
-
|
|
1043
|
-
if (this.config.verbose) {
|
|
1044
|
-
console.log(
|
|
1045
|
-
`[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
|
|
1046
|
-
`Querying ${hoursToCheck} hour partitions for pending transactions...`
|
|
1047
|
-
);
|
|
1048
|
-
}
|
|
1049
|
-
|
|
1050
|
-
// Query transactions by partition for each hour (parallel for speed)
|
|
1051
|
-
const transactionsByHour = await Promise.all(
|
|
1052
|
-
cohortHours.map(async (cohortHour) => {
|
|
1053
|
-
const [ok, err, txns] = await tryFn(() =>
|
|
1054
|
-
this.transactionResource.query({
|
|
1055
|
-
cohortHour,
|
|
1056
|
-
applied: false
|
|
1057
|
-
})
|
|
1058
|
-
);
|
|
1059
|
-
return ok ? txns : [];
|
|
1060
|
-
})
|
|
1061
|
-
);
|
|
1062
|
-
|
|
1063
|
-
// Flatten all transactions
|
|
1064
|
-
const transactions = transactionsByHour.flat();
|
|
1065
|
-
|
|
1066
|
-
if (transactions.length === 0) {
|
|
1067
|
-
if (this.config.verbose) {
|
|
1068
|
-
console.log(
|
|
1069
|
-
`[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
|
|
1070
|
-
`No pending transactions found. Next run in ${this.config.consolidationInterval}s`
|
|
1071
|
-
);
|
|
1072
|
-
}
|
|
1073
|
-
return;
|
|
1074
|
-
}
|
|
1075
|
-
|
|
1076
|
-
// Get unique originalIds
|
|
1077
|
-
const uniqueIds = [...new Set(transactions.map(t => t.originalId))];
|
|
1078
|
-
|
|
1079
|
-
if (this.config.verbose) {
|
|
1080
|
-
console.log(
|
|
1081
|
-
`[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
|
|
1082
|
-
`Found ${transactions.length} pending transactions for ${uniqueIds.length} records. ` +
|
|
1083
|
-
`Consolidating with concurrency=${this.config.consolidationConcurrency}...`
|
|
1084
|
-
);
|
|
1085
|
-
}
|
|
1086
|
-
|
|
1087
|
-
// Consolidate each record in parallel with concurrency limit
|
|
1088
|
-
const { results, errors } = await PromisePool
|
|
1089
|
-
.for(uniqueIds)
|
|
1090
|
-
.withConcurrency(this.config.consolidationConcurrency)
|
|
1091
|
-
.process(async (id) => {
|
|
1092
|
-
return await this.consolidateRecord(id);
|
|
1093
|
-
});
|
|
1094
|
-
|
|
1095
|
-
const duration = Date.now() - startTime;
|
|
1096
|
-
|
|
1097
|
-
if (errors && errors.length > 0) {
|
|
1098
|
-
console.error(
|
|
1099
|
-
`[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
|
|
1100
|
-
`Consolidation completed with ${errors.length} errors in ${duration}ms:`,
|
|
1101
|
-
errors
|
|
1102
|
-
);
|
|
1103
|
-
}
|
|
1104
|
-
|
|
1105
|
-
if (this.config.verbose) {
|
|
1106
|
-
console.log(
|
|
1107
|
-
`[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
|
|
1108
|
-
`Consolidation complete: ${results.length} records consolidated in ${duration}ms ` +
|
|
1109
|
-
`(${errors.length} errors). Next run in ${this.config.consolidationInterval}s`
|
|
1110
|
-
);
|
|
1111
|
-
}
|
|
1112
|
-
|
|
1113
|
-
this.emit('eventual-consistency.consolidated', {
|
|
1114
|
-
resource: this.config.resource,
|
|
1115
|
-
field: this.config.field,
|
|
1116
|
-
recordCount: uniqueIds.length,
|
|
1117
|
-
successCount: results.length,
|
|
1118
|
-
errorCount: errors.length,
|
|
1119
|
-
duration
|
|
1120
|
-
});
|
|
1121
|
-
} catch (error) {
|
|
1122
|
-
const duration = Date.now() - startTime;
|
|
1123
|
-
console.error(
|
|
1124
|
-
`[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
|
|
1125
|
-
`Consolidation error after ${duration}ms:`,
|
|
1126
|
-
error
|
|
1127
|
-
);
|
|
1128
|
-
this.emit('eventual-consistency.consolidation-error', error);
|
|
1129
|
-
}
|
|
1130
|
-
}
|
|
1131
|
-
|
|
1132
|
-
async consolidateRecord(originalId) {
|
|
1133
|
-
// Clean up stale locks before attempting to acquire
|
|
1134
|
-
await this.cleanupStaleLocks();
|
|
1135
|
-
|
|
1136
|
-
// Acquire distributed lock to prevent concurrent consolidation
|
|
1137
|
-
const lockId = `lock-${originalId}`;
|
|
1138
|
-
const [lockAcquired, lockErr, lock] = await tryFn(() =>
|
|
1139
|
-
this.lockResource.insert({
|
|
1140
|
-
id: lockId,
|
|
1141
|
-
lockedAt: Date.now(),
|
|
1142
|
-
workerId: process.pid ? String(process.pid) : 'unknown'
|
|
1143
|
-
})
|
|
1144
|
-
);
|
|
1145
|
-
|
|
1146
|
-
// If lock couldn't be acquired, another worker is consolidating
|
|
1147
|
-
if (!lockAcquired) {
|
|
1148
|
-
if (this.config.verbose) {
|
|
1149
|
-
console.log(`[EventualConsistency] Lock for ${originalId} already held, skipping`);
|
|
1150
|
-
}
|
|
1151
|
-
// Get current value and return (another worker will consolidate)
|
|
1152
|
-
const [recordOk, recordErr, record] = await tryFn(() =>
|
|
1153
|
-
this.targetResource.get(originalId)
|
|
1154
|
-
);
|
|
1155
|
-
return (recordOk && record) ? (record[this.config.field] || 0) : 0;
|
|
1156
|
-
}
|
|
1157
|
-
|
|
1158
|
-
try {
|
|
1159
|
-
// Get all unapplied transactions for this record
|
|
1160
|
-
const [ok, err, transactions] = await tryFn(() =>
|
|
1161
|
-
this.transactionResource.query({
|
|
1162
|
-
originalId,
|
|
1163
|
-
applied: false
|
|
1164
|
-
})
|
|
1165
|
-
);
|
|
1166
|
-
|
|
1167
|
-
if (!ok || !transactions || transactions.length === 0) {
|
|
1168
|
-
// No pending transactions - try to get current value from record
|
|
1169
|
-
const [recordOk, recordErr, record] = await tryFn(() =>
|
|
1170
|
-
this.targetResource.get(originalId)
|
|
1171
|
-
);
|
|
1172
|
-
const currentValue = (recordOk && record) ? (record[this.config.field] || 0) : 0;
|
|
1173
|
-
|
|
1174
|
-
if (this.config.verbose) {
|
|
1175
|
-
console.log(
|
|
1176
|
-
`[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
|
|
1177
|
-
`No pending transactions for ${originalId}, skipping`
|
|
1178
|
-
);
|
|
1179
|
-
}
|
|
1180
|
-
return currentValue;
|
|
1181
|
-
}
|
|
1182
|
-
|
|
1183
|
-
// Get the LAST APPLIED VALUE from transactions (not from record - avoids S3 eventual consistency issues)
|
|
1184
|
-
// This is the source of truth for the current value
|
|
1185
|
-
const [appliedOk, appliedErr, appliedTransactions] = await tryFn(() =>
|
|
1186
|
-
this.transactionResource.query({
|
|
1187
|
-
originalId,
|
|
1188
|
-
applied: true
|
|
1189
|
-
})
|
|
1190
|
-
);
|
|
1191
|
-
|
|
1192
|
-
let currentValue = 0;
|
|
1193
|
-
|
|
1194
|
-
if (appliedOk && appliedTransactions && appliedTransactions.length > 0) {
|
|
1195
|
-
// Check if record exists - if deleted, ignore old applied transactions
|
|
1196
|
-
const [recordExistsOk, recordExistsErr, recordExists] = await tryFn(() =>
|
|
1197
|
-
this.targetResource.get(originalId)
|
|
1198
|
-
);
|
|
1199
|
-
|
|
1200
|
-
if (!recordExistsOk || !recordExists) {
|
|
1201
|
-
// Record was deleted - ignore applied transactions and start fresh
|
|
1202
|
-
// This prevents old values from being carried over after deletion
|
|
1203
|
-
if (this.config.verbose) {
|
|
1204
|
-
console.log(
|
|
1205
|
-
`[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
|
|
1206
|
-
`Record ${originalId} doesn't exist, deleting ${appliedTransactions.length} old applied transactions`
|
|
1207
|
-
);
|
|
1208
|
-
}
|
|
1209
|
-
|
|
1210
|
-
// Delete old applied transactions to prevent them from being used when record is recreated
|
|
1211
|
-
const { results, errors } = await PromisePool
|
|
1212
|
-
.for(appliedTransactions)
|
|
1213
|
-
.withConcurrency(10)
|
|
1214
|
-
.process(async (txn) => {
|
|
1215
|
-
const [deleted] = await tryFn(() => this.transactionResource.delete(txn.id));
|
|
1216
|
-
return deleted;
|
|
1217
|
-
});
|
|
1218
|
-
|
|
1219
|
-
if (this.config.verbose && errors && errors.length > 0) {
|
|
1220
|
-
console.warn(
|
|
1221
|
-
`[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
|
|
1222
|
-
`Failed to delete ${errors.length} old applied transactions`
|
|
1223
|
-
);
|
|
1224
|
-
}
|
|
1225
|
-
|
|
1226
|
-
currentValue = 0;
|
|
1227
|
-
} else {
|
|
1228
|
-
// Record exists - use applied transactions to calculate current value
|
|
1229
|
-
// Sort by timestamp to get chronological order
|
|
1230
|
-
appliedTransactions.sort((a, b) =>
|
|
1231
|
-
new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()
|
|
1232
|
-
);
|
|
1233
|
-
|
|
1234
|
-
// Check if there's a 'set' operation in applied transactions
|
|
1235
|
-
const hasSetInApplied = appliedTransactions.some(t => t.operation === 'set');
|
|
1236
|
-
|
|
1237
|
-
if (!hasSetInApplied) {
|
|
1238
|
-
// No 'set' operation in applied transactions means we're missing the base value
|
|
1239
|
-
// This can only happen if:
|
|
1240
|
-
// 1. Record had an initial value before first transaction
|
|
1241
|
-
// 2. First consolidation didn't create an anchor transaction (legacy behavior)
|
|
1242
|
-
// Solution: Get the current record value and create an anchor transaction now
|
|
1243
|
-
const recordValue = recordExists[this.config.field] || 0;
|
|
1244
|
-
|
|
1245
|
-
// Calculate what the base value was by subtracting all applied deltas
|
|
1246
|
-
let appliedDelta = 0;
|
|
1247
|
-
for (const t of appliedTransactions) {
|
|
1248
|
-
if (t.operation === 'add') appliedDelta += t.value;
|
|
1249
|
-
else if (t.operation === 'sub') appliedDelta -= t.value;
|
|
1250
|
-
}
|
|
1251
|
-
|
|
1252
|
-
const baseValue = recordValue - appliedDelta;
|
|
1253
|
-
|
|
1254
|
-
// Create and save anchor transaction with the base value
|
|
1255
|
-
// Only create if baseValue is non-zero AND we don't already have an anchor transaction
|
|
1256
|
-
const hasExistingAnchor = appliedTransactions.some(t => t.source === 'anchor');
|
|
1257
|
-
if (baseValue !== 0 && !hasExistingAnchor) {
|
|
1258
|
-
// Use the timestamp of the first applied transaction for cohort info
|
|
1259
|
-
const firstTransactionDate = new Date(appliedTransactions[0].timestamp);
|
|
1260
|
-
const cohortInfo = this.getCohortInfo(firstTransactionDate);
|
|
1261
|
-
const anchorTransaction = {
|
|
1262
|
-
id: idGenerator(),
|
|
1263
|
-
originalId: originalId,
|
|
1264
|
-
field: this.config.field,
|
|
1265
|
-
value: baseValue,
|
|
1266
|
-
operation: 'set',
|
|
1267
|
-
timestamp: new Date(firstTransactionDate.getTime() - 1).toISOString(), // 1ms before first txn to ensure it's first
|
|
1268
|
-
cohortDate: cohortInfo.date,
|
|
1269
|
-
cohortHour: cohortInfo.hour,
|
|
1270
|
-
cohortMonth: cohortInfo.month,
|
|
1271
|
-
source: 'anchor',
|
|
1272
|
-
applied: true
|
|
1273
|
-
};
|
|
1274
|
-
|
|
1275
|
-
await this.transactionResource.insert(anchorTransaction);
|
|
1276
|
-
|
|
1277
|
-
// Prepend to applied transactions for this consolidation
|
|
1278
|
-
appliedTransactions.unshift(anchorTransaction);
|
|
1279
|
-
}
|
|
1280
|
-
}
|
|
1281
|
-
|
|
1282
|
-
// Apply reducer to get the last consolidated value
|
|
1283
|
-
currentValue = this.config.reducer(appliedTransactions);
|
|
1284
|
-
}
|
|
1285
|
-
} else {
|
|
1286
|
-
// No applied transactions - this is the FIRST consolidation
|
|
1287
|
-
// Try to get initial value from record
|
|
1288
|
-
const [recordOk, recordErr, record] = await tryFn(() =>
|
|
1289
|
-
this.targetResource.get(originalId)
|
|
1290
|
-
);
|
|
1291
|
-
currentValue = (recordOk && record) ? (record[this.config.field] || 0) : 0;
|
|
1292
|
-
|
|
1293
|
-
// If there's an initial value, create and save an anchor transaction
|
|
1294
|
-
// This ensures all future consolidations have a reliable base value
|
|
1295
|
-
if (currentValue !== 0) {
|
|
1296
|
-
// Use timestamp of the first pending transaction (or current time if none)
|
|
1297
|
-
let anchorTimestamp;
|
|
1298
|
-
if (transactions && transactions.length > 0) {
|
|
1299
|
-
const firstPendingDate = new Date(transactions[0].timestamp);
|
|
1300
|
-
anchorTimestamp = new Date(firstPendingDate.getTime() - 1).toISOString();
|
|
1301
|
-
} else {
|
|
1302
|
-
anchorTimestamp = new Date().toISOString();
|
|
1303
|
-
}
|
|
1304
|
-
|
|
1305
|
-
const cohortInfo = this.getCohortInfo(new Date(anchorTimestamp));
|
|
1306
|
-
const anchorTransaction = {
|
|
1307
|
-
id: idGenerator(),
|
|
1308
|
-
originalId: originalId,
|
|
1309
|
-
field: this.config.field,
|
|
1310
|
-
value: currentValue,
|
|
1311
|
-
operation: 'set',
|
|
1312
|
-
timestamp: anchorTimestamp,
|
|
1313
|
-
cohortDate: cohortInfo.date,
|
|
1314
|
-
cohortHour: cohortInfo.hour,
|
|
1315
|
-
cohortMonth: cohortInfo.month,
|
|
1316
|
-
source: 'anchor',
|
|
1317
|
-
applied: true
|
|
1318
|
-
};
|
|
1319
|
-
|
|
1320
|
-
await this.transactionResource.insert(anchorTransaction);
|
|
1321
|
-
|
|
1322
|
-
if (this.config.verbose) {
|
|
1323
|
-
console.log(
|
|
1324
|
-
`[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
|
|
1325
|
-
`Created anchor transaction for ${originalId} with base value ${currentValue}`
|
|
1326
|
-
);
|
|
1327
|
-
}
|
|
1328
|
-
}
|
|
1329
|
-
}
|
|
1330
|
-
|
|
1331
|
-
if (this.config.verbose) {
|
|
1332
|
-
console.log(
|
|
1333
|
-
`[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
|
|
1334
|
-
`Consolidating ${originalId}: ${transactions.length} pending transactions ` +
|
|
1335
|
-
`(current: ${currentValue} from ${appliedOk && appliedTransactions?.length > 0 ? 'applied transactions' : 'record'})`
|
|
1336
|
-
);
|
|
1337
|
-
}
|
|
1338
|
-
|
|
1339
|
-
// Sort pending transactions by timestamp
|
|
1340
|
-
transactions.sort((a, b) =>
|
|
1341
|
-
new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()
|
|
1342
|
-
);
|
|
1343
|
-
|
|
1344
|
-
// If there's a current value and no 'set' operations in pending transactions,
|
|
1345
|
-
// prepend a synthetic set transaction to preserve the current value
|
|
1346
|
-
const hasSetOperation = transactions.some(t => t.operation === 'set');
|
|
1347
|
-
if (currentValue !== 0 && !hasSetOperation) {
|
|
1348
|
-
transactions.unshift(this._createSyntheticSetTransaction(currentValue));
|
|
1349
|
-
}
|
|
1350
|
-
|
|
1351
|
-
// Apply reducer to get consolidated value
|
|
1352
|
-
const consolidatedValue = this.config.reducer(transactions);
|
|
1353
|
-
|
|
1354
|
-
if (this.config.verbose) {
|
|
1355
|
-
console.log(
|
|
1356
|
-
`[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
|
|
1357
|
-
`${originalId}: ${currentValue} → ${consolidatedValue} ` +
|
|
1358
|
-
`(${consolidatedValue > currentValue ? '+' : ''}${consolidatedValue - currentValue})`
|
|
1359
|
-
);
|
|
1360
|
-
}
|
|
1361
|
-
|
|
1362
|
-
// Update the original record
|
|
1363
|
-
// NOTE: We do NOT attempt to insert non-existent records because:
|
|
1364
|
-
// 1. Target resources typically have required fields we don't know about
|
|
1365
|
-
// 2. Record creation should be the application's responsibility
|
|
1366
|
-
// 3. Transactions will remain pending until the record is created
|
|
1367
|
-
const [updateOk, updateErr] = await tryFn(() =>
|
|
1368
|
-
this.targetResource.update(originalId, {
|
|
1369
|
-
[this.config.field]: consolidatedValue
|
|
1370
|
-
})
|
|
1371
|
-
);
|
|
1372
|
-
|
|
1373
|
-
if (!updateOk) {
|
|
1374
|
-
// Check if record doesn't exist
|
|
1375
|
-
if (updateErr?.message?.includes('does not exist')) {
|
|
1376
|
-
// Record doesn't exist - skip consolidation and keep transactions pending
|
|
1377
|
-
if (this.config.verbose) {
|
|
1378
|
-
console.warn(
|
|
1379
|
-
`[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
|
|
1380
|
-
`Record ${originalId} doesn't exist. Skipping consolidation. ` +
|
|
1381
|
-
`${transactions.length} transactions will remain pending until record is created.`
|
|
1382
|
-
);
|
|
1383
|
-
}
|
|
1384
|
-
|
|
1385
|
-
// Return the consolidated value (for informational purposes)
|
|
1386
|
-
// Transactions remain pending and will be picked up when record exists
|
|
1387
|
-
return consolidatedValue;
|
|
1388
|
-
}
|
|
1389
|
-
|
|
1390
|
-
// Update failed for another reason - this is a real error
|
|
1391
|
-
console.error(
|
|
1392
|
-
`[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
|
|
1393
|
-
`FAILED to update ${originalId}: ${updateErr?.message || updateErr}`,
|
|
1394
|
-
{ error: updateErr, consolidatedValue, currentValue }
|
|
1395
|
-
);
|
|
1396
|
-
throw updateErr;
|
|
1397
|
-
}
|
|
1398
|
-
|
|
1399
|
-
if (updateOk) {
|
|
1400
|
-
// Mark transactions as applied (skip synthetic ones) - use PromisePool for controlled concurrency
|
|
1401
|
-
const transactionsToUpdate = transactions.filter(txn => txn.id !== '__synthetic__');
|
|
1402
|
-
|
|
1403
|
-
const { results, errors } = await PromisePool
|
|
1404
|
-
.for(transactionsToUpdate)
|
|
1405
|
-
.withConcurrency(10) // Limit parallel updates
|
|
1406
|
-
.process(async (txn) => {
|
|
1407
|
-
const [ok, err] = await tryFn(() =>
|
|
1408
|
-
this.transactionResource.update(txn.id, { applied: true })
|
|
1409
|
-
);
|
|
1410
|
-
|
|
1411
|
-
if (!ok && this.config.verbose) {
|
|
1412
|
-
console.warn(`[EventualConsistency] Failed to mark transaction ${txn.id} as applied:`, err?.message);
|
|
1413
|
-
}
|
|
1414
|
-
|
|
1415
|
-
return ok;
|
|
1416
|
-
});
|
|
1417
|
-
|
|
1418
|
-
if (errors && errors.length > 0 && this.config.verbose) {
|
|
1419
|
-
console.warn(`[EventualConsistency] ${errors.length} transactions failed to mark as applied`);
|
|
1420
|
-
}
|
|
1421
|
-
|
|
1422
|
-
// Update analytics if enabled (only for real transactions, not synthetic)
|
|
1423
|
-
if (this.config.enableAnalytics && transactionsToUpdate.length > 0) {
|
|
1424
|
-
await this.updateAnalytics(transactionsToUpdate);
|
|
1425
|
-
}
|
|
1426
|
-
|
|
1427
|
-
// Invalidate cache for this record after consolidation
|
|
1428
|
-
if (this.targetResource && this.targetResource.cache && typeof this.targetResource.cache.delete === 'function') {
|
|
1429
|
-
try {
|
|
1430
|
-
const cacheKey = await this.targetResource.cacheKeyFor({ id: originalId });
|
|
1431
|
-
await this.targetResource.cache.delete(cacheKey);
|
|
1432
|
-
|
|
1433
|
-
if (this.config.verbose) {
|
|
1434
|
-
console.log(
|
|
1435
|
-
`[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
|
|
1436
|
-
`Cache invalidated for ${originalId}`
|
|
1437
|
-
);
|
|
1438
|
-
}
|
|
1439
|
-
} catch (cacheErr) {
|
|
1440
|
-
// Log but don't fail consolidation if cache invalidation fails
|
|
1441
|
-
if (this.config.verbose) {
|
|
1442
|
-
console.warn(
|
|
1443
|
-
`[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
|
|
1444
|
-
`Failed to invalidate cache for ${originalId}: ${cacheErr?.message}`
|
|
1445
|
-
);
|
|
1446
|
-
}
|
|
1447
|
-
}
|
|
1448
|
-
}
|
|
1449
|
-
}
|
|
1450
|
-
|
|
1451
|
-
return consolidatedValue;
|
|
1452
|
-
} finally {
|
|
1453
|
-
// Always release the lock
|
|
1454
|
-
const [lockReleased, lockReleaseErr] = await tryFn(() => this.lockResource.delete(lockId));
|
|
1455
|
-
|
|
1456
|
-
if (!lockReleased && this.config.verbose) {
|
|
1457
|
-
console.warn(`[EventualConsistency] Failed to release lock ${lockId}:`, lockReleaseErr?.message);
|
|
1458
|
-
}
|
|
1459
|
-
}
|
|
1460
|
-
}
|
|
1461
|
-
|
|
1462
|
-
async getConsolidatedValue(originalId, options = {}) {
|
|
1463
|
-
const includeApplied = options.includeApplied || false;
|
|
1464
|
-
const startDate = options.startDate;
|
|
1465
|
-
const endDate = options.endDate;
|
|
1466
|
-
|
|
1467
|
-
// Build query
|
|
1468
|
-
const query = { originalId };
|
|
1469
|
-
if (!includeApplied) {
|
|
1470
|
-
query.applied = false;
|
|
1471
|
-
}
|
|
1472
|
-
|
|
1473
|
-
// Get transactions
|
|
1474
|
-
const [ok, err, transactions] = await tryFn(() =>
|
|
1475
|
-
this.transactionResource.query(query)
|
|
1476
|
-
);
|
|
1477
|
-
|
|
1478
|
-
if (!ok || !transactions || transactions.length === 0) {
|
|
1479
|
-
// If no transactions, check if record exists and return its current value
|
|
1480
|
-
const [recordOk, recordErr, record] = await tryFn(() =>
|
|
1481
|
-
this.targetResource.get(originalId)
|
|
1482
|
-
);
|
|
1483
|
-
|
|
1484
|
-
if (recordOk && record) {
|
|
1485
|
-
return record[this.config.field] || 0;
|
|
1486
|
-
}
|
|
1487
|
-
|
|
1488
|
-
return 0;
|
|
1489
|
-
}
|
|
1490
|
-
|
|
1491
|
-
// Filter by date range if specified
|
|
1492
|
-
let filtered = transactions;
|
|
1493
|
-
if (startDate || endDate) {
|
|
1494
|
-
filtered = transactions.filter(t => {
|
|
1495
|
-
const timestamp = new Date(t.timestamp);
|
|
1496
|
-
if (startDate && timestamp < new Date(startDate)) return false;
|
|
1497
|
-
if (endDate && timestamp > new Date(endDate)) return false;
|
|
1498
|
-
return true;
|
|
1499
|
-
});
|
|
1500
|
-
}
|
|
1501
|
-
|
|
1502
|
-
// Get current value from record
|
|
1503
|
-
const [recordOk, recordErr, record] = await tryFn(() =>
|
|
1504
|
-
this.targetResource.get(originalId)
|
|
1505
|
-
);
|
|
1506
|
-
const currentValue = (recordOk && record) ? (record[this.config.field] || 0) : 0;
|
|
1507
|
-
|
|
1508
|
-
// Check if there's a 'set' operation in filtered transactions
|
|
1509
|
-
const hasSetOperation = filtered.some(t => t.operation === 'set');
|
|
1510
|
-
|
|
1511
|
-
// If current value exists and no 'set', prepend synthetic set transaction
|
|
1512
|
-
if (currentValue !== 0 && !hasSetOperation) {
|
|
1513
|
-
filtered.unshift(this._createSyntheticSetTransaction(currentValue));
|
|
1514
|
-
}
|
|
1515
|
-
|
|
1516
|
-
// Sort by timestamp
|
|
1517
|
-
filtered.sort((a, b) =>
|
|
1518
|
-
new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()
|
|
1519
|
-
);
|
|
1520
|
-
|
|
1521
|
-
// Apply reducer
|
|
1522
|
-
return this.config.reducer(filtered);
|
|
1523
|
-
}
|
|
1524
|
-
|
|
1525
|
-
// Helper method to get cohort statistics
|
|
1526
|
-
async getCohortStats(cohortDate) {
|
|
1527
|
-
const [ok, err, transactions] = await tryFn(() =>
|
|
1528
|
-
this.transactionResource.query({
|
|
1529
|
-
cohortDate
|
|
1530
|
-
})
|
|
1531
|
-
);
|
|
1532
|
-
|
|
1533
|
-
if (!ok) return null;
|
|
1534
|
-
|
|
1535
|
-
const stats = {
|
|
1536
|
-
date: cohortDate,
|
|
1537
|
-
transactionCount: transactions.length,
|
|
1538
|
-
totalValue: 0,
|
|
1539
|
-
byOperation: { set: 0, add: 0, sub: 0 },
|
|
1540
|
-
byOriginalId: {}
|
|
1541
|
-
};
|
|
1542
|
-
|
|
1543
|
-
for (const txn of transactions) {
|
|
1544
|
-
stats.totalValue += txn.value || 0;
|
|
1545
|
-
stats.byOperation[txn.operation] = (stats.byOperation[txn.operation] || 0) + 1;
|
|
1546
|
-
|
|
1547
|
-
if (!stats.byOriginalId[txn.originalId]) {
|
|
1548
|
-
stats.byOriginalId[txn.originalId] = {
|
|
1549
|
-
count: 0,
|
|
1550
|
-
value: 0
|
|
1551
|
-
};
|
|
1552
|
-
}
|
|
1553
|
-
stats.byOriginalId[txn.originalId].count++;
|
|
1554
|
-
stats.byOriginalId[txn.originalId].value += txn.value || 0;
|
|
1555
|
-
}
|
|
1556
|
-
|
|
1557
|
-
return stats;
|
|
1558
|
-
}
|
|
1559
|
-
|
|
1560
|
-
/**
|
|
1561
|
-
* Recalculate from scratch by resetting all transactions to pending
|
|
1562
|
-
* This is useful for debugging, recovery, or when you want to recompute everything
|
|
1563
|
-
* @param {string} originalId - The ID of the record to recalculate
|
|
1564
|
-
* @returns {Promise<number>} The recalculated value
|
|
1565
|
-
*/
|
|
1566
|
-
async recalculateRecord(originalId) {
|
|
1567
|
-
// Clean up stale locks before attempting to acquire
|
|
1568
|
-
await this.cleanupStaleLocks();
|
|
1569
|
-
|
|
1570
|
-
// Acquire distributed lock to prevent concurrent operations
|
|
1571
|
-
const lockId = `lock-recalculate-${originalId}`;
|
|
1572
|
-
const [lockAcquired, lockErr, lock] = await tryFn(() =>
|
|
1573
|
-
this.lockResource.insert({
|
|
1574
|
-
id: lockId,
|
|
1575
|
-
lockedAt: Date.now(),
|
|
1576
|
-
workerId: process.pid ? String(process.pid) : 'unknown'
|
|
1577
|
-
})
|
|
1578
|
-
);
|
|
1579
|
-
|
|
1580
|
-
// If lock couldn't be acquired, another worker is operating on this record
|
|
1581
|
-
if (!lockAcquired) {
|
|
1582
|
-
if (this.config.verbose) {
|
|
1583
|
-
console.log(`[EventualConsistency] Recalculate lock for ${originalId} already held, skipping`);
|
|
1584
|
-
}
|
|
1585
|
-
throw new Error(`Cannot recalculate ${originalId}: lock already held by another worker`);
|
|
1586
|
-
}
|
|
1587
|
-
|
|
1588
|
-
try {
|
|
1589
|
-
if (this.config.verbose) {
|
|
1590
|
-
console.log(
|
|
1591
|
-
`[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
|
|
1592
|
-
`Starting recalculation for ${originalId} (resetting all transactions to pending)`
|
|
1593
|
-
);
|
|
1594
|
-
}
|
|
1595
|
-
|
|
1596
|
-
// Get ALL transactions for this record (both applied and pending)
|
|
1597
|
-
const [allOk, allErr, allTransactions] = await tryFn(() =>
|
|
1598
|
-
this.transactionResource.query({
|
|
1599
|
-
originalId
|
|
1600
|
-
})
|
|
1601
|
-
);
|
|
1602
|
-
|
|
1603
|
-
if (!allOk || !allTransactions || allTransactions.length === 0) {
|
|
1604
|
-
if (this.config.verbose) {
|
|
1605
|
-
console.log(
|
|
1606
|
-
`[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
|
|
1607
|
-
`No transactions found for ${originalId}, nothing to recalculate`
|
|
1608
|
-
);
|
|
1609
|
-
}
|
|
1610
|
-
return 0;
|
|
1611
|
-
}
|
|
1612
|
-
|
|
1613
|
-
if (this.config.verbose) {
|
|
1614
|
-
console.log(
|
|
1615
|
-
`[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
|
|
1616
|
-
`Found ${allTransactions.length} total transactions for ${originalId}, marking all as pending...`
|
|
1617
|
-
);
|
|
1618
|
-
}
|
|
1619
|
-
|
|
1620
|
-
// Mark ALL transactions as pending (applied: false)
|
|
1621
|
-
// Exclude anchor transactions (they should always be applied)
|
|
1622
|
-
const transactionsToReset = allTransactions.filter(txn => txn.source !== 'anchor');
|
|
1623
|
-
|
|
1624
|
-
const { results, errors } = await PromisePool
|
|
1625
|
-
.for(transactionsToReset)
|
|
1626
|
-
.withConcurrency(10)
|
|
1627
|
-
.process(async (txn) => {
|
|
1628
|
-
const [ok, err] = await tryFn(() =>
|
|
1629
|
-
this.transactionResource.update(txn.id, { applied: false })
|
|
1630
|
-
);
|
|
1631
|
-
|
|
1632
|
-
if (!ok && this.config.verbose) {
|
|
1633
|
-
console.warn(`[EventualConsistency] Failed to reset transaction ${txn.id}:`, err?.message);
|
|
1634
|
-
}
|
|
1635
|
-
|
|
1636
|
-
return ok;
|
|
1637
|
-
});
|
|
1638
|
-
|
|
1639
|
-
if (errors && errors.length > 0) {
|
|
1640
|
-
console.warn(
|
|
1641
|
-
`[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
|
|
1642
|
-
`Failed to reset ${errors.length} transactions during recalculation`
|
|
1643
|
-
);
|
|
1644
|
-
}
|
|
1645
|
-
|
|
1646
|
-
if (this.config.verbose) {
|
|
1647
|
-
console.log(
|
|
1648
|
-
`[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
|
|
1649
|
-
`Reset ${results.length} transactions to pending, now resetting record value and running consolidation...`
|
|
1650
|
-
);
|
|
1651
|
-
}
|
|
1652
|
-
|
|
1653
|
-
// Reset the record's field value to 0 to prevent double-counting
|
|
1654
|
-
// This ensures consolidation starts fresh without using the old value as an anchor
|
|
1655
|
-
const [resetOk, resetErr] = await tryFn(() =>
|
|
1656
|
-
this.targetResource.update(originalId, {
|
|
1657
|
-
[this.config.field]: 0
|
|
1658
|
-
})
|
|
1659
|
-
);
|
|
1660
|
-
|
|
1661
|
-
if (!resetOk && this.config.verbose) {
|
|
1662
|
-
console.warn(
|
|
1663
|
-
`[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
|
|
1664
|
-
`Failed to reset record value for ${originalId}: ${resetErr?.message}`
|
|
1665
|
-
);
|
|
1666
|
-
}
|
|
1667
|
-
|
|
1668
|
-
// Now run normal consolidation which will process all pending transactions
|
|
1669
|
-
const consolidatedValue = await this.consolidateRecord(originalId);
|
|
1670
|
-
|
|
1671
|
-
if (this.config.verbose) {
|
|
1672
|
-
console.log(
|
|
1673
|
-
`[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
|
|
1674
|
-
`Recalculation complete for ${originalId}: final value = ${consolidatedValue}`
|
|
1675
|
-
);
|
|
1676
|
-
}
|
|
1677
|
-
|
|
1678
|
-
return consolidatedValue;
|
|
1679
|
-
} finally {
|
|
1680
|
-
// Always release the lock
|
|
1681
|
-
const [lockReleased, lockReleaseErr] = await tryFn(() => this.lockResource.delete(lockId));
|
|
1682
|
-
|
|
1683
|
-
if (!lockReleased && this.config.verbose) {
|
|
1684
|
-
console.warn(`[EventualConsistency] Failed to release recalculate lock ${lockId}:`, lockReleaseErr?.message);
|
|
1685
|
-
}
|
|
1686
|
-
}
|
|
1687
|
-
}
|
|
1688
|
-
|
|
1689
|
-
/**
|
|
1690
|
-
* Clean up stale locks that exceed the configured timeout
|
|
1691
|
-
* Uses distributed locking to prevent multiple containers from cleaning simultaneously
|
|
1692
|
-
*/
|
|
1693
|
-
async cleanupStaleLocks() {
|
|
1694
|
-
const now = Date.now();
|
|
1695
|
-
const lockTimeoutMs = this.config.lockTimeout * 1000; // Convert seconds to ms
|
|
1696
|
-
const cutoffTime = now - lockTimeoutMs;
|
|
1697
|
-
|
|
1698
|
-
// Acquire distributed lock for cleanup operation
|
|
1699
|
-
const cleanupLockId = `lock-cleanup-${this.config.resource}-${this.config.field}`;
|
|
1700
|
-
const [lockAcquired] = await tryFn(() =>
|
|
1701
|
-
this.lockResource.insert({
|
|
1702
|
-
id: cleanupLockId,
|
|
1703
|
-
lockedAt: Date.now(),
|
|
1704
|
-
workerId: process.pid ? String(process.pid) : 'unknown'
|
|
1705
|
-
})
|
|
1706
|
-
);
|
|
1707
|
-
|
|
1708
|
-
// If another container is already cleaning, skip
|
|
1709
|
-
if (!lockAcquired) {
|
|
1710
|
-
if (this.config.verbose) {
|
|
1711
|
-
console.log(`[EventualConsistency] Lock cleanup already running in another container`);
|
|
1712
|
-
}
|
|
1713
|
-
return;
|
|
1714
|
-
}
|
|
1715
|
-
|
|
1716
|
-
try {
|
|
1717
|
-
// Get all locks
|
|
1718
|
-
const [ok, err, locks] = await tryFn(() => this.lockResource.list());
|
|
1719
|
-
|
|
1720
|
-
if (!ok || !locks || locks.length === 0) return;
|
|
1721
|
-
|
|
1722
|
-
// Find stale locks (excluding the cleanup lock itself)
|
|
1723
|
-
const staleLocks = locks.filter(lock =>
|
|
1724
|
-
lock.id !== cleanupLockId && lock.lockedAt < cutoffTime
|
|
1725
|
-
);
|
|
1726
|
-
|
|
1727
|
-
if (staleLocks.length === 0) return;
|
|
1728
|
-
|
|
1729
|
-
if (this.config.verbose) {
|
|
1730
|
-
console.log(`[EventualConsistency] Cleaning up ${staleLocks.length} stale locks`);
|
|
1731
|
-
}
|
|
1732
|
-
|
|
1733
|
-
// Delete stale locks using PromisePool
|
|
1734
|
-
const { results, errors } = await PromisePool
|
|
1735
|
-
.for(staleLocks)
|
|
1736
|
-
.withConcurrency(5)
|
|
1737
|
-
.process(async (lock) => {
|
|
1738
|
-
const [deleted] = await tryFn(() => this.lockResource.delete(lock.id));
|
|
1739
|
-
return deleted;
|
|
1740
|
-
});
|
|
1741
|
-
|
|
1742
|
-
if (errors && errors.length > 0 && this.config.verbose) {
|
|
1743
|
-
console.warn(`[EventualConsistency] ${errors.length} stale locks failed to delete`);
|
|
1744
|
-
}
|
|
1745
|
-
} catch (error) {
|
|
1746
|
-
if (this.config.verbose) {
|
|
1747
|
-
console.warn(`[EventualConsistency] Error cleaning up stale locks:`, error.message);
|
|
1748
|
-
}
|
|
1749
|
-
} finally {
|
|
1750
|
-
// Always release cleanup lock
|
|
1751
|
-
await tryFn(() => this.lockResource.delete(cleanupLockId));
|
|
1752
|
-
}
|
|
1753
|
-
}
|
|
1754
|
-
|
|
1755
|
-
/**
|
|
1756
|
-
* Start garbage collection timer for old applied transactions
|
|
1757
|
-
*/
|
|
1758
|
-
startGarbageCollectionTimer() {
|
|
1759
|
-
const gcIntervalMs = this.config.gcInterval * 1000; // Convert seconds to ms
|
|
1760
|
-
|
|
1761
|
-
this.gcTimer = setInterval(async () => {
|
|
1762
|
-
await this.runGarbageCollection();
|
|
1763
|
-
}, gcIntervalMs);
|
|
1764
|
-
}
|
|
1765
|
-
|
|
1766
|
-
startGarbageCollectionTimerForHandler(handler, resourceName, fieldName) {
|
|
1767
|
-
const gcIntervalMs = this.config.gcInterval * 1000; // Convert seconds to ms
|
|
1768
|
-
|
|
1769
|
-
handler.gcTimer = setInterval(async () => {
|
|
1770
|
-
await this.runGarbageCollectionForHandler(handler, resourceName, fieldName);
|
|
1771
|
-
}, gcIntervalMs);
|
|
1772
|
-
}
|
|
1773
|
-
|
|
1774
|
-
async runGarbageCollectionForHandler(handler, resourceName, fieldName) {
|
|
1775
|
-
// Temporarily swap config to use this handler
|
|
1776
|
-
const oldResource = this.config.resource;
|
|
1777
|
-
const oldField = this.config.field;
|
|
1778
|
-
const oldTransactionResource = this.transactionResource;
|
|
1779
|
-
const oldTargetResource = this.targetResource;
|
|
1780
|
-
const oldLockResource = this.lockResource;
|
|
1781
|
-
|
|
1782
|
-
this.config.resource = resourceName;
|
|
1783
|
-
this.config.field = fieldName;
|
|
1784
|
-
this.transactionResource = handler.transactionResource;
|
|
1785
|
-
this.targetResource = handler.targetResource;
|
|
1786
|
-
this.lockResource = handler.lockResource;
|
|
1787
|
-
|
|
1788
|
-
try {
|
|
1789
|
-
await this.runGarbageCollection();
|
|
1790
|
-
} finally {
|
|
1791
|
-
// Restore
|
|
1792
|
-
this.config.resource = oldResource;
|
|
1793
|
-
this.config.field = oldField;
|
|
1794
|
-
this.transactionResource = oldTransactionResource;
|
|
1795
|
-
this.targetResource = oldTargetResource;
|
|
1796
|
-
this.lockResource = oldLockResource;
|
|
1797
|
-
}
|
|
1798
|
-
}
|
|
1799
|
-
|
|
1800
|
-
/**
|
|
1801
|
-
* Delete old applied transactions based on retention policy
|
|
1802
|
-
* Uses distributed locking to prevent multiple containers from running GC simultaneously
|
|
1803
|
-
*/
|
|
1804
|
-
async runGarbageCollection() {
|
|
1805
|
-
// Acquire distributed lock for GC operation
|
|
1806
|
-
const gcLockId = `lock-gc-${this.config.resource}-${this.config.field}`;
|
|
1807
|
-
const [lockAcquired] = await tryFn(() =>
|
|
1808
|
-
this.lockResource.insert({
|
|
1809
|
-
id: gcLockId,
|
|
1810
|
-
lockedAt: Date.now(),
|
|
1811
|
-
workerId: process.pid ? String(process.pid) : 'unknown'
|
|
1812
|
-
})
|
|
1813
|
-
);
|
|
1814
|
-
|
|
1815
|
-
// If another container is already running GC, skip
|
|
1816
|
-
if (!lockAcquired) {
|
|
1817
|
-
if (this.config.verbose) {
|
|
1818
|
-
console.log(`[EventualConsistency] GC already running in another container`);
|
|
1819
|
-
}
|
|
1820
|
-
return;
|
|
1821
|
-
}
|
|
1822
|
-
|
|
1823
|
-
try {
|
|
1824
|
-
const now = Date.now();
|
|
1825
|
-
const retentionMs = this.config.transactionRetention * 24 * 60 * 60 * 1000; // Days to ms
|
|
1826
|
-
const cutoffDate = new Date(now - retentionMs);
|
|
1827
|
-
const cutoffIso = cutoffDate.toISOString();
|
|
1828
|
-
|
|
1829
|
-
if (this.config.verbose) {
|
|
1830
|
-
console.log(`[EventualConsistency] Running GC for transactions older than ${cutoffIso} (${this.config.transactionRetention} days)`);
|
|
1831
|
-
}
|
|
1832
|
-
|
|
1833
|
-
// Query old applied transactions
|
|
1834
|
-
const cutoffMonth = cutoffDate.toISOString().substring(0, 7); // YYYY-MM
|
|
1835
|
-
|
|
1836
|
-
const [ok, err, oldTransactions] = await tryFn(() =>
|
|
1837
|
-
this.transactionResource.query({
|
|
1838
|
-
applied: true,
|
|
1839
|
-
timestamp: { '<': cutoffIso }
|
|
1840
|
-
})
|
|
1841
|
-
);
|
|
1842
|
-
|
|
1843
|
-
if (!ok) {
|
|
1844
|
-
if (this.config.verbose) {
|
|
1845
|
-
console.warn(`[EventualConsistency] GC failed to query transactions:`, err?.message);
|
|
1846
|
-
}
|
|
1847
|
-
return;
|
|
1848
|
-
}
|
|
1849
|
-
|
|
1850
|
-
if (!oldTransactions || oldTransactions.length === 0) {
|
|
1851
|
-
if (this.config.verbose) {
|
|
1852
|
-
console.log(`[EventualConsistency] No old transactions to clean up`);
|
|
1853
|
-
}
|
|
1854
|
-
return;
|
|
1855
|
-
}
|
|
1856
|
-
|
|
1857
|
-
if (this.config.verbose) {
|
|
1858
|
-
console.log(`[EventualConsistency] Deleting ${oldTransactions.length} old transactions`);
|
|
1859
|
-
}
|
|
1860
|
-
|
|
1861
|
-
// Delete old transactions using PromisePool
|
|
1862
|
-
const { results, errors } = await PromisePool
|
|
1863
|
-
.for(oldTransactions)
|
|
1864
|
-
.withConcurrency(10)
|
|
1865
|
-
.process(async (txn) => {
|
|
1866
|
-
const [deleted] = await tryFn(() => this.transactionResource.delete(txn.id));
|
|
1867
|
-
return deleted;
|
|
1868
|
-
});
|
|
1869
|
-
|
|
1870
|
-
if (this.config.verbose) {
|
|
1871
|
-
console.log(`[EventualConsistency] GC completed: ${results.length} deleted, ${errors.length} errors`);
|
|
1872
|
-
}
|
|
1873
|
-
|
|
1874
|
-
this.emit('eventual-consistency.gc-completed', {
|
|
1875
|
-
resource: this.config.resource,
|
|
1876
|
-
field: this.config.field,
|
|
1877
|
-
deletedCount: results.length,
|
|
1878
|
-
errorCount: errors.length
|
|
1879
|
-
});
|
|
1880
|
-
} catch (error) {
|
|
1881
|
-
if (this.config.verbose) {
|
|
1882
|
-
console.warn(`[EventualConsistency] GC error:`, error.message);
|
|
1883
|
-
}
|
|
1884
|
-
this.emit('eventual-consistency.gc-error', error);
|
|
1885
|
-
} finally {
|
|
1886
|
-
// Always release GC lock
|
|
1887
|
-
await tryFn(() => this.lockResource.delete(gcLockId));
|
|
1888
|
-
}
|
|
1889
|
-
}
|
|
1890
|
-
|
|
1891
|
-
/**
|
|
1892
|
-
* Update analytics with consolidated transactions
|
|
1893
|
-
* @param {Array} transactions - Array of transactions that were just consolidated
|
|
1894
|
-
* @private
|
|
1895
|
-
*/
|
|
1896
|
-
async updateAnalytics(transactions) {
|
|
1897
|
-
if (!this.analyticsResource || transactions.length === 0) return;
|
|
1898
|
-
|
|
1899
|
-
if (this.config.verbose) {
|
|
1900
|
-
console.log(
|
|
1901
|
-
`[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
|
|
1902
|
-
`Updating analytics for ${transactions.length} transactions...`
|
|
1903
|
-
);
|
|
1904
|
-
}
|
|
1905
|
-
|
|
1906
|
-
try {
|
|
1907
|
-
// Group transactions by cohort hour
|
|
1908
|
-
const byHour = this._groupByCohort(transactions, 'cohortHour');
|
|
1909
|
-
const cohortCount = Object.keys(byHour).length;
|
|
1910
|
-
|
|
1911
|
-
if (this.config.verbose) {
|
|
1912
|
-
console.log(
|
|
1913
|
-
`[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
|
|
1914
|
-
`Updating ${cohortCount} hourly analytics cohorts...`
|
|
1915
|
-
);
|
|
1916
|
-
}
|
|
1917
|
-
|
|
1918
|
-
// Update hourly analytics
|
|
1919
|
-
for (const [cohort, txns] of Object.entries(byHour)) {
|
|
1920
|
-
await this._upsertAnalytics('hour', cohort, txns);
|
|
1921
|
-
}
|
|
1922
|
-
|
|
1923
|
-
// Roll up to daily and monthly if configured
|
|
1924
|
-
if (this.config.analyticsConfig.rollupStrategy === 'incremental') {
|
|
1925
|
-
const uniqueHours = Object.keys(byHour);
|
|
1926
|
-
|
|
1927
|
-
if (this.config.verbose) {
|
|
1928
|
-
console.log(
|
|
1929
|
-
`[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
|
|
1930
|
-
`Rolling up ${uniqueHours.length} hours to daily/monthly analytics...`
|
|
1931
|
-
);
|
|
1932
|
-
}
|
|
1933
|
-
|
|
1934
|
-
for (const cohortHour of uniqueHours) {
|
|
1935
|
-
await this._rollupAnalytics(cohortHour);
|
|
1936
|
-
}
|
|
1937
|
-
}
|
|
1938
|
-
|
|
1939
|
-
if (this.config.verbose) {
|
|
1940
|
-
console.log(
|
|
1941
|
-
`[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
|
|
1942
|
-
`Analytics update complete for ${cohortCount} cohorts`
|
|
1943
|
-
);
|
|
1944
|
-
}
|
|
1945
|
-
} catch (error) {
|
|
1946
|
-
console.warn(
|
|
1947
|
-
`[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
|
|
1948
|
-
`Analytics update error:`,
|
|
1949
|
-
error.message
|
|
1950
|
-
);
|
|
1951
|
-
}
|
|
1952
|
-
}
|
|
1953
|
-
|
|
1954
|
-
/**
|
|
1955
|
-
* Group transactions by cohort
|
|
1956
|
-
* @private
|
|
1957
|
-
*/
|
|
1958
|
-
_groupByCohort(transactions, cohortField) {
|
|
1959
|
-
const groups = {};
|
|
1960
|
-
for (const txn of transactions) {
|
|
1961
|
-
const cohort = txn[cohortField];
|
|
1962
|
-
if (!cohort) continue;
|
|
1963
|
-
|
|
1964
|
-
if (!groups[cohort]) {
|
|
1965
|
-
groups[cohort] = [];
|
|
1966
|
-
}
|
|
1967
|
-
groups[cohort].push(txn);
|
|
1968
|
-
}
|
|
1969
|
-
return groups;
|
|
1970
|
-
}
|
|
1971
|
-
|
|
1972
|
-
/**
|
|
1973
|
-
* Upsert analytics for a specific period and cohort
|
|
1974
|
-
* @private
|
|
1975
|
-
*/
|
|
1976
|
-
async _upsertAnalytics(period, cohort, transactions) {
|
|
1977
|
-
const id = `${period}-${cohort}`;
|
|
1978
|
-
|
|
1979
|
-
// Calculate metrics
|
|
1980
|
-
const transactionCount = transactions.length;
|
|
1981
|
-
|
|
1982
|
-
// Calculate signed values (considering operation type)
|
|
1983
|
-
const signedValues = transactions.map(t => {
|
|
1984
|
-
if (t.operation === 'sub') return -t.value;
|
|
1985
|
-
return t.value;
|
|
1986
|
-
});
|
|
1987
|
-
|
|
1988
|
-
const totalValue = signedValues.reduce((sum, v) => sum + v, 0);
|
|
1989
|
-
const avgValue = totalValue / transactionCount;
|
|
1990
|
-
const minValue = Math.min(...signedValues);
|
|
1991
|
-
const maxValue = Math.max(...signedValues);
|
|
1992
|
-
|
|
1993
|
-
// Calculate operation breakdown
|
|
1994
|
-
const operations = this._calculateOperationBreakdown(transactions);
|
|
1995
|
-
|
|
1996
|
-
// Count distinct records
|
|
1997
|
-
const recordCount = new Set(transactions.map(t => t.originalId)).size;
|
|
1998
|
-
|
|
1999
|
-
const now = new Date().toISOString();
|
|
2000
|
-
|
|
2001
|
-
// Try to get existing analytics
|
|
2002
|
-
const [existingOk, existingErr, existing] = await tryFn(() =>
|
|
2003
|
-
this.analyticsResource.get(id)
|
|
2004
|
-
);
|
|
2005
|
-
|
|
2006
|
-
if (existingOk && existing) {
|
|
2007
|
-
// Update existing analytics (incremental)
|
|
2008
|
-
const newTransactionCount = existing.transactionCount + transactionCount;
|
|
2009
|
-
const newTotalValue = existing.totalValue + totalValue;
|
|
2010
|
-
const newAvgValue = newTotalValue / newTransactionCount;
|
|
2011
|
-
const newMinValue = Math.min(existing.minValue, minValue);
|
|
2012
|
-
const newMaxValue = Math.max(existing.maxValue, maxValue);
|
|
2013
|
-
|
|
2014
|
-
// Merge operation breakdown
|
|
2015
|
-
const newOperations = { ...existing.operations };
|
|
2016
|
-
for (const [op, stats] of Object.entries(operations)) {
|
|
2017
|
-
if (!newOperations[op]) {
|
|
2018
|
-
newOperations[op] = { count: 0, sum: 0 };
|
|
2019
|
-
}
|
|
2020
|
-
newOperations[op].count += stats.count;
|
|
2021
|
-
newOperations[op].sum += stats.sum;
|
|
2022
|
-
}
|
|
2023
|
-
|
|
2024
|
-
// Update record count (approximate - we don't track all unique IDs)
|
|
2025
|
-
const newRecordCount = Math.max(existing.recordCount, recordCount);
|
|
2026
|
-
|
|
2027
|
-
await tryFn(() =>
|
|
2028
|
-
this.analyticsResource.update(id, {
|
|
2029
|
-
transactionCount: newTransactionCount,
|
|
2030
|
-
totalValue: newTotalValue,
|
|
2031
|
-
avgValue: newAvgValue,
|
|
2032
|
-
minValue: newMinValue,
|
|
2033
|
-
maxValue: newMaxValue,
|
|
2034
|
-
operations: newOperations,
|
|
2035
|
-
recordCount: newRecordCount,
|
|
2036
|
-
updatedAt: now
|
|
2037
|
-
})
|
|
2038
|
-
);
|
|
2039
|
-
} else {
|
|
2040
|
-
// Create new analytics
|
|
2041
|
-
await tryFn(() =>
|
|
2042
|
-
this.analyticsResource.insert({
|
|
2043
|
-
id,
|
|
2044
|
-
period,
|
|
2045
|
-
cohort,
|
|
2046
|
-
transactionCount,
|
|
2047
|
-
totalValue,
|
|
2048
|
-
avgValue,
|
|
2049
|
-
minValue,
|
|
2050
|
-
maxValue,
|
|
2051
|
-
operations,
|
|
2052
|
-
recordCount,
|
|
2053
|
-
consolidatedAt: now,
|
|
2054
|
-
updatedAt: now
|
|
2055
|
-
})
|
|
2056
|
-
);
|
|
2057
|
-
}
|
|
2058
|
-
}
|
|
2059
|
-
|
|
2060
|
-
/**
|
|
2061
|
-
* Calculate operation breakdown
|
|
2062
|
-
* @private
|
|
2063
|
-
*/
|
|
2064
|
-
_calculateOperationBreakdown(transactions) {
|
|
2065
|
-
const breakdown = {};
|
|
2066
|
-
|
|
2067
|
-
for (const txn of transactions) {
|
|
2068
|
-
const op = txn.operation;
|
|
2069
|
-
if (!breakdown[op]) {
|
|
2070
|
-
breakdown[op] = { count: 0, sum: 0 };
|
|
2071
|
-
}
|
|
2072
|
-
breakdown[op].count++;
|
|
2073
|
-
|
|
2074
|
-
// Use signed value for sum (sub operations are negative)
|
|
2075
|
-
const signedValue = op === 'sub' ? -txn.value : txn.value;
|
|
2076
|
-
breakdown[op].sum += signedValue;
|
|
2077
|
-
}
|
|
2078
|
-
|
|
2079
|
-
return breakdown;
|
|
2080
|
-
}
|
|
2081
|
-
|
|
2082
|
-
/**
|
|
2083
|
-
* Roll up hourly analytics to daily and monthly
|
|
2084
|
-
* @private
|
|
2085
|
-
*/
|
|
2086
|
-
async _rollupAnalytics(cohortHour) {
|
|
2087
|
-
// cohortHour format: '2025-10-09T14'
|
|
2088
|
-
const cohortDate = cohortHour.substring(0, 10); // '2025-10-09'
|
|
2089
|
-
const cohortMonth = cohortHour.substring(0, 7); // '2025-10'
|
|
2090
|
-
|
|
2091
|
-
// Roll up to day
|
|
2092
|
-
await this._rollupPeriod('day', cohortDate, cohortDate);
|
|
2093
|
-
|
|
2094
|
-
// Roll up to month
|
|
2095
|
-
await this._rollupPeriod('month', cohortMonth, cohortMonth);
|
|
2096
|
-
}
|
|
2097
|
-
|
|
2098
|
-
/**
|
|
2099
|
-
* Roll up analytics for a specific period
|
|
2100
|
-
* @private
|
|
2101
|
-
*/
|
|
2102
|
-
async _rollupPeriod(period, cohort, sourcePrefix) {
|
|
2103
|
-
// Get all source analytics (e.g., all hours for a day)
|
|
2104
|
-
const sourcePeriod = period === 'day' ? 'hour' : 'day';
|
|
2105
|
-
|
|
2106
|
-
const [ok, err, allAnalytics] = await tryFn(() =>
|
|
2107
|
-
this.analyticsResource.list()
|
|
2108
|
-
);
|
|
2109
|
-
|
|
2110
|
-
if (!ok || !allAnalytics) return;
|
|
2111
|
-
|
|
2112
|
-
// Filter to matching cohorts
|
|
2113
|
-
const sourceAnalytics = allAnalytics.filter(a =>
|
|
2114
|
-
a.period === sourcePeriod && a.cohort.startsWith(sourcePrefix)
|
|
2115
|
-
);
|
|
2116
|
-
|
|
2117
|
-
if (sourceAnalytics.length === 0) return;
|
|
2118
|
-
|
|
2119
|
-
// Aggregate metrics
|
|
2120
|
-
const transactionCount = sourceAnalytics.reduce((sum, a) => sum + a.transactionCount, 0);
|
|
2121
|
-
const totalValue = sourceAnalytics.reduce((sum, a) => sum + a.totalValue, 0);
|
|
2122
|
-
const avgValue = totalValue / transactionCount;
|
|
2123
|
-
const minValue = Math.min(...sourceAnalytics.map(a => a.minValue));
|
|
2124
|
-
const maxValue = Math.max(...sourceAnalytics.map(a => a.maxValue));
|
|
2125
|
-
|
|
2126
|
-
// Merge operation breakdown
|
|
2127
|
-
const operations = {};
|
|
2128
|
-
for (const analytics of sourceAnalytics) {
|
|
2129
|
-
for (const [op, stats] of Object.entries(analytics.operations || {})) {
|
|
2130
|
-
if (!operations[op]) {
|
|
2131
|
-
operations[op] = { count: 0, sum: 0 };
|
|
2132
|
-
}
|
|
2133
|
-
operations[op].count += stats.count;
|
|
2134
|
-
operations[op].sum += stats.sum;
|
|
2135
|
-
}
|
|
2136
|
-
}
|
|
2137
|
-
|
|
2138
|
-
// Approximate record count (max of all periods)
|
|
2139
|
-
const recordCount = Math.max(...sourceAnalytics.map(a => a.recordCount));
|
|
2140
|
-
|
|
2141
|
-
const id = `${period}-${cohort}`;
|
|
2142
|
-
const now = new Date().toISOString();
|
|
2143
|
-
|
|
2144
|
-
// Upsert rolled-up analytics
|
|
2145
|
-
const [existingOk, existingErr, existing] = await tryFn(() =>
|
|
2146
|
-
this.analyticsResource.get(id)
|
|
2147
|
-
);
|
|
2148
|
-
|
|
2149
|
-
if (existingOk && existing) {
|
|
2150
|
-
await tryFn(() =>
|
|
2151
|
-
this.analyticsResource.update(id, {
|
|
2152
|
-
transactionCount,
|
|
2153
|
-
totalValue,
|
|
2154
|
-
avgValue,
|
|
2155
|
-
minValue,
|
|
2156
|
-
maxValue,
|
|
2157
|
-
operations,
|
|
2158
|
-
recordCount,
|
|
2159
|
-
updatedAt: now
|
|
2160
|
-
})
|
|
2161
|
-
);
|
|
2162
|
-
} else {
|
|
2163
|
-
await tryFn(() =>
|
|
2164
|
-
this.analyticsResource.insert({
|
|
2165
|
-
id,
|
|
2166
|
-
period,
|
|
2167
|
-
cohort,
|
|
2168
|
-
transactionCount,
|
|
2169
|
-
totalValue,
|
|
2170
|
-
avgValue,
|
|
2171
|
-
minValue,
|
|
2172
|
-
maxValue,
|
|
2173
|
-
operations,
|
|
2174
|
-
recordCount,
|
|
2175
|
-
consolidatedAt: now,
|
|
2176
|
-
updatedAt: now
|
|
2177
|
-
})
|
|
2178
|
-
);
|
|
2179
|
-
}
|
|
2180
|
-
}
|
|
2181
|
-
|
|
2182
|
-
/**
|
|
2183
|
-
* Get analytics for a specific period
|
|
2184
|
-
* @param {string} resourceName - Resource name
|
|
2185
|
-
* @param {string} field - Field name
|
|
2186
|
-
* @param {Object} options - Query options
|
|
2187
|
-
* @returns {Promise<Array>} Analytics data
|
|
2188
|
-
*/
|
|
2189
|
-
async getAnalytics(resourceName, field, options = {}) {
|
|
2190
|
-
// Get handler for this resource/field combination
|
|
2191
|
-
const fieldHandlers = this.fieldHandlers.get(resourceName);
|
|
2192
|
-
if (!fieldHandlers) {
|
|
2193
|
-
throw new Error(`No eventual consistency configured for resource: ${resourceName}`);
|
|
2194
|
-
}
|
|
2195
|
-
|
|
2196
|
-
const handler = fieldHandlers.get(field);
|
|
2197
|
-
if (!handler) {
|
|
2198
|
-
throw new Error(`No eventual consistency configured for field: ${resourceName}.${field}`);
|
|
2199
|
-
}
|
|
2200
|
-
|
|
2201
|
-
if (!handler.analyticsResource) {
|
|
2202
|
-
throw new Error('Analytics not enabled for this plugin');
|
|
2203
|
-
}
|
|
2204
|
-
|
|
2205
|
-
const { period = 'day', date, startDate, endDate, month, year, breakdown = false } = options;
|
|
2206
|
-
|
|
2207
|
-
const [ok, err, allAnalytics] = await tryFn(() =>
|
|
2208
|
-
handler.analyticsResource.list()
|
|
2209
|
-
);
|
|
2210
|
-
|
|
2211
|
-
if (!ok || !allAnalytics) {
|
|
2212
|
-
return [];
|
|
2213
|
-
}
|
|
2214
|
-
|
|
2215
|
-
// Filter by period
|
|
2216
|
-
let filtered = allAnalytics.filter(a => a.period === period);
|
|
2217
|
-
|
|
2218
|
-
// Filter by date/range
|
|
2219
|
-
if (date) {
|
|
2220
|
-
if (period === 'hour') {
|
|
2221
|
-
// Match all hours of the date
|
|
2222
|
-
filtered = filtered.filter(a => a.cohort.startsWith(date));
|
|
2223
|
-
} else {
|
|
2224
|
-
filtered = filtered.filter(a => a.cohort === date);
|
|
2225
|
-
}
|
|
2226
|
-
} else if (startDate && endDate) {
|
|
2227
|
-
filtered = filtered.filter(a => a.cohort >= startDate && a.cohort <= endDate);
|
|
2228
|
-
} else if (month) {
|
|
2229
|
-
filtered = filtered.filter(a => a.cohort.startsWith(month));
|
|
2230
|
-
} else if (year) {
|
|
2231
|
-
filtered = filtered.filter(a => a.cohort.startsWith(String(year)));
|
|
2232
|
-
}
|
|
2233
|
-
|
|
2234
|
-
// Sort by cohort
|
|
2235
|
-
filtered.sort((a, b) => a.cohort.localeCompare(b.cohort));
|
|
2236
|
-
|
|
2237
|
-
// Return with or without breakdown
|
|
2238
|
-
if (breakdown === 'operations') {
|
|
2239
|
-
return filtered.map(a => ({
|
|
2240
|
-
cohort: a.cohort,
|
|
2241
|
-
...a.operations
|
|
2242
|
-
}));
|
|
2243
|
-
}
|
|
2244
|
-
|
|
2245
|
-
return filtered.map(a => ({
|
|
2246
|
-
cohort: a.cohort,
|
|
2247
|
-
count: a.transactionCount,
|
|
2248
|
-
sum: a.totalValue,
|
|
2249
|
-
avg: a.avgValue,
|
|
2250
|
-
min: a.minValue,
|
|
2251
|
-
max: a.maxValue,
|
|
2252
|
-
operations: a.operations,
|
|
2253
|
-
recordCount: a.recordCount
|
|
2254
|
-
}));
|
|
2255
|
-
}
|
|
2256
|
-
|
|
2257
|
-
/**
|
|
2258
|
-
* Fill gaps in analytics data with zeros for continuous time series
|
|
2259
|
-
* @private
|
|
2260
|
-
* @param {Array} data - Sparse analytics data
|
|
2261
|
-
* @param {string} period - Period type ('hour', 'day', 'month')
|
|
2262
|
-
* @param {string} startDate - Start date (ISO format)
|
|
2263
|
-
* @param {string} endDate - End date (ISO format)
|
|
2264
|
-
* @returns {Array} Complete time series with gaps filled
|
|
2265
|
-
*/
|
|
2266
|
-
_fillGaps(data, period, startDate, endDate) {
|
|
2267
|
-
if (!data || data.length === 0) {
|
|
2268
|
-
// If no data, still generate empty series
|
|
2269
|
-
data = [];
|
|
2270
|
-
}
|
|
2271
|
-
|
|
2272
|
-
// Create a map of existing data by cohort
|
|
2273
|
-
const dataMap = new Map();
|
|
2274
|
-
data.forEach(item => {
|
|
2275
|
-
dataMap.set(item.cohort, item);
|
|
2276
|
-
});
|
|
2277
|
-
|
|
2278
|
-
const result = [];
|
|
2279
|
-
const emptyRecord = {
|
|
2280
|
-
count: 0,
|
|
2281
|
-
sum: 0,
|
|
2282
|
-
avg: 0,
|
|
2283
|
-
min: 0,
|
|
2284
|
-
max: 0,
|
|
2285
|
-
recordCount: 0
|
|
2286
|
-
};
|
|
2287
|
-
|
|
2288
|
-
if (period === 'hour') {
|
|
2289
|
-
// Generate all hours between startDate and endDate
|
|
2290
|
-
const start = new Date(startDate + 'T00:00:00Z');
|
|
2291
|
-
const end = new Date(endDate + 'T23:59:59Z');
|
|
2292
|
-
|
|
2293
|
-
for (let dt = new Date(start); dt <= end; dt.setHours(dt.getHours() + 1)) {
|
|
2294
|
-
const cohort = dt.toISOString().substring(0, 13); // YYYY-MM-DDTHH
|
|
2295
|
-
result.push(dataMap.get(cohort) || { cohort, ...emptyRecord });
|
|
2296
|
-
}
|
|
2297
|
-
} else if (period === 'day') {
|
|
2298
|
-
// Generate all days between startDate and endDate
|
|
2299
|
-
const start = new Date(startDate);
|
|
2300
|
-
const end = new Date(endDate);
|
|
2301
|
-
|
|
2302
|
-
for (let dt = new Date(start); dt <= end; dt.setDate(dt.getDate() + 1)) {
|
|
2303
|
-
const cohort = dt.toISOString().substring(0, 10); // YYYY-MM-DD
|
|
2304
|
-
result.push(dataMap.get(cohort) || { cohort, ...emptyRecord });
|
|
2305
|
-
}
|
|
2306
|
-
} else if (period === 'month') {
|
|
2307
|
-
// Generate all months between startDate and endDate
|
|
2308
|
-
const startYear = parseInt(startDate.substring(0, 4));
|
|
2309
|
-
const startMonth = parseInt(startDate.substring(5, 7));
|
|
2310
|
-
const endYear = parseInt(endDate.substring(0, 4));
|
|
2311
|
-
const endMonth = parseInt(endDate.substring(5, 7));
|
|
2312
|
-
|
|
2313
|
-
for (let year = startYear; year <= endYear; year++) {
|
|
2314
|
-
const firstMonth = (year === startYear) ? startMonth : 1;
|
|
2315
|
-
const lastMonth = (year === endYear) ? endMonth : 12;
|
|
2316
|
-
|
|
2317
|
-
for (let month = firstMonth; month <= lastMonth; month++) {
|
|
2318
|
-
const cohort = `${year}-${month.toString().padStart(2, '0')}`;
|
|
2319
|
-
result.push(dataMap.get(cohort) || { cohort, ...emptyRecord });
|
|
2320
|
-
}
|
|
2321
|
-
}
|
|
2322
|
-
}
|
|
2323
|
-
|
|
2324
|
-
return result;
|
|
2325
|
-
}
|
|
2326
|
-
|
|
2327
|
-
/**
|
|
2328
|
-
* Get analytics for entire month, broken down by days
|
|
2329
|
-
* @param {string} resourceName - Resource name
|
|
2330
|
-
* @param {string} field - Field name
|
|
2331
|
-
* @param {string} month - Month in YYYY-MM format
|
|
2332
|
-
* @param {Object} options - Options
|
|
2333
|
-
* @param {boolean} options.fillGaps - Fill missing days with zeros (default: false)
|
|
2334
|
-
* @returns {Promise<Array>} Daily analytics for the month
|
|
2335
|
-
*/
|
|
2336
|
-
async getMonthByDay(resourceName, field, month, options = {}) {
|
|
2337
|
-
// month format: '2025-10'
|
|
2338
|
-
const year = parseInt(month.substring(0, 4));
|
|
2339
|
-
const monthNum = parseInt(month.substring(5, 7));
|
|
2340
|
-
|
|
2341
|
-
// Get first and last day of month
|
|
2342
|
-
const firstDay = new Date(year, monthNum - 1, 1);
|
|
2343
|
-
const lastDay = new Date(year, monthNum, 0);
|
|
2344
|
-
|
|
2345
|
-
const startDate = firstDay.toISOString().substring(0, 10);
|
|
2346
|
-
const endDate = lastDay.toISOString().substring(0, 10);
|
|
2347
|
-
|
|
2348
|
-
const data = await this.getAnalytics(resourceName, field, {
|
|
2349
|
-
period: 'day',
|
|
2350
|
-
startDate,
|
|
2351
|
-
endDate
|
|
2352
|
-
});
|
|
2353
|
-
|
|
2354
|
-
if (options.fillGaps) {
|
|
2355
|
-
return this._fillGaps(data, 'day', startDate, endDate);
|
|
2356
|
-
}
|
|
2357
|
-
|
|
2358
|
-
return data;
|
|
2359
|
-
}
|
|
2360
|
-
|
|
2361
|
-
/**
|
|
2362
|
-
* Get analytics for entire day, broken down by hours
|
|
2363
|
-
* @param {string} resourceName - Resource name
|
|
2364
|
-
* @param {string} field - Field name
|
|
2365
|
-
* @param {string} date - Date in YYYY-MM-DD format
|
|
2366
|
-
* @param {Object} options - Options
|
|
2367
|
-
* @param {boolean} options.fillGaps - Fill missing hours with zeros (default: false)
|
|
2368
|
-
* @returns {Promise<Array>} Hourly analytics for the day
|
|
2369
|
-
*/
|
|
2370
|
-
async getDayByHour(resourceName, field, date, options = {}) {
|
|
2371
|
-
// date format: '2025-10-09'
|
|
2372
|
-
const data = await this.getAnalytics(resourceName, field, {
|
|
2373
|
-
period: 'hour',
|
|
2374
|
-
date
|
|
2375
|
-
});
|
|
2376
|
-
|
|
2377
|
-
if (options.fillGaps) {
|
|
2378
|
-
return this._fillGaps(data, 'hour', date, date);
|
|
2379
|
-
}
|
|
2380
|
-
|
|
2381
|
-
return data;
|
|
2382
|
-
}
|
|
2383
|
-
|
|
2384
|
-
/**
|
|
2385
|
-
* Get analytics for last N days, broken down by days
|
|
2386
|
-
* @param {string} resourceName - Resource name
|
|
2387
|
-
* @param {string} field - Field name
|
|
2388
|
-
* @param {number} days - Number of days to look back (default: 7)
|
|
2389
|
-
* @param {Object} options - Options
|
|
2390
|
-
* @param {boolean} options.fillGaps - Fill missing days with zeros (default: false)
|
|
2391
|
-
* @returns {Promise<Array>} Daily analytics
|
|
2392
|
-
*/
|
|
2393
|
-
async getLastNDays(resourceName, field, days = 7, options = {}) {
|
|
2394
|
-
const dates = Array.from({ length: days }, (_, i) => {
|
|
2395
|
-
const date = new Date();
|
|
2396
|
-
date.setDate(date.getDate() - i);
|
|
2397
|
-
return date.toISOString().substring(0, 10);
|
|
2398
|
-
}).reverse();
|
|
2399
|
-
|
|
2400
|
-
const data = await this.getAnalytics(resourceName, field, {
|
|
2401
|
-
period: 'day',
|
|
2402
|
-
startDate: dates[0],
|
|
2403
|
-
endDate: dates[dates.length - 1]
|
|
2404
|
-
});
|
|
2405
|
-
|
|
2406
|
-
if (options.fillGaps) {
|
|
2407
|
-
return this._fillGaps(data, 'day', dates[0], dates[dates.length - 1]);
|
|
2408
|
-
}
|
|
2409
|
-
|
|
2410
|
-
return data;
|
|
2411
|
-
}
|
|
2412
|
-
|
|
2413
|
-
/**
|
|
2414
|
-
* Get analytics for entire year, broken down by months
|
|
2415
|
-
* @param {string} resourceName - Resource name
|
|
2416
|
-
* @param {string} field - Field name
|
|
2417
|
-
* @param {number} year - Year (e.g., 2025)
|
|
2418
|
-
* @param {Object} options - Options
|
|
2419
|
-
* @param {boolean} options.fillGaps - Fill missing months with zeros (default: false)
|
|
2420
|
-
* @returns {Promise<Array>} Monthly analytics for the year
|
|
2421
|
-
*/
|
|
2422
|
-
async getYearByMonth(resourceName, field, year, options = {}) {
|
|
2423
|
-
const data = await this.getAnalytics(resourceName, field, {
|
|
2424
|
-
period: 'month',
|
|
2425
|
-
year
|
|
2426
|
-
});
|
|
2427
|
-
|
|
2428
|
-
if (options.fillGaps) {
|
|
2429
|
-
const startDate = `${year}-01`;
|
|
2430
|
-
const endDate = `${year}-12`;
|
|
2431
|
-
return this._fillGaps(data, 'month', startDate, endDate);
|
|
2432
|
-
}
|
|
2433
|
-
|
|
2434
|
-
return data;
|
|
2435
|
-
}
|
|
2436
|
-
|
|
2437
|
-
/**
|
|
2438
|
-
* Get analytics for entire month, broken down by hours
|
|
2439
|
-
* @param {string} resourceName - Resource name
|
|
2440
|
-
* @param {string} field - Field name
|
|
2441
|
-
* @param {string} month - Month in YYYY-MM format (or 'last' for previous month)
|
|
2442
|
-
* @param {Object} options - Options
|
|
2443
|
-
* @param {boolean} options.fillGaps - Fill missing hours with zeros (default: false)
|
|
2444
|
-
* @returns {Promise<Array>} Hourly analytics for the month (up to 24*31=744 records)
|
|
2445
|
-
*/
|
|
2446
|
-
async getMonthByHour(resourceName, field, month, options = {}) {
|
|
2447
|
-
// month format: '2025-10' or 'last'
|
|
2448
|
-
let year, monthNum;
|
|
2449
|
-
|
|
2450
|
-
if (month === 'last') {
|
|
2451
|
-
const now = new Date();
|
|
2452
|
-
now.setMonth(now.getMonth() - 1);
|
|
2453
|
-
year = now.getFullYear();
|
|
2454
|
-
monthNum = now.getMonth() + 1;
|
|
2455
|
-
} else {
|
|
2456
|
-
year = parseInt(month.substring(0, 4));
|
|
2457
|
-
monthNum = parseInt(month.substring(5, 7));
|
|
2458
|
-
}
|
|
2459
|
-
|
|
2460
|
-
// Get first and last day of month
|
|
2461
|
-
const firstDay = new Date(year, monthNum - 1, 1);
|
|
2462
|
-
const lastDay = new Date(year, monthNum, 0);
|
|
2463
|
-
|
|
2464
|
-
const startDate = firstDay.toISOString().substring(0, 10);
|
|
2465
|
-
const endDate = lastDay.toISOString().substring(0, 10);
|
|
2466
|
-
|
|
2467
|
-
const data = await this.getAnalytics(resourceName, field, {
|
|
2468
|
-
period: 'hour',
|
|
2469
|
-
startDate,
|
|
2470
|
-
endDate
|
|
2471
|
-
});
|
|
2472
|
-
|
|
2473
|
-
if (options.fillGaps) {
|
|
2474
|
-
return this._fillGaps(data, 'hour', startDate, endDate);
|
|
2475
|
-
}
|
|
2476
|
-
|
|
2477
|
-
return data;
|
|
2478
|
-
}
|
|
2479
|
-
|
|
2480
|
-
/**
|
|
2481
|
-
* Get top records by volume
|
|
2482
|
-
* @param {string} resourceName - Resource name
|
|
2483
|
-
* @param {string} field - Field name
|
|
2484
|
-
* @param {Object} options - Query options
|
|
2485
|
-
* @returns {Promise<Array>} Top records
|
|
2486
|
-
*/
|
|
2487
|
-
async getTopRecords(resourceName, field, options = {}) {
|
|
2488
|
-
// Get handler for this resource/field combination
|
|
2489
|
-
const fieldHandlers = this.fieldHandlers.get(resourceName);
|
|
2490
|
-
if (!fieldHandlers) {
|
|
2491
|
-
throw new Error(`No eventual consistency configured for resource: ${resourceName}`);
|
|
2492
|
-
}
|
|
2493
|
-
|
|
2494
|
-
const handler = fieldHandlers.get(field);
|
|
2495
|
-
if (!handler) {
|
|
2496
|
-
throw new Error(`No eventual consistency configured for field: ${resourceName}.${field}`);
|
|
2497
|
-
}
|
|
2498
|
-
|
|
2499
|
-
if (!handler.transactionResource) {
|
|
2500
|
-
throw new Error('Transaction resource not initialized');
|
|
2501
|
-
}
|
|
2502
|
-
|
|
2503
|
-
const { period = 'day', date, metric = 'transactionCount', limit = 10 } = options;
|
|
2504
|
-
|
|
2505
|
-
// Get all transactions for the period
|
|
2506
|
-
const [ok, err, transactions] = await tryFn(() =>
|
|
2507
|
-
handler.transactionResource.list()
|
|
2508
|
-
);
|
|
2509
|
-
|
|
2510
|
-
if (!ok || !transactions) {
|
|
2511
|
-
return [];
|
|
2512
|
-
}
|
|
2513
|
-
|
|
2514
|
-
// Filter by date
|
|
2515
|
-
let filtered = transactions;
|
|
2516
|
-
if (date) {
|
|
2517
|
-
if (period === 'hour') {
|
|
2518
|
-
filtered = transactions.filter(t => t.cohortHour && t.cohortHour.startsWith(date));
|
|
2519
|
-
} else if (period === 'day') {
|
|
2520
|
-
filtered = transactions.filter(t => t.cohortDate === date);
|
|
2521
|
-
} else if (period === 'month') {
|
|
2522
|
-
filtered = transactions.filter(t => t.cohortMonth && t.cohortMonth.startsWith(date));
|
|
2523
|
-
}
|
|
2524
|
-
}
|
|
2525
|
-
|
|
2526
|
-
// Group by originalId
|
|
2527
|
-
const byRecord = {};
|
|
2528
|
-
for (const txn of filtered) {
|
|
2529
|
-
const recordId = txn.originalId;
|
|
2530
|
-
if (!byRecord[recordId]) {
|
|
2531
|
-
byRecord[recordId] = { count: 0, sum: 0 };
|
|
2532
|
-
}
|
|
2533
|
-
byRecord[recordId].count++;
|
|
2534
|
-
byRecord[recordId].sum += txn.value;
|
|
2535
|
-
}
|
|
2536
|
-
|
|
2537
|
-
// Convert to array and sort
|
|
2538
|
-
const records = Object.entries(byRecord).map(([recordId, stats]) => ({
|
|
2539
|
-
recordId,
|
|
2540
|
-
count: stats.count,
|
|
2541
|
-
sum: stats.sum
|
|
2542
|
-
}));
|
|
2543
|
-
|
|
2544
|
-
// Sort by metric
|
|
2545
|
-
records.sort((a, b) => {
|
|
2546
|
-
if (metric === 'transactionCount') {
|
|
2547
|
-
return b.count - a.count;
|
|
2548
|
-
} else if (metric === 'totalValue') {
|
|
2549
|
-
return b.sum - a.sum;
|
|
2550
|
-
}
|
|
2551
|
-
return 0;
|
|
2552
|
-
});
|
|
2553
|
-
|
|
2554
|
-
// Limit results
|
|
2555
|
-
return records.slice(0, limit);
|
|
2556
|
-
}
|
|
2557
|
-
}
|
|
2558
|
-
|
|
2559
|
-
export default EventualConsistencyPlugin;
|