s3db.js 10.0.17 → 10.0.19

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,2559 +0,0 @@
1
- import Plugin from "./plugin.class.js";
2
- import tryFn from "../concerns/try-fn.js";
3
- import { idGenerator } from "../concerns/id.js";
4
- import { PromisePool } from "@supercharge/promise-pool";
5
-
6
- export class EventualConsistencyPlugin extends Plugin {
7
- constructor(options = {}) {
8
- super(options);
9
-
10
- // Validate resources structure
11
- if (!options.resources || typeof options.resources !== 'object') {
12
- throw new Error(
13
- "EventualConsistencyPlugin requires 'resources' option.\n" +
14
- "Example: { resources: { urls: ['clicks', 'views'], posts: ['likes'] } }"
15
- );
16
- }
17
-
18
- // Auto-detect timezone from environment or system
19
- const detectedTimezone = this._detectTimezone();
20
-
21
- // Create shared configuration
22
- this.config = {
23
- cohort: {
24
- timezone: options.cohort?.timezone || detectedTimezone
25
- },
26
- reducer: options.reducer || ((transactions) => {
27
- let baseValue = 0;
28
- for (const t of transactions) {
29
- if (t.operation === 'set') {
30
- baseValue = t.value;
31
- } else if (t.operation === 'add') {
32
- baseValue += t.value;
33
- } else if (t.operation === 'sub') {
34
- baseValue -= t.value;
35
- }
36
- }
37
- return baseValue;
38
- }),
39
- consolidationInterval: options.consolidationInterval ?? 300,
40
- consolidationConcurrency: options.consolidationConcurrency || 5,
41
- consolidationWindow: options.consolidationWindow || 24,
42
- autoConsolidate: options.autoConsolidate !== false,
43
- lateArrivalStrategy: options.lateArrivalStrategy || 'warn',
44
- batchTransactions: options.batchTransactions || false,
45
- batchSize: options.batchSize || 100,
46
- mode: options.mode || 'async',
47
- lockTimeout: options.lockTimeout || 300,
48
- transactionRetention: options.transactionRetention || 30,
49
- gcInterval: options.gcInterval || 86400,
50
- verbose: options.verbose || false,
51
- enableAnalytics: options.enableAnalytics || false,
52
- analyticsConfig: {
53
- periods: options.analyticsConfig?.periods || ['hour', 'day', 'month'],
54
- metrics: options.analyticsConfig?.metrics || ['count', 'sum', 'avg', 'min', 'max'],
55
- rollupStrategy: options.analyticsConfig?.rollupStrategy || 'incremental',
56
- retentionDays: options.analyticsConfig?.retentionDays || 365
57
- },
58
- // Checkpoint configuration for high-volume scenarios
59
- enableCheckpoints: options.enableCheckpoints !== false, // Default: true
60
- checkpointStrategy: options.checkpointStrategy || 'hourly', // 'hourly', 'daily', 'manual', 'disabled'
61
- checkpointRetention: options.checkpointRetention || 90, // Days to keep checkpoints
62
- checkpointThreshold: options.checkpointThreshold || 1000, // Min transactions before creating checkpoint
63
- deleteConsolidatedTransactions: options.deleteConsolidatedTransactions !== false, // Delete transactions after checkpoint
64
- autoCheckpoint: options.autoCheckpoint !== false // Auto-create checkpoints for old cohorts
65
- };
66
-
67
- // Create field handlers map
68
- this.fieldHandlers = new Map(); // Map<resourceName, Map<fieldName, handler>>
69
-
70
- // Parse resources configuration
71
- for (const [resourceName, fields] of Object.entries(options.resources)) {
72
- if (!Array.isArray(fields)) {
73
- throw new Error(
74
- `EventualConsistencyPlugin resources.${resourceName} must be an array of field names`
75
- );
76
- }
77
-
78
- const resourceHandlers = new Map();
79
- for (const fieldName of fields) {
80
- // Create a field handler for each resource/field combination
81
- resourceHandlers.set(fieldName, this._createFieldHandler(resourceName, fieldName));
82
- }
83
- this.fieldHandlers.set(resourceName, resourceHandlers);
84
- }
85
-
86
- // Warn about batching in distributed environments
87
- if (this.config.batchTransactions && !this.config.verbose) {
88
- console.warn(
89
- `[EventualConsistency] WARNING: batchTransactions is enabled. ` +
90
- `This stores transactions in memory and will lose data if container crashes. ` +
91
- `Not recommended for distributed/production environments.`
92
- );
93
- }
94
-
95
- // Log initialization if verbose
96
- if (this.config.verbose) {
97
- const totalFields = Array.from(this.fieldHandlers.values())
98
- .reduce((sum, handlers) => sum + handlers.size, 0);
99
- console.log(
100
- `[EventualConsistency] Initialized with ${this.fieldHandlers.size} resource(s), ` +
101
- `${totalFields} field(s) total`
102
- );
103
-
104
- // Log detected timezone if not explicitly set
105
- if (!options.cohort?.timezone) {
106
- console.log(
107
- `[EventualConsistency] Auto-detected timezone: ${this.config.cohort.timezone} ` +
108
- `(from ${process.env.TZ ? 'TZ env var' : 'system Intl API'})`
109
- );
110
- }
111
- }
112
- }
113
-
114
- /**
115
- * Create a field handler for a specific resource/field combination
116
- * @private
117
- */
118
- _createFieldHandler(resourceName, fieldName) {
119
- return {
120
- resource: resourceName,
121
- field: fieldName,
122
- transactionResource: null,
123
- targetResource: null,
124
- analyticsResource: null,
125
- lockResource: null,
126
- checkpointResource: null, // NEW: Checkpoint resource for high-volume optimization
127
- consolidationTimer: null,
128
- gcTimer: null,
129
- pendingTransactions: new Map(),
130
- deferredSetup: false
131
- };
132
- }
133
-
134
- async onSetup() {
135
- // Iterate over all resource/field combinations
136
- for (const [resourceName, fieldHandlers] of this.fieldHandlers) {
137
- const targetResource = this.database.resources[resourceName];
138
-
139
- if (!targetResource) {
140
- // Resource doesn't exist yet - mark for deferred setup
141
- for (const handler of fieldHandlers.values()) {
142
- handler.deferredSetup = true;
143
- }
144
- // Watch for this resource to be created
145
- this._watchForResource(resourceName);
146
- continue;
147
- }
148
-
149
- // Resource exists - setup all fields for this resource
150
- for (const [fieldName, handler] of fieldHandlers) {
151
- handler.targetResource = targetResource;
152
- await this._completeFieldSetup(handler);
153
- }
154
- }
155
- }
156
-
157
- /**
158
- * Watch for a specific resource creation
159
- * @private
160
- */
161
- _watchForResource(resourceName) {
162
- const hookCallback = async ({ resource, config }) => {
163
- if (config.name === resourceName) {
164
- const fieldHandlers = this.fieldHandlers.get(resourceName);
165
- if (!fieldHandlers) return;
166
-
167
- // Setup all fields for this resource
168
- for (const [fieldName, handler] of fieldHandlers) {
169
- if (handler.deferredSetup) {
170
- handler.targetResource = resource;
171
- handler.deferredSetup = false;
172
- await this._completeFieldSetup(handler);
173
- }
174
- }
175
- }
176
- };
177
-
178
- this.database.addHook('afterCreateResource', hookCallback);
179
- }
180
-
181
- /**
182
- * Complete setup for a single field handler
183
- * @private
184
- */
185
- async _completeFieldSetup(handler) {
186
- if (!handler.targetResource) return;
187
-
188
- const config = this.config;
189
- const resourceName = handler.resource;
190
- const fieldName = handler.field;
191
-
192
- // Create transaction resource with partitions
193
- const transactionResourceName = `${resourceName}_transactions_${fieldName}`;
194
- const partitionConfig = this.createPartitionConfig();
195
-
196
- const [ok, err, transactionResource] = await tryFn(() =>
197
- this.database.createResource({
198
- name: transactionResourceName,
199
- attributes: {
200
- id: 'string|required',
201
- originalId: 'string|required',
202
- field: 'string|required',
203
- value: 'number|required',
204
- operation: 'string|required',
205
- timestamp: 'string|required',
206
- cohortDate: 'string|required',
207
- cohortHour: 'string|required',
208
- cohortMonth: 'string|optional',
209
- source: 'string|optional',
210
- applied: 'boolean|optional'
211
- },
212
- behavior: 'body-overflow',
213
- timestamps: true,
214
- partitions: partitionConfig,
215
- asyncPartitions: true,
216
- createdBy: 'EventualConsistencyPlugin'
217
- })
218
- );
219
-
220
- if (!ok && !this.database.resources[transactionResourceName]) {
221
- throw new Error(`Failed to create transaction resource for ${resourceName}.${fieldName}: ${err?.message}`);
222
- }
223
-
224
- handler.transactionResource = ok ? transactionResource : this.database.resources[transactionResourceName];
225
-
226
- // Create lock resource
227
- const lockResourceName = `${resourceName}_consolidation_locks_${fieldName}`;
228
- const [lockOk, lockErr, lockResource] = await tryFn(() =>
229
- this.database.createResource({
230
- name: lockResourceName,
231
- attributes: {
232
- id: 'string|required',
233
- lockedAt: 'number|required',
234
- workerId: 'string|optional'
235
- },
236
- behavior: 'body-only',
237
- timestamps: false,
238
- createdBy: 'EventualConsistencyPlugin'
239
- })
240
- );
241
-
242
- if (!lockOk && !this.database.resources[lockResourceName]) {
243
- throw new Error(`Failed to create lock resource for ${resourceName}.${fieldName}: ${lockErr?.message}`);
244
- }
245
-
246
- handler.lockResource = lockOk ? lockResource : this.database.resources[lockResourceName];
247
-
248
- // Create analytics resource if enabled
249
- if (config.enableAnalytics) {
250
- await this._createAnalyticsResourceForHandler(handler);
251
- }
252
-
253
- // Add helper methods to the target resource
254
- this._addHelperMethodsForHandler(handler);
255
-
256
- // Setup timers (TODO: implement timer management for handlers)
257
- // For now, we'll skip auto-consolidation in multi-resource mode
258
-
259
- if (config.verbose) {
260
- console.log(
261
- `[EventualConsistency] ${resourceName}.${fieldName} - ` +
262
- `Setup complete. Resources: ${transactionResourceName}, ${lockResourceName}` +
263
- `${config.enableAnalytics ? `, ${resourceName}_analytics_${fieldName}` : ''}`
264
- );
265
- }
266
- }
267
-
268
- /**
269
- * Create analytics resource for a field handler
270
- * @private
271
- */
272
- async _createAnalyticsResourceForHandler(handler) {
273
- const resourceName = handler.resource;
274
- const fieldName = handler.field;
275
- const analyticsResourceName = `${resourceName}_analytics_${fieldName}`;
276
-
277
- const [ok, err, analyticsResource] = await tryFn(() =>
278
- this.database.createResource({
279
- name: analyticsResourceName,
280
- attributes: {
281
- id: 'string|required',
282
- period: 'string|required',
283
- cohort: 'string|required',
284
- transactionCount: 'number|required',
285
- totalValue: 'number|required',
286
- avgValue: 'number|required',
287
- minValue: 'number|required',
288
- maxValue: 'number|required',
289
- operations: 'object|optional',
290
- recordCount: 'number|required',
291
- consolidatedAt: 'string|required',
292
- updatedAt: 'string|required'
293
- },
294
- behavior: 'body-overflow',
295
- timestamps: false,
296
- createdBy: 'EventualConsistencyPlugin'
297
- })
298
- );
299
-
300
- if (!ok && !this.database.resources[analyticsResourceName]) {
301
- throw new Error(`Failed to create analytics resource for ${resourceName}.${fieldName}: ${err?.message}`);
302
- }
303
-
304
- handler.analyticsResource = ok ? analyticsResource : this.database.resources[analyticsResourceName];
305
- }
306
-
307
- /**
308
- * Add helper methods to the target resource for a field handler
309
- * @private
310
- */
311
- _addHelperMethodsForHandler(handler) {
312
- const resource = handler.targetResource;
313
- const fieldName = handler.field;
314
-
315
- // Store handler reference on the resource for later access
316
- if (!resource._eventualConsistencyPlugins) {
317
- resource._eventualConsistencyPlugins = {};
318
- }
319
- resource._eventualConsistencyPlugins[fieldName] = handler;
320
-
321
- // Add helper methods if not already added
322
- if (!resource.add) {
323
- this.addHelperMethods(); // Add all helper methods once
324
- }
325
- }
326
-
327
- async onStart() {
328
- // Start timers and emit events for all field handlers
329
- for (const [resourceName, fieldHandlers] of this.fieldHandlers) {
330
- for (const [fieldName, handler] of fieldHandlers) {
331
- if (!handler.deferredSetup) {
332
- // Start auto-consolidation timer if enabled
333
- if (this.config.autoConsolidate && this.config.mode === 'async') {
334
- this.startConsolidationTimerForHandler(handler, resourceName, fieldName);
335
- }
336
-
337
- // Start garbage collection timer
338
- if (this.config.transactionRetention && this.config.transactionRetention > 0) {
339
- this.startGarbageCollectionTimerForHandler(handler, resourceName, fieldName);
340
- }
341
-
342
- this.emit('eventual-consistency.started', {
343
- resource: resourceName,
344
- field: fieldName,
345
- cohort: this.config.cohort
346
- });
347
- }
348
- }
349
- }
350
- }
351
-
352
- async onStop() {
353
- // Stop all timers for all handlers
354
- for (const [resourceName, fieldHandlers] of this.fieldHandlers) {
355
- for (const [fieldName, handler] of fieldHandlers) {
356
- // Stop consolidation timer
357
- if (handler.consolidationTimer) {
358
- clearInterval(handler.consolidationTimer);
359
- handler.consolidationTimer = null;
360
- }
361
-
362
- // Stop garbage collection timer
363
- if (handler.gcTimer) {
364
- clearInterval(handler.gcTimer);
365
- handler.gcTimer = null;
366
- }
367
-
368
- // Flush pending transactions
369
- if (handler.pendingTransactions && handler.pendingTransactions.size > 0) {
370
- await this._flushPendingTransactions(handler);
371
- }
372
-
373
- this.emit('eventual-consistency.stopped', {
374
- resource: resourceName,
375
- field: fieldName
376
- });
377
- }
378
- }
379
- }
380
-
381
- createPartitionConfig() {
382
- // Create partitions for transactions
383
- const partitions = {
384
- // Composite partition by originalId + applied status
385
- // This is THE MOST CRITICAL optimization for consolidation!
386
- // Why: Consolidation always queries { originalId, applied: false }
387
- // Without this: Reads ALL transactions (applied + pending) and filters manually
388
- // With this: Reads ONLY pending transactions - can be 1000x faster!
389
- byOriginalIdAndApplied: {
390
- fields: {
391
- originalId: 'string',
392
- applied: 'boolean'
393
- }
394
- },
395
- // Partition by time cohorts for batch consolidation across many records
396
- byHour: {
397
- fields: {
398
- cohortHour: 'string'
399
- }
400
- },
401
- byDay: {
402
- fields: {
403
- cohortDate: 'string'
404
- }
405
- },
406
- byMonth: {
407
- fields: {
408
- cohortMonth: 'string'
409
- }
410
- }
411
- };
412
-
413
- return partitions;
414
- }
415
-
416
- /**
417
- * Auto-detect timezone from environment or system
418
- * @private
419
- */
420
- _detectTimezone() {
421
- // 1. Try TZ environment variable (common in Docker/K8s)
422
- if (process.env.TZ) {
423
- return process.env.TZ;
424
- }
425
-
426
- // 2. Try Intl API (works in Node.js and browsers)
427
- try {
428
- const systemTimezone = Intl.DateTimeFormat().resolvedOptions().timeZone;
429
- if (systemTimezone) {
430
- return systemTimezone;
431
- }
432
- } catch (err) {
433
- // Intl API not available or failed
434
- }
435
-
436
- // 3. Fallback to UTC
437
- return 'UTC';
438
- }
439
-
440
- /**
441
- * Helper method to resolve field and plugin from arguments
442
- * @private
443
- */
444
- _resolveFieldAndPlugin(resource, field, value) {
445
- if (!resource._eventualConsistencyPlugins) {
446
- throw new Error(`No eventual consistency plugins configured for this resource`);
447
- }
448
-
449
- const fieldPlugin = resource._eventualConsistencyPlugins[field];
450
-
451
- if (!fieldPlugin) {
452
- const availableFields = Object.keys(resource._eventualConsistencyPlugins).join(', ');
453
- throw new Error(
454
- `No eventual consistency plugin found for field "${field}". ` +
455
- `Available fields: ${availableFields}`
456
- );
457
- }
458
-
459
- return { field, value, plugin: fieldPlugin };
460
- }
461
-
462
- /**
463
- * Helper method to perform atomic consolidation in sync mode
464
- * @private
465
- */
466
- async _syncModeConsolidate(id, field) {
467
- // consolidateRecord already has distributed locking and handles persistence (upsert)
468
- const consolidatedValue = await this.consolidateRecord(id);
469
- return consolidatedValue;
470
- }
471
-
472
- /**
473
- * Create synthetic 'set' transaction from current value
474
- * @private
475
- */
476
- _createSyntheticSetTransaction(currentValue) {
477
- return {
478
- id: '__synthetic__',
479
- operation: 'set',
480
- value: currentValue,
481
- timestamp: new Date(0).toISOString(),
482
- synthetic: true
483
- };
484
- }
485
-
486
- addHelperMethods() {
487
- // Get any handler from the first resource to access the resource instance
488
- const firstResource = this.fieldHandlers.values().next().value;
489
- if (!firstResource) return;
490
-
491
- const firstHandler = firstResource.values().next().value;
492
- if (!firstHandler || !firstHandler.targetResource) return;
493
-
494
- const resource = firstHandler.targetResource;
495
- const plugin = this;
496
-
497
- // Add method to set value (replaces current value)
498
- // Signature: set(id, field, value)
499
- resource.set = async (id, field, value) => {
500
- const { plugin: handler } =
501
- plugin._resolveFieldAndPlugin(resource, field, value);
502
-
503
- // Create transaction inline
504
- const now = new Date();
505
- const cohortInfo = plugin.getCohortInfo(now);
506
-
507
- const transaction = {
508
- id: idGenerator(),
509
- originalId: id,
510
- field: handler.field,
511
- value: value,
512
- operation: 'set',
513
- timestamp: now.toISOString(),
514
- cohortDate: cohortInfo.date,
515
- cohortHour: cohortInfo.hour,
516
- cohortMonth: cohortInfo.month,
517
- source: 'set',
518
- applied: false
519
- };
520
-
521
- await handler.transactionResource.insert(transaction);
522
-
523
- // In sync mode, immediately consolidate
524
- if (plugin.config.mode === 'sync') {
525
- // Temporarily set config for legacy methods
526
- const oldResource = plugin.config.resource;
527
- const oldField = plugin.config.field;
528
- const oldTransactionResource = plugin.transactionResource;
529
- const oldTargetResource = plugin.targetResource;
530
- const oldLockResource = plugin.lockResource;
531
- const oldAnalyticsResource = plugin.analyticsResource;
532
-
533
- plugin.config.resource = handler.resource;
534
- plugin.config.field = handler.field;
535
- plugin.transactionResource = handler.transactionResource;
536
- plugin.targetResource = handler.targetResource;
537
- plugin.lockResource = handler.lockResource;
538
- plugin.analyticsResource = handler.analyticsResource;
539
-
540
- const result = await plugin._syncModeConsolidate(id, field);
541
-
542
- // Restore
543
- plugin.config.resource = oldResource;
544
- plugin.config.field = oldField;
545
- plugin.transactionResource = oldTransactionResource;
546
- plugin.targetResource = oldTargetResource;
547
- plugin.lockResource = oldLockResource;
548
- plugin.analyticsResource = oldAnalyticsResource;
549
-
550
- return result;
551
- }
552
-
553
- return value;
554
- };
555
-
556
- // Add method to increment value
557
- // Signature: add(id, field, amount)
558
- resource.add = async (id, field, amount) => {
559
- const { plugin: handler } =
560
- plugin._resolveFieldAndPlugin(resource, field, amount);
561
-
562
- // Create transaction inline
563
- const now = new Date();
564
- const cohortInfo = plugin.getCohortInfo(now);
565
-
566
- const transaction = {
567
- id: idGenerator(),
568
- originalId: id,
569
- field: handler.field,
570
- value: amount,
571
- operation: 'add',
572
- timestamp: now.toISOString(),
573
- cohortDate: cohortInfo.date,
574
- cohortHour: cohortInfo.hour,
575
- cohortMonth: cohortInfo.month,
576
- source: 'add',
577
- applied: false
578
- };
579
-
580
- await handler.transactionResource.insert(transaction);
581
-
582
- // In sync mode, immediately consolidate
583
- if (plugin.config.mode === 'sync') {
584
- const oldResource = plugin.config.resource;
585
- const oldField = plugin.config.field;
586
- const oldTransactionResource = plugin.transactionResource;
587
- const oldTargetResource = plugin.targetResource;
588
- const oldLockResource = plugin.lockResource;
589
- const oldAnalyticsResource = plugin.analyticsResource;
590
-
591
- plugin.config.resource = handler.resource;
592
- plugin.config.field = handler.field;
593
- plugin.transactionResource = handler.transactionResource;
594
- plugin.targetResource = handler.targetResource;
595
- plugin.lockResource = handler.lockResource;
596
- plugin.analyticsResource = handler.analyticsResource;
597
-
598
- const result = await plugin._syncModeConsolidate(id, field);
599
-
600
- plugin.config.resource = oldResource;
601
- plugin.config.field = oldField;
602
- plugin.transactionResource = oldTransactionResource;
603
- plugin.targetResource = oldTargetResource;
604
- plugin.lockResource = oldLockResource;
605
- plugin.analyticsResource = oldAnalyticsResource;
606
-
607
- return result;
608
- }
609
-
610
- // Async mode - return current value (optimistic)
611
- const [ok, err, record] = await tryFn(() => handler.targetResource.get(id));
612
- const currentValue = (ok && record) ? (record[field] || 0) : 0;
613
- return currentValue + amount;
614
- };
615
-
616
- // Add method to decrement value
617
- // Signature: sub(id, field, amount)
618
- resource.sub = async (id, field, amount) => {
619
- const { plugin: handler } =
620
- plugin._resolveFieldAndPlugin(resource, field, amount);
621
-
622
- // Create transaction inline
623
- const now = new Date();
624
- const cohortInfo = plugin.getCohortInfo(now);
625
-
626
- const transaction = {
627
- id: idGenerator(),
628
- originalId: id,
629
- field: handler.field,
630
- value: amount,
631
- operation: 'sub',
632
- timestamp: now.toISOString(),
633
- cohortDate: cohortInfo.date,
634
- cohortHour: cohortInfo.hour,
635
- cohortMonth: cohortInfo.month,
636
- source: 'sub',
637
- applied: false
638
- };
639
-
640
- await handler.transactionResource.insert(transaction);
641
-
642
- // In sync mode, immediately consolidate
643
- if (plugin.config.mode === 'sync') {
644
- const oldResource = plugin.config.resource;
645
- const oldField = plugin.config.field;
646
- const oldTransactionResource = plugin.transactionResource;
647
- const oldTargetResource = plugin.targetResource;
648
- const oldLockResource = plugin.lockResource;
649
- const oldAnalyticsResource = plugin.analyticsResource;
650
-
651
- plugin.config.resource = handler.resource;
652
- plugin.config.field = handler.field;
653
- plugin.transactionResource = handler.transactionResource;
654
- plugin.targetResource = handler.targetResource;
655
- plugin.lockResource = handler.lockResource;
656
- plugin.analyticsResource = handler.analyticsResource;
657
-
658
- const result = await plugin._syncModeConsolidate(id, field);
659
-
660
- plugin.config.resource = oldResource;
661
- plugin.config.field = oldField;
662
- plugin.transactionResource = oldTransactionResource;
663
- plugin.targetResource = oldTargetResource;
664
- plugin.lockResource = oldLockResource;
665
- plugin.analyticsResource = oldAnalyticsResource;
666
-
667
- return result;
668
- }
669
-
670
- // Async mode - return current value (optimistic)
671
- const [ok, err, record] = await tryFn(() => handler.targetResource.get(id));
672
- const currentValue = (ok && record) ? (record[field] || 0) : 0;
673
- return currentValue - amount;
674
- };
675
-
676
- // Add method to manually trigger consolidation
677
- // Signature: consolidate(id, field)
678
- resource.consolidate = async (id, field) => {
679
- if (!field) {
680
- throw new Error(`Field parameter is required: consolidate(id, field)`);
681
- }
682
-
683
- const handler = resource._eventualConsistencyPlugins[field];
684
-
685
- if (!handler) {
686
- const availableFields = Object.keys(resource._eventualConsistencyPlugins).join(', ');
687
- throw new Error(
688
- `No eventual consistency plugin found for field "${field}". ` +
689
- `Available fields: ${availableFields}`
690
- );
691
- }
692
-
693
- // Temporarily set config for legacy methods
694
- const oldResource = plugin.config.resource;
695
- const oldField = plugin.config.field;
696
- const oldTransactionResource = plugin.transactionResource;
697
- const oldTargetResource = plugin.targetResource;
698
- const oldLockResource = plugin.lockResource;
699
- const oldAnalyticsResource = plugin.analyticsResource;
700
-
701
- plugin.config.resource = handler.resource;
702
- plugin.config.field = handler.field;
703
- plugin.transactionResource = handler.transactionResource;
704
- plugin.targetResource = handler.targetResource;
705
- plugin.lockResource = handler.lockResource;
706
- plugin.analyticsResource = handler.analyticsResource;
707
-
708
- const result = await plugin.consolidateRecord(id);
709
-
710
- plugin.config.resource = oldResource;
711
- plugin.config.field = oldField;
712
- plugin.transactionResource = oldTransactionResource;
713
- plugin.targetResource = oldTargetResource;
714
- plugin.lockResource = oldLockResource;
715
- plugin.analyticsResource = oldAnalyticsResource;
716
-
717
- return result;
718
- };
719
-
720
- // Add method to get consolidated value without applying
721
- // Signature: getConsolidatedValue(id, field, options)
722
- resource.getConsolidatedValue = async (id, field, options = {}) => {
723
- const handler = resource._eventualConsistencyPlugins[field];
724
-
725
- if (!handler) {
726
- const availableFields = Object.keys(resource._eventualConsistencyPlugins).join(', ');
727
- throw new Error(
728
- `No eventual consistency plugin found for field "${field}". ` +
729
- `Available fields: ${availableFields}`
730
- );
731
- }
732
-
733
- // Temporarily set config for legacy methods
734
- const oldResource = plugin.config.resource;
735
- const oldField = plugin.config.field;
736
- const oldTransactionResource = plugin.transactionResource;
737
- const oldTargetResource = plugin.targetResource;
738
-
739
- plugin.config.resource = handler.resource;
740
- plugin.config.field = handler.field;
741
- plugin.transactionResource = handler.transactionResource;
742
- plugin.targetResource = handler.targetResource;
743
-
744
- const result = await plugin.getConsolidatedValue(id, options);
745
-
746
- plugin.config.resource = oldResource;
747
- plugin.config.field = oldField;
748
- plugin.transactionResource = oldTransactionResource;
749
- plugin.targetResource = oldTargetResource;
750
-
751
- return result;
752
- };
753
-
754
- // Add method to recalculate from scratch
755
- // Signature: recalculate(id, field)
756
- resource.recalculate = async (id, field) => {
757
- if (!field) {
758
- throw new Error(`Field parameter is required: recalculate(id, field)`);
759
- }
760
-
761
- const handler = resource._eventualConsistencyPlugins[field];
762
-
763
- if (!handler) {
764
- const availableFields = Object.keys(resource._eventualConsistencyPlugins).join(', ');
765
- throw new Error(
766
- `No eventual consistency plugin found for field "${field}". ` +
767
- `Available fields: ${availableFields}`
768
- );
769
- }
770
-
771
- // Temporarily set config for legacy methods
772
- const oldResource = plugin.config.resource;
773
- const oldField = plugin.config.field;
774
- const oldTransactionResource = plugin.transactionResource;
775
- const oldTargetResource = plugin.targetResource;
776
- const oldLockResource = plugin.lockResource;
777
- const oldAnalyticsResource = plugin.analyticsResource;
778
-
779
- plugin.config.resource = handler.resource;
780
- plugin.config.field = handler.field;
781
- plugin.transactionResource = handler.transactionResource;
782
- plugin.targetResource = handler.targetResource;
783
- plugin.lockResource = handler.lockResource;
784
- plugin.analyticsResource = handler.analyticsResource;
785
-
786
- const result = await plugin.recalculateRecord(id);
787
-
788
- plugin.config.resource = oldResource;
789
- plugin.config.field = oldField;
790
- plugin.transactionResource = oldTransactionResource;
791
- plugin.targetResource = oldTargetResource;
792
- plugin.lockResource = oldLockResource;
793
- plugin.analyticsResource = oldAnalyticsResource;
794
-
795
- return result;
796
- };
797
- }
798
-
799
- async createTransaction(handler, data) {
800
- const now = new Date();
801
- const cohortInfo = this.getCohortInfo(now);
802
-
803
- // Check for late arrivals (transaction older than watermark)
804
- const watermarkMs = this.config.consolidationWindow * 60 * 60 * 1000;
805
- const watermarkTime = now.getTime() - watermarkMs;
806
- const cohortHourDate = new Date(cohortInfo.hour + ':00:00Z');
807
-
808
- if (cohortHourDate.getTime() < watermarkTime) {
809
- // Late arrival detected!
810
- const hoursLate = Math.floor((now.getTime() - cohortHourDate.getTime()) / (60 * 60 * 1000));
811
-
812
- if (this.config.lateArrivalStrategy === 'ignore') {
813
- if (this.config.verbose) {
814
- console.warn(
815
- `[EventualConsistency] Late arrival ignored: transaction for ${cohortInfo.hour} ` +
816
- `is ${hoursLate}h late (watermark: ${this.config.consolidationWindow}h)`
817
- );
818
- }
819
- return null;
820
- } else if (this.config.lateArrivalStrategy === 'warn') {
821
- console.warn(
822
- `[EventualConsistency] Late arrival detected: transaction for ${cohortInfo.hour} ` +
823
- `is ${hoursLate}h late (watermark: ${this.config.consolidationWindow}h). ` +
824
- `Processing anyway, but consolidation may not pick it up.`
825
- );
826
- }
827
- // 'process' strategy: continue normally
828
- }
829
-
830
- const transaction = {
831
- id: idGenerator(),
832
- originalId: data.originalId,
833
- field: handler.field,
834
- value: data.value || 0,
835
- operation: data.operation || 'set',
836
- timestamp: now.toISOString(),
837
- cohortDate: cohortInfo.date,
838
- cohortHour: cohortInfo.hour,
839
- cohortMonth: cohortInfo.month,
840
- source: data.source || 'unknown',
841
- applied: false
842
- };
843
-
844
- // Batch transactions if configured
845
- if (this.config.batchTransactions) {
846
- handler.pendingTransactions.set(transaction.id, transaction);
847
-
848
- if (this.config.verbose) {
849
- console.log(
850
- `[EventualConsistency] ${handler.resource}.${handler.field} - ` +
851
- `Transaction batched: ${data.operation} ${data.value} for ${data.originalId} ` +
852
- `(batch: ${handler.pendingTransactions.size}/${this.config.batchSize})`
853
- );
854
- }
855
-
856
- // Flush if batch size reached
857
- if (handler.pendingTransactions.size >= this.config.batchSize) {
858
- await this._flushPendingTransactions(handler);
859
- }
860
- } else {
861
- await handler.transactionResource.insert(transaction);
862
-
863
- if (this.config.verbose) {
864
- console.log(
865
- `[EventualConsistency] ${handler.resource}.${handler.field} - ` +
866
- `Transaction created: ${data.operation} ${data.value} for ${data.originalId} ` +
867
- `(cohort: ${cohortInfo.hour}, applied: false)`
868
- );
869
- }
870
- }
871
-
872
- return transaction;
873
- }
874
-
875
- async flushPendingTransactions() {
876
- if (this.pendingTransactions.size === 0) return;
877
-
878
- const transactions = Array.from(this.pendingTransactions.values());
879
-
880
- try {
881
- // Insert all pending transactions in parallel
882
- await Promise.all(
883
- transactions.map(transaction =>
884
- this.transactionResource.insert(transaction)
885
- )
886
- );
887
-
888
- // Only clear after successful inserts (prevents data loss on crashes)
889
- this.pendingTransactions.clear();
890
- } catch (error) {
891
- // Keep pending transactions for retry on next flush
892
- console.error('Failed to flush pending transactions:', error);
893
- throw error;
894
- }
895
- }
896
-
897
- getCohortInfo(date) {
898
- const tz = this.config.cohort.timezone;
899
-
900
- // Simple timezone offset calculation (can be enhanced with a library)
901
- const offset = this.getTimezoneOffset(tz);
902
- const localDate = new Date(date.getTime() + offset);
903
-
904
- const year = localDate.getFullYear();
905
- const month = String(localDate.getMonth() + 1).padStart(2, '0');
906
- const day = String(localDate.getDate()).padStart(2, '0');
907
- const hour = String(localDate.getHours()).padStart(2, '0');
908
-
909
- return {
910
- date: `${year}-${month}-${day}`,
911
- hour: `${year}-${month}-${day}T${hour}`, // ISO-like format for hour partition
912
- month: `${year}-${month}`
913
- };
914
- }
915
-
916
- getTimezoneOffset(timezone) {
917
- // Try to calculate offset using Intl API (handles DST automatically)
918
- try {
919
- const now = new Date();
920
-
921
- // Get UTC time
922
- const utcDate = new Date(now.toLocaleString('en-US', { timeZone: 'UTC' }));
923
-
924
- // Get time in target timezone
925
- const tzDate = new Date(now.toLocaleString('en-US', { timeZone: timezone }));
926
-
927
- // Calculate offset in milliseconds
928
- return tzDate.getTime() - utcDate.getTime();
929
- } catch (err) {
930
- // Intl API failed, fallback to manual offsets (without DST support)
931
- const offsets = {
932
- 'UTC': 0,
933
- 'America/New_York': -5 * 3600000,
934
- 'America/Chicago': -6 * 3600000,
935
- 'America/Denver': -7 * 3600000,
936
- 'America/Los_Angeles': -8 * 3600000,
937
- 'America/Sao_Paulo': -3 * 3600000,
938
- 'Europe/London': 0,
939
- 'Europe/Paris': 1 * 3600000,
940
- 'Europe/Berlin': 1 * 3600000,
941
- 'Asia/Tokyo': 9 * 3600000,
942
- 'Asia/Shanghai': 8 * 3600000,
943
- 'Australia/Sydney': 10 * 3600000
944
- };
945
-
946
- if (this.config.verbose && !offsets[timezone]) {
947
- console.warn(
948
- `[EventualConsistency] Unknown timezone '${timezone}', using UTC. ` +
949
- `Consider using a valid IANA timezone (e.g., 'America/New_York')`
950
- );
951
- }
952
-
953
- return offsets[timezone] || 0;
954
- }
955
- }
956
-
957
- startConsolidationTimer() {
958
- const intervalMs = this.config.consolidationInterval * 1000; // Convert seconds to ms
959
-
960
- if (this.config.verbose) {
961
- const nextRun = new Date(Date.now() + intervalMs);
962
- console.log(
963
- `[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
964
- `Consolidation timer started. Next run at ${nextRun.toISOString()} ` +
965
- `(every ${this.config.consolidationInterval}s)`
966
- );
967
- }
968
-
969
- this.consolidationTimer = setInterval(async () => {
970
- await this.runConsolidation();
971
- }, intervalMs);
972
- }
973
-
974
- startConsolidationTimerForHandler(handler, resourceName, fieldName) {
975
- const intervalMs = this.config.consolidationInterval * 1000; // Convert seconds to ms
976
-
977
- if (this.config.verbose) {
978
- const nextRun = new Date(Date.now() + intervalMs);
979
- console.log(
980
- `[EventualConsistency] ${resourceName}.${fieldName} - ` +
981
- `Consolidation timer started. Next run at ${nextRun.toISOString()} ` +
982
- `(every ${this.config.consolidationInterval}s)`
983
- );
984
- }
985
-
986
- handler.consolidationTimer = setInterval(async () => {
987
- await this.runConsolidationForHandler(handler, resourceName, fieldName);
988
- }, intervalMs);
989
- }
990
-
991
- async runConsolidationForHandler(handler, resourceName, fieldName) {
992
- // Temporarily swap config to use this handler
993
- const oldResource = this.config.resource;
994
- const oldField = this.config.field;
995
- const oldTransactionResource = this.transactionResource;
996
- const oldTargetResource = this.targetResource;
997
- const oldLockResource = this.lockResource;
998
- const oldAnalyticsResource = this.analyticsResource;
999
-
1000
- this.config.resource = resourceName;
1001
- this.config.field = fieldName;
1002
- this.transactionResource = handler.transactionResource;
1003
- this.targetResource = handler.targetResource;
1004
- this.lockResource = handler.lockResource;
1005
- this.analyticsResource = handler.analyticsResource;
1006
-
1007
- try {
1008
- await this.runConsolidation();
1009
- } finally {
1010
- // Restore
1011
- this.config.resource = oldResource;
1012
- this.config.field = oldField;
1013
- this.transactionResource = oldTransactionResource;
1014
- this.targetResource = oldTargetResource;
1015
- this.lockResource = oldLockResource;
1016
- this.analyticsResource = oldAnalyticsResource;
1017
- }
1018
- }
1019
-
1020
- async runConsolidation() {
1021
- const startTime = Date.now();
1022
-
1023
- if (this.config.verbose) {
1024
- console.log(
1025
- `[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
1026
- `Starting consolidation run at ${new Date().toISOString()}`
1027
- );
1028
- }
1029
-
1030
- try {
1031
- // Query unapplied transactions from recent cohorts (last 24 hours by default)
1032
- // This uses hourly partition for O(1) performance instead of full scan
1033
- const now = new Date();
1034
- const hoursToCheck = this.config.consolidationWindow || 24; // Configurable lookback window (in hours)
1035
- const cohortHours = [];
1036
-
1037
- for (let i = 0; i < hoursToCheck; i++) {
1038
- const date = new Date(now.getTime() - (i * 60 * 60 * 1000)); // Subtract hours
1039
- const cohortInfo = this.getCohortInfo(date);
1040
- cohortHours.push(cohortInfo.hour);
1041
- }
1042
-
1043
- if (this.config.verbose) {
1044
- console.log(
1045
- `[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
1046
- `Querying ${hoursToCheck} hour partitions for pending transactions...`
1047
- );
1048
- }
1049
-
1050
- // Query transactions by partition for each hour (parallel for speed)
1051
- const transactionsByHour = await Promise.all(
1052
- cohortHours.map(async (cohortHour) => {
1053
- const [ok, err, txns] = await tryFn(() =>
1054
- this.transactionResource.query({
1055
- cohortHour,
1056
- applied: false
1057
- })
1058
- );
1059
- return ok ? txns : [];
1060
- })
1061
- );
1062
-
1063
- // Flatten all transactions
1064
- const transactions = transactionsByHour.flat();
1065
-
1066
- if (transactions.length === 0) {
1067
- if (this.config.verbose) {
1068
- console.log(
1069
- `[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
1070
- `No pending transactions found. Next run in ${this.config.consolidationInterval}s`
1071
- );
1072
- }
1073
- return;
1074
- }
1075
-
1076
- // Get unique originalIds
1077
- const uniqueIds = [...new Set(transactions.map(t => t.originalId))];
1078
-
1079
- if (this.config.verbose) {
1080
- console.log(
1081
- `[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
1082
- `Found ${transactions.length} pending transactions for ${uniqueIds.length} records. ` +
1083
- `Consolidating with concurrency=${this.config.consolidationConcurrency}...`
1084
- );
1085
- }
1086
-
1087
- // Consolidate each record in parallel with concurrency limit
1088
- const { results, errors } = await PromisePool
1089
- .for(uniqueIds)
1090
- .withConcurrency(this.config.consolidationConcurrency)
1091
- .process(async (id) => {
1092
- return await this.consolidateRecord(id);
1093
- });
1094
-
1095
- const duration = Date.now() - startTime;
1096
-
1097
- if (errors && errors.length > 0) {
1098
- console.error(
1099
- `[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
1100
- `Consolidation completed with ${errors.length} errors in ${duration}ms:`,
1101
- errors
1102
- );
1103
- }
1104
-
1105
- if (this.config.verbose) {
1106
- console.log(
1107
- `[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
1108
- `Consolidation complete: ${results.length} records consolidated in ${duration}ms ` +
1109
- `(${errors.length} errors). Next run in ${this.config.consolidationInterval}s`
1110
- );
1111
- }
1112
-
1113
- this.emit('eventual-consistency.consolidated', {
1114
- resource: this.config.resource,
1115
- field: this.config.field,
1116
- recordCount: uniqueIds.length,
1117
- successCount: results.length,
1118
- errorCount: errors.length,
1119
- duration
1120
- });
1121
- } catch (error) {
1122
- const duration = Date.now() - startTime;
1123
- console.error(
1124
- `[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
1125
- `Consolidation error after ${duration}ms:`,
1126
- error
1127
- );
1128
- this.emit('eventual-consistency.consolidation-error', error);
1129
- }
1130
- }
1131
-
1132
- async consolidateRecord(originalId) {
1133
- // Clean up stale locks before attempting to acquire
1134
- await this.cleanupStaleLocks();
1135
-
1136
- // Acquire distributed lock to prevent concurrent consolidation
1137
- const lockId = `lock-${originalId}`;
1138
- const [lockAcquired, lockErr, lock] = await tryFn(() =>
1139
- this.lockResource.insert({
1140
- id: lockId,
1141
- lockedAt: Date.now(),
1142
- workerId: process.pid ? String(process.pid) : 'unknown'
1143
- })
1144
- );
1145
-
1146
- // If lock couldn't be acquired, another worker is consolidating
1147
- if (!lockAcquired) {
1148
- if (this.config.verbose) {
1149
- console.log(`[EventualConsistency] Lock for ${originalId} already held, skipping`);
1150
- }
1151
- // Get current value and return (another worker will consolidate)
1152
- const [recordOk, recordErr, record] = await tryFn(() =>
1153
- this.targetResource.get(originalId)
1154
- );
1155
- return (recordOk && record) ? (record[this.config.field] || 0) : 0;
1156
- }
1157
-
1158
- try {
1159
- // Get all unapplied transactions for this record
1160
- const [ok, err, transactions] = await tryFn(() =>
1161
- this.transactionResource.query({
1162
- originalId,
1163
- applied: false
1164
- })
1165
- );
1166
-
1167
- if (!ok || !transactions || transactions.length === 0) {
1168
- // No pending transactions - try to get current value from record
1169
- const [recordOk, recordErr, record] = await tryFn(() =>
1170
- this.targetResource.get(originalId)
1171
- );
1172
- const currentValue = (recordOk && record) ? (record[this.config.field] || 0) : 0;
1173
-
1174
- if (this.config.verbose) {
1175
- console.log(
1176
- `[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
1177
- `No pending transactions for ${originalId}, skipping`
1178
- );
1179
- }
1180
- return currentValue;
1181
- }
1182
-
1183
- // Get the LAST APPLIED VALUE from transactions (not from record - avoids S3 eventual consistency issues)
1184
- // This is the source of truth for the current value
1185
- const [appliedOk, appliedErr, appliedTransactions] = await tryFn(() =>
1186
- this.transactionResource.query({
1187
- originalId,
1188
- applied: true
1189
- })
1190
- );
1191
-
1192
- let currentValue = 0;
1193
-
1194
- if (appliedOk && appliedTransactions && appliedTransactions.length > 0) {
1195
- // Check if record exists - if deleted, ignore old applied transactions
1196
- const [recordExistsOk, recordExistsErr, recordExists] = await tryFn(() =>
1197
- this.targetResource.get(originalId)
1198
- );
1199
-
1200
- if (!recordExistsOk || !recordExists) {
1201
- // Record was deleted - ignore applied transactions and start fresh
1202
- // This prevents old values from being carried over after deletion
1203
- if (this.config.verbose) {
1204
- console.log(
1205
- `[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
1206
- `Record ${originalId} doesn't exist, deleting ${appliedTransactions.length} old applied transactions`
1207
- );
1208
- }
1209
-
1210
- // Delete old applied transactions to prevent them from being used when record is recreated
1211
- const { results, errors } = await PromisePool
1212
- .for(appliedTransactions)
1213
- .withConcurrency(10)
1214
- .process(async (txn) => {
1215
- const [deleted] = await tryFn(() => this.transactionResource.delete(txn.id));
1216
- return deleted;
1217
- });
1218
-
1219
- if (this.config.verbose && errors && errors.length > 0) {
1220
- console.warn(
1221
- `[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
1222
- `Failed to delete ${errors.length} old applied transactions`
1223
- );
1224
- }
1225
-
1226
- currentValue = 0;
1227
- } else {
1228
- // Record exists - use applied transactions to calculate current value
1229
- // Sort by timestamp to get chronological order
1230
- appliedTransactions.sort((a, b) =>
1231
- new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()
1232
- );
1233
-
1234
- // Check if there's a 'set' operation in applied transactions
1235
- const hasSetInApplied = appliedTransactions.some(t => t.operation === 'set');
1236
-
1237
- if (!hasSetInApplied) {
1238
- // No 'set' operation in applied transactions means we're missing the base value
1239
- // This can only happen if:
1240
- // 1. Record had an initial value before first transaction
1241
- // 2. First consolidation didn't create an anchor transaction (legacy behavior)
1242
- // Solution: Get the current record value and create an anchor transaction now
1243
- const recordValue = recordExists[this.config.field] || 0;
1244
-
1245
- // Calculate what the base value was by subtracting all applied deltas
1246
- let appliedDelta = 0;
1247
- for (const t of appliedTransactions) {
1248
- if (t.operation === 'add') appliedDelta += t.value;
1249
- else if (t.operation === 'sub') appliedDelta -= t.value;
1250
- }
1251
-
1252
- const baseValue = recordValue - appliedDelta;
1253
-
1254
- // Create and save anchor transaction with the base value
1255
- // Only create if baseValue is non-zero AND we don't already have an anchor transaction
1256
- const hasExistingAnchor = appliedTransactions.some(t => t.source === 'anchor');
1257
- if (baseValue !== 0 && !hasExistingAnchor) {
1258
- // Use the timestamp of the first applied transaction for cohort info
1259
- const firstTransactionDate = new Date(appliedTransactions[0].timestamp);
1260
- const cohortInfo = this.getCohortInfo(firstTransactionDate);
1261
- const anchorTransaction = {
1262
- id: idGenerator(),
1263
- originalId: originalId,
1264
- field: this.config.field,
1265
- value: baseValue,
1266
- operation: 'set',
1267
- timestamp: new Date(firstTransactionDate.getTime() - 1).toISOString(), // 1ms before first txn to ensure it's first
1268
- cohortDate: cohortInfo.date,
1269
- cohortHour: cohortInfo.hour,
1270
- cohortMonth: cohortInfo.month,
1271
- source: 'anchor',
1272
- applied: true
1273
- };
1274
-
1275
- await this.transactionResource.insert(anchorTransaction);
1276
-
1277
- // Prepend to applied transactions for this consolidation
1278
- appliedTransactions.unshift(anchorTransaction);
1279
- }
1280
- }
1281
-
1282
- // Apply reducer to get the last consolidated value
1283
- currentValue = this.config.reducer(appliedTransactions);
1284
- }
1285
- } else {
1286
- // No applied transactions - this is the FIRST consolidation
1287
- // Try to get initial value from record
1288
- const [recordOk, recordErr, record] = await tryFn(() =>
1289
- this.targetResource.get(originalId)
1290
- );
1291
- currentValue = (recordOk && record) ? (record[this.config.field] || 0) : 0;
1292
-
1293
- // If there's an initial value, create and save an anchor transaction
1294
- // This ensures all future consolidations have a reliable base value
1295
- if (currentValue !== 0) {
1296
- // Use timestamp of the first pending transaction (or current time if none)
1297
- let anchorTimestamp;
1298
- if (transactions && transactions.length > 0) {
1299
- const firstPendingDate = new Date(transactions[0].timestamp);
1300
- anchorTimestamp = new Date(firstPendingDate.getTime() - 1).toISOString();
1301
- } else {
1302
- anchorTimestamp = new Date().toISOString();
1303
- }
1304
-
1305
- const cohortInfo = this.getCohortInfo(new Date(anchorTimestamp));
1306
- const anchorTransaction = {
1307
- id: idGenerator(),
1308
- originalId: originalId,
1309
- field: this.config.field,
1310
- value: currentValue,
1311
- operation: 'set',
1312
- timestamp: anchorTimestamp,
1313
- cohortDate: cohortInfo.date,
1314
- cohortHour: cohortInfo.hour,
1315
- cohortMonth: cohortInfo.month,
1316
- source: 'anchor',
1317
- applied: true
1318
- };
1319
-
1320
- await this.transactionResource.insert(anchorTransaction);
1321
-
1322
- if (this.config.verbose) {
1323
- console.log(
1324
- `[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
1325
- `Created anchor transaction for ${originalId} with base value ${currentValue}`
1326
- );
1327
- }
1328
- }
1329
- }
1330
-
1331
- if (this.config.verbose) {
1332
- console.log(
1333
- `[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
1334
- `Consolidating ${originalId}: ${transactions.length} pending transactions ` +
1335
- `(current: ${currentValue} from ${appliedOk && appliedTransactions?.length > 0 ? 'applied transactions' : 'record'})`
1336
- );
1337
- }
1338
-
1339
- // Sort pending transactions by timestamp
1340
- transactions.sort((a, b) =>
1341
- new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()
1342
- );
1343
-
1344
- // If there's a current value and no 'set' operations in pending transactions,
1345
- // prepend a synthetic set transaction to preserve the current value
1346
- const hasSetOperation = transactions.some(t => t.operation === 'set');
1347
- if (currentValue !== 0 && !hasSetOperation) {
1348
- transactions.unshift(this._createSyntheticSetTransaction(currentValue));
1349
- }
1350
-
1351
- // Apply reducer to get consolidated value
1352
- const consolidatedValue = this.config.reducer(transactions);
1353
-
1354
- if (this.config.verbose) {
1355
- console.log(
1356
- `[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
1357
- `${originalId}: ${currentValue} → ${consolidatedValue} ` +
1358
- `(${consolidatedValue > currentValue ? '+' : ''}${consolidatedValue - currentValue})`
1359
- );
1360
- }
1361
-
1362
- // Update the original record
1363
- // NOTE: We do NOT attempt to insert non-existent records because:
1364
- // 1. Target resources typically have required fields we don't know about
1365
- // 2. Record creation should be the application's responsibility
1366
- // 3. Transactions will remain pending until the record is created
1367
- const [updateOk, updateErr] = await tryFn(() =>
1368
- this.targetResource.update(originalId, {
1369
- [this.config.field]: consolidatedValue
1370
- })
1371
- );
1372
-
1373
- if (!updateOk) {
1374
- // Check if record doesn't exist
1375
- if (updateErr?.message?.includes('does not exist')) {
1376
- // Record doesn't exist - skip consolidation and keep transactions pending
1377
- if (this.config.verbose) {
1378
- console.warn(
1379
- `[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
1380
- `Record ${originalId} doesn't exist. Skipping consolidation. ` +
1381
- `${transactions.length} transactions will remain pending until record is created.`
1382
- );
1383
- }
1384
-
1385
- // Return the consolidated value (for informational purposes)
1386
- // Transactions remain pending and will be picked up when record exists
1387
- return consolidatedValue;
1388
- }
1389
-
1390
- // Update failed for another reason - this is a real error
1391
- console.error(
1392
- `[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
1393
- `FAILED to update ${originalId}: ${updateErr?.message || updateErr}`,
1394
- { error: updateErr, consolidatedValue, currentValue }
1395
- );
1396
- throw updateErr;
1397
- }
1398
-
1399
- if (updateOk) {
1400
- // Mark transactions as applied (skip synthetic ones) - use PromisePool for controlled concurrency
1401
- const transactionsToUpdate = transactions.filter(txn => txn.id !== '__synthetic__');
1402
-
1403
- const { results, errors } = await PromisePool
1404
- .for(transactionsToUpdate)
1405
- .withConcurrency(10) // Limit parallel updates
1406
- .process(async (txn) => {
1407
- const [ok, err] = await tryFn(() =>
1408
- this.transactionResource.update(txn.id, { applied: true })
1409
- );
1410
-
1411
- if (!ok && this.config.verbose) {
1412
- console.warn(`[EventualConsistency] Failed to mark transaction ${txn.id} as applied:`, err?.message);
1413
- }
1414
-
1415
- return ok;
1416
- });
1417
-
1418
- if (errors && errors.length > 0 && this.config.verbose) {
1419
- console.warn(`[EventualConsistency] ${errors.length} transactions failed to mark as applied`);
1420
- }
1421
-
1422
- // Update analytics if enabled (only for real transactions, not synthetic)
1423
- if (this.config.enableAnalytics && transactionsToUpdate.length > 0) {
1424
- await this.updateAnalytics(transactionsToUpdate);
1425
- }
1426
-
1427
- // Invalidate cache for this record after consolidation
1428
- if (this.targetResource && this.targetResource.cache && typeof this.targetResource.cache.delete === 'function') {
1429
- try {
1430
- const cacheKey = await this.targetResource.cacheKeyFor({ id: originalId });
1431
- await this.targetResource.cache.delete(cacheKey);
1432
-
1433
- if (this.config.verbose) {
1434
- console.log(
1435
- `[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
1436
- `Cache invalidated for ${originalId}`
1437
- );
1438
- }
1439
- } catch (cacheErr) {
1440
- // Log but don't fail consolidation if cache invalidation fails
1441
- if (this.config.verbose) {
1442
- console.warn(
1443
- `[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
1444
- `Failed to invalidate cache for ${originalId}: ${cacheErr?.message}`
1445
- );
1446
- }
1447
- }
1448
- }
1449
- }
1450
-
1451
- return consolidatedValue;
1452
- } finally {
1453
- // Always release the lock
1454
- const [lockReleased, lockReleaseErr] = await tryFn(() => this.lockResource.delete(lockId));
1455
-
1456
- if (!lockReleased && this.config.verbose) {
1457
- console.warn(`[EventualConsistency] Failed to release lock ${lockId}:`, lockReleaseErr?.message);
1458
- }
1459
- }
1460
- }
1461
-
1462
- async getConsolidatedValue(originalId, options = {}) {
1463
- const includeApplied = options.includeApplied || false;
1464
- const startDate = options.startDate;
1465
- const endDate = options.endDate;
1466
-
1467
- // Build query
1468
- const query = { originalId };
1469
- if (!includeApplied) {
1470
- query.applied = false;
1471
- }
1472
-
1473
- // Get transactions
1474
- const [ok, err, transactions] = await tryFn(() =>
1475
- this.transactionResource.query(query)
1476
- );
1477
-
1478
- if (!ok || !transactions || transactions.length === 0) {
1479
- // If no transactions, check if record exists and return its current value
1480
- const [recordOk, recordErr, record] = await tryFn(() =>
1481
- this.targetResource.get(originalId)
1482
- );
1483
-
1484
- if (recordOk && record) {
1485
- return record[this.config.field] || 0;
1486
- }
1487
-
1488
- return 0;
1489
- }
1490
-
1491
- // Filter by date range if specified
1492
- let filtered = transactions;
1493
- if (startDate || endDate) {
1494
- filtered = transactions.filter(t => {
1495
- const timestamp = new Date(t.timestamp);
1496
- if (startDate && timestamp < new Date(startDate)) return false;
1497
- if (endDate && timestamp > new Date(endDate)) return false;
1498
- return true;
1499
- });
1500
- }
1501
-
1502
- // Get current value from record
1503
- const [recordOk, recordErr, record] = await tryFn(() =>
1504
- this.targetResource.get(originalId)
1505
- );
1506
- const currentValue = (recordOk && record) ? (record[this.config.field] || 0) : 0;
1507
-
1508
- // Check if there's a 'set' operation in filtered transactions
1509
- const hasSetOperation = filtered.some(t => t.operation === 'set');
1510
-
1511
- // If current value exists and no 'set', prepend synthetic set transaction
1512
- if (currentValue !== 0 && !hasSetOperation) {
1513
- filtered.unshift(this._createSyntheticSetTransaction(currentValue));
1514
- }
1515
-
1516
- // Sort by timestamp
1517
- filtered.sort((a, b) =>
1518
- new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()
1519
- );
1520
-
1521
- // Apply reducer
1522
- return this.config.reducer(filtered);
1523
- }
1524
-
1525
- // Helper method to get cohort statistics
1526
- async getCohortStats(cohortDate) {
1527
- const [ok, err, transactions] = await tryFn(() =>
1528
- this.transactionResource.query({
1529
- cohortDate
1530
- })
1531
- );
1532
-
1533
- if (!ok) return null;
1534
-
1535
- const stats = {
1536
- date: cohortDate,
1537
- transactionCount: transactions.length,
1538
- totalValue: 0,
1539
- byOperation: { set: 0, add: 0, sub: 0 },
1540
- byOriginalId: {}
1541
- };
1542
-
1543
- for (const txn of transactions) {
1544
- stats.totalValue += txn.value || 0;
1545
- stats.byOperation[txn.operation] = (stats.byOperation[txn.operation] || 0) + 1;
1546
-
1547
- if (!stats.byOriginalId[txn.originalId]) {
1548
- stats.byOriginalId[txn.originalId] = {
1549
- count: 0,
1550
- value: 0
1551
- };
1552
- }
1553
- stats.byOriginalId[txn.originalId].count++;
1554
- stats.byOriginalId[txn.originalId].value += txn.value || 0;
1555
- }
1556
-
1557
- return stats;
1558
- }
1559
-
1560
- /**
1561
- * Recalculate from scratch by resetting all transactions to pending
1562
- * This is useful for debugging, recovery, or when you want to recompute everything
1563
- * @param {string} originalId - The ID of the record to recalculate
1564
- * @returns {Promise<number>} The recalculated value
1565
- */
1566
- async recalculateRecord(originalId) {
1567
- // Clean up stale locks before attempting to acquire
1568
- await this.cleanupStaleLocks();
1569
-
1570
- // Acquire distributed lock to prevent concurrent operations
1571
- const lockId = `lock-recalculate-${originalId}`;
1572
- const [lockAcquired, lockErr, lock] = await tryFn(() =>
1573
- this.lockResource.insert({
1574
- id: lockId,
1575
- lockedAt: Date.now(),
1576
- workerId: process.pid ? String(process.pid) : 'unknown'
1577
- })
1578
- );
1579
-
1580
- // If lock couldn't be acquired, another worker is operating on this record
1581
- if (!lockAcquired) {
1582
- if (this.config.verbose) {
1583
- console.log(`[EventualConsistency] Recalculate lock for ${originalId} already held, skipping`);
1584
- }
1585
- throw new Error(`Cannot recalculate ${originalId}: lock already held by another worker`);
1586
- }
1587
-
1588
- try {
1589
- if (this.config.verbose) {
1590
- console.log(
1591
- `[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
1592
- `Starting recalculation for ${originalId} (resetting all transactions to pending)`
1593
- );
1594
- }
1595
-
1596
- // Get ALL transactions for this record (both applied and pending)
1597
- const [allOk, allErr, allTransactions] = await tryFn(() =>
1598
- this.transactionResource.query({
1599
- originalId
1600
- })
1601
- );
1602
-
1603
- if (!allOk || !allTransactions || allTransactions.length === 0) {
1604
- if (this.config.verbose) {
1605
- console.log(
1606
- `[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
1607
- `No transactions found for ${originalId}, nothing to recalculate`
1608
- );
1609
- }
1610
- return 0;
1611
- }
1612
-
1613
- if (this.config.verbose) {
1614
- console.log(
1615
- `[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
1616
- `Found ${allTransactions.length} total transactions for ${originalId}, marking all as pending...`
1617
- );
1618
- }
1619
-
1620
- // Mark ALL transactions as pending (applied: false)
1621
- // Exclude anchor transactions (they should always be applied)
1622
- const transactionsToReset = allTransactions.filter(txn => txn.source !== 'anchor');
1623
-
1624
- const { results, errors } = await PromisePool
1625
- .for(transactionsToReset)
1626
- .withConcurrency(10)
1627
- .process(async (txn) => {
1628
- const [ok, err] = await tryFn(() =>
1629
- this.transactionResource.update(txn.id, { applied: false })
1630
- );
1631
-
1632
- if (!ok && this.config.verbose) {
1633
- console.warn(`[EventualConsistency] Failed to reset transaction ${txn.id}:`, err?.message);
1634
- }
1635
-
1636
- return ok;
1637
- });
1638
-
1639
- if (errors && errors.length > 0) {
1640
- console.warn(
1641
- `[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
1642
- `Failed to reset ${errors.length} transactions during recalculation`
1643
- );
1644
- }
1645
-
1646
- if (this.config.verbose) {
1647
- console.log(
1648
- `[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
1649
- `Reset ${results.length} transactions to pending, now resetting record value and running consolidation...`
1650
- );
1651
- }
1652
-
1653
- // Reset the record's field value to 0 to prevent double-counting
1654
- // This ensures consolidation starts fresh without using the old value as an anchor
1655
- const [resetOk, resetErr] = await tryFn(() =>
1656
- this.targetResource.update(originalId, {
1657
- [this.config.field]: 0
1658
- })
1659
- );
1660
-
1661
- if (!resetOk && this.config.verbose) {
1662
- console.warn(
1663
- `[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
1664
- `Failed to reset record value for ${originalId}: ${resetErr?.message}`
1665
- );
1666
- }
1667
-
1668
- // Now run normal consolidation which will process all pending transactions
1669
- const consolidatedValue = await this.consolidateRecord(originalId);
1670
-
1671
- if (this.config.verbose) {
1672
- console.log(
1673
- `[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
1674
- `Recalculation complete for ${originalId}: final value = ${consolidatedValue}`
1675
- );
1676
- }
1677
-
1678
- return consolidatedValue;
1679
- } finally {
1680
- // Always release the lock
1681
- const [lockReleased, lockReleaseErr] = await tryFn(() => this.lockResource.delete(lockId));
1682
-
1683
- if (!lockReleased && this.config.verbose) {
1684
- console.warn(`[EventualConsistency] Failed to release recalculate lock ${lockId}:`, lockReleaseErr?.message);
1685
- }
1686
- }
1687
- }
1688
-
1689
- /**
1690
- * Clean up stale locks that exceed the configured timeout
1691
- * Uses distributed locking to prevent multiple containers from cleaning simultaneously
1692
- */
1693
- async cleanupStaleLocks() {
1694
- const now = Date.now();
1695
- const lockTimeoutMs = this.config.lockTimeout * 1000; // Convert seconds to ms
1696
- const cutoffTime = now - lockTimeoutMs;
1697
-
1698
- // Acquire distributed lock for cleanup operation
1699
- const cleanupLockId = `lock-cleanup-${this.config.resource}-${this.config.field}`;
1700
- const [lockAcquired] = await tryFn(() =>
1701
- this.lockResource.insert({
1702
- id: cleanupLockId,
1703
- lockedAt: Date.now(),
1704
- workerId: process.pid ? String(process.pid) : 'unknown'
1705
- })
1706
- );
1707
-
1708
- // If another container is already cleaning, skip
1709
- if (!lockAcquired) {
1710
- if (this.config.verbose) {
1711
- console.log(`[EventualConsistency] Lock cleanup already running in another container`);
1712
- }
1713
- return;
1714
- }
1715
-
1716
- try {
1717
- // Get all locks
1718
- const [ok, err, locks] = await tryFn(() => this.lockResource.list());
1719
-
1720
- if (!ok || !locks || locks.length === 0) return;
1721
-
1722
- // Find stale locks (excluding the cleanup lock itself)
1723
- const staleLocks = locks.filter(lock =>
1724
- lock.id !== cleanupLockId && lock.lockedAt < cutoffTime
1725
- );
1726
-
1727
- if (staleLocks.length === 0) return;
1728
-
1729
- if (this.config.verbose) {
1730
- console.log(`[EventualConsistency] Cleaning up ${staleLocks.length} stale locks`);
1731
- }
1732
-
1733
- // Delete stale locks using PromisePool
1734
- const { results, errors } = await PromisePool
1735
- .for(staleLocks)
1736
- .withConcurrency(5)
1737
- .process(async (lock) => {
1738
- const [deleted] = await tryFn(() => this.lockResource.delete(lock.id));
1739
- return deleted;
1740
- });
1741
-
1742
- if (errors && errors.length > 0 && this.config.verbose) {
1743
- console.warn(`[EventualConsistency] ${errors.length} stale locks failed to delete`);
1744
- }
1745
- } catch (error) {
1746
- if (this.config.verbose) {
1747
- console.warn(`[EventualConsistency] Error cleaning up stale locks:`, error.message);
1748
- }
1749
- } finally {
1750
- // Always release cleanup lock
1751
- await tryFn(() => this.lockResource.delete(cleanupLockId));
1752
- }
1753
- }
1754
-
1755
- /**
1756
- * Start garbage collection timer for old applied transactions
1757
- */
1758
- startGarbageCollectionTimer() {
1759
- const gcIntervalMs = this.config.gcInterval * 1000; // Convert seconds to ms
1760
-
1761
- this.gcTimer = setInterval(async () => {
1762
- await this.runGarbageCollection();
1763
- }, gcIntervalMs);
1764
- }
1765
-
1766
- startGarbageCollectionTimerForHandler(handler, resourceName, fieldName) {
1767
- const gcIntervalMs = this.config.gcInterval * 1000; // Convert seconds to ms
1768
-
1769
- handler.gcTimer = setInterval(async () => {
1770
- await this.runGarbageCollectionForHandler(handler, resourceName, fieldName);
1771
- }, gcIntervalMs);
1772
- }
1773
-
1774
- async runGarbageCollectionForHandler(handler, resourceName, fieldName) {
1775
- // Temporarily swap config to use this handler
1776
- const oldResource = this.config.resource;
1777
- const oldField = this.config.field;
1778
- const oldTransactionResource = this.transactionResource;
1779
- const oldTargetResource = this.targetResource;
1780
- const oldLockResource = this.lockResource;
1781
-
1782
- this.config.resource = resourceName;
1783
- this.config.field = fieldName;
1784
- this.transactionResource = handler.transactionResource;
1785
- this.targetResource = handler.targetResource;
1786
- this.lockResource = handler.lockResource;
1787
-
1788
- try {
1789
- await this.runGarbageCollection();
1790
- } finally {
1791
- // Restore
1792
- this.config.resource = oldResource;
1793
- this.config.field = oldField;
1794
- this.transactionResource = oldTransactionResource;
1795
- this.targetResource = oldTargetResource;
1796
- this.lockResource = oldLockResource;
1797
- }
1798
- }
1799
-
1800
- /**
1801
- * Delete old applied transactions based on retention policy
1802
- * Uses distributed locking to prevent multiple containers from running GC simultaneously
1803
- */
1804
- async runGarbageCollection() {
1805
- // Acquire distributed lock for GC operation
1806
- const gcLockId = `lock-gc-${this.config.resource}-${this.config.field}`;
1807
- const [lockAcquired] = await tryFn(() =>
1808
- this.lockResource.insert({
1809
- id: gcLockId,
1810
- lockedAt: Date.now(),
1811
- workerId: process.pid ? String(process.pid) : 'unknown'
1812
- })
1813
- );
1814
-
1815
- // If another container is already running GC, skip
1816
- if (!lockAcquired) {
1817
- if (this.config.verbose) {
1818
- console.log(`[EventualConsistency] GC already running in another container`);
1819
- }
1820
- return;
1821
- }
1822
-
1823
- try {
1824
- const now = Date.now();
1825
- const retentionMs = this.config.transactionRetention * 24 * 60 * 60 * 1000; // Days to ms
1826
- const cutoffDate = new Date(now - retentionMs);
1827
- const cutoffIso = cutoffDate.toISOString();
1828
-
1829
- if (this.config.verbose) {
1830
- console.log(`[EventualConsistency] Running GC for transactions older than ${cutoffIso} (${this.config.transactionRetention} days)`);
1831
- }
1832
-
1833
- // Query old applied transactions
1834
- const cutoffMonth = cutoffDate.toISOString().substring(0, 7); // YYYY-MM
1835
-
1836
- const [ok, err, oldTransactions] = await tryFn(() =>
1837
- this.transactionResource.query({
1838
- applied: true,
1839
- timestamp: { '<': cutoffIso }
1840
- })
1841
- );
1842
-
1843
- if (!ok) {
1844
- if (this.config.verbose) {
1845
- console.warn(`[EventualConsistency] GC failed to query transactions:`, err?.message);
1846
- }
1847
- return;
1848
- }
1849
-
1850
- if (!oldTransactions || oldTransactions.length === 0) {
1851
- if (this.config.verbose) {
1852
- console.log(`[EventualConsistency] No old transactions to clean up`);
1853
- }
1854
- return;
1855
- }
1856
-
1857
- if (this.config.verbose) {
1858
- console.log(`[EventualConsistency] Deleting ${oldTransactions.length} old transactions`);
1859
- }
1860
-
1861
- // Delete old transactions using PromisePool
1862
- const { results, errors } = await PromisePool
1863
- .for(oldTransactions)
1864
- .withConcurrency(10)
1865
- .process(async (txn) => {
1866
- const [deleted] = await tryFn(() => this.transactionResource.delete(txn.id));
1867
- return deleted;
1868
- });
1869
-
1870
- if (this.config.verbose) {
1871
- console.log(`[EventualConsistency] GC completed: ${results.length} deleted, ${errors.length} errors`);
1872
- }
1873
-
1874
- this.emit('eventual-consistency.gc-completed', {
1875
- resource: this.config.resource,
1876
- field: this.config.field,
1877
- deletedCount: results.length,
1878
- errorCount: errors.length
1879
- });
1880
- } catch (error) {
1881
- if (this.config.verbose) {
1882
- console.warn(`[EventualConsistency] GC error:`, error.message);
1883
- }
1884
- this.emit('eventual-consistency.gc-error', error);
1885
- } finally {
1886
- // Always release GC lock
1887
- await tryFn(() => this.lockResource.delete(gcLockId));
1888
- }
1889
- }
1890
-
1891
- /**
1892
- * Update analytics with consolidated transactions
1893
- * @param {Array} transactions - Array of transactions that were just consolidated
1894
- * @private
1895
- */
1896
- async updateAnalytics(transactions) {
1897
- if (!this.analyticsResource || transactions.length === 0) return;
1898
-
1899
- if (this.config.verbose) {
1900
- console.log(
1901
- `[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
1902
- `Updating analytics for ${transactions.length} transactions...`
1903
- );
1904
- }
1905
-
1906
- try {
1907
- // Group transactions by cohort hour
1908
- const byHour = this._groupByCohort(transactions, 'cohortHour');
1909
- const cohortCount = Object.keys(byHour).length;
1910
-
1911
- if (this.config.verbose) {
1912
- console.log(
1913
- `[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
1914
- `Updating ${cohortCount} hourly analytics cohorts...`
1915
- );
1916
- }
1917
-
1918
- // Update hourly analytics
1919
- for (const [cohort, txns] of Object.entries(byHour)) {
1920
- await this._upsertAnalytics('hour', cohort, txns);
1921
- }
1922
-
1923
- // Roll up to daily and monthly if configured
1924
- if (this.config.analyticsConfig.rollupStrategy === 'incremental') {
1925
- const uniqueHours = Object.keys(byHour);
1926
-
1927
- if (this.config.verbose) {
1928
- console.log(
1929
- `[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
1930
- `Rolling up ${uniqueHours.length} hours to daily/monthly analytics...`
1931
- );
1932
- }
1933
-
1934
- for (const cohortHour of uniqueHours) {
1935
- await this._rollupAnalytics(cohortHour);
1936
- }
1937
- }
1938
-
1939
- if (this.config.verbose) {
1940
- console.log(
1941
- `[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
1942
- `Analytics update complete for ${cohortCount} cohorts`
1943
- );
1944
- }
1945
- } catch (error) {
1946
- console.warn(
1947
- `[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
1948
- `Analytics update error:`,
1949
- error.message
1950
- );
1951
- }
1952
- }
1953
-
1954
- /**
1955
- * Group transactions by cohort
1956
- * @private
1957
- */
1958
- _groupByCohort(transactions, cohortField) {
1959
- const groups = {};
1960
- for (const txn of transactions) {
1961
- const cohort = txn[cohortField];
1962
- if (!cohort) continue;
1963
-
1964
- if (!groups[cohort]) {
1965
- groups[cohort] = [];
1966
- }
1967
- groups[cohort].push(txn);
1968
- }
1969
- return groups;
1970
- }
1971
-
1972
- /**
1973
- * Upsert analytics for a specific period and cohort
1974
- * @private
1975
- */
1976
- async _upsertAnalytics(period, cohort, transactions) {
1977
- const id = `${period}-${cohort}`;
1978
-
1979
- // Calculate metrics
1980
- const transactionCount = transactions.length;
1981
-
1982
- // Calculate signed values (considering operation type)
1983
- const signedValues = transactions.map(t => {
1984
- if (t.operation === 'sub') return -t.value;
1985
- return t.value;
1986
- });
1987
-
1988
- const totalValue = signedValues.reduce((sum, v) => sum + v, 0);
1989
- const avgValue = totalValue / transactionCount;
1990
- const minValue = Math.min(...signedValues);
1991
- const maxValue = Math.max(...signedValues);
1992
-
1993
- // Calculate operation breakdown
1994
- const operations = this._calculateOperationBreakdown(transactions);
1995
-
1996
- // Count distinct records
1997
- const recordCount = new Set(transactions.map(t => t.originalId)).size;
1998
-
1999
- const now = new Date().toISOString();
2000
-
2001
- // Try to get existing analytics
2002
- const [existingOk, existingErr, existing] = await tryFn(() =>
2003
- this.analyticsResource.get(id)
2004
- );
2005
-
2006
- if (existingOk && existing) {
2007
- // Update existing analytics (incremental)
2008
- const newTransactionCount = existing.transactionCount + transactionCount;
2009
- const newTotalValue = existing.totalValue + totalValue;
2010
- const newAvgValue = newTotalValue / newTransactionCount;
2011
- const newMinValue = Math.min(existing.minValue, minValue);
2012
- const newMaxValue = Math.max(existing.maxValue, maxValue);
2013
-
2014
- // Merge operation breakdown
2015
- const newOperations = { ...existing.operations };
2016
- for (const [op, stats] of Object.entries(operations)) {
2017
- if (!newOperations[op]) {
2018
- newOperations[op] = { count: 0, sum: 0 };
2019
- }
2020
- newOperations[op].count += stats.count;
2021
- newOperations[op].sum += stats.sum;
2022
- }
2023
-
2024
- // Update record count (approximate - we don't track all unique IDs)
2025
- const newRecordCount = Math.max(existing.recordCount, recordCount);
2026
-
2027
- await tryFn(() =>
2028
- this.analyticsResource.update(id, {
2029
- transactionCount: newTransactionCount,
2030
- totalValue: newTotalValue,
2031
- avgValue: newAvgValue,
2032
- minValue: newMinValue,
2033
- maxValue: newMaxValue,
2034
- operations: newOperations,
2035
- recordCount: newRecordCount,
2036
- updatedAt: now
2037
- })
2038
- );
2039
- } else {
2040
- // Create new analytics
2041
- await tryFn(() =>
2042
- this.analyticsResource.insert({
2043
- id,
2044
- period,
2045
- cohort,
2046
- transactionCount,
2047
- totalValue,
2048
- avgValue,
2049
- minValue,
2050
- maxValue,
2051
- operations,
2052
- recordCount,
2053
- consolidatedAt: now,
2054
- updatedAt: now
2055
- })
2056
- );
2057
- }
2058
- }
2059
-
2060
- /**
2061
- * Calculate operation breakdown
2062
- * @private
2063
- */
2064
- _calculateOperationBreakdown(transactions) {
2065
- const breakdown = {};
2066
-
2067
- for (const txn of transactions) {
2068
- const op = txn.operation;
2069
- if (!breakdown[op]) {
2070
- breakdown[op] = { count: 0, sum: 0 };
2071
- }
2072
- breakdown[op].count++;
2073
-
2074
- // Use signed value for sum (sub operations are negative)
2075
- const signedValue = op === 'sub' ? -txn.value : txn.value;
2076
- breakdown[op].sum += signedValue;
2077
- }
2078
-
2079
- return breakdown;
2080
- }
2081
-
2082
- /**
2083
- * Roll up hourly analytics to daily and monthly
2084
- * @private
2085
- */
2086
- async _rollupAnalytics(cohortHour) {
2087
- // cohortHour format: '2025-10-09T14'
2088
- const cohortDate = cohortHour.substring(0, 10); // '2025-10-09'
2089
- const cohortMonth = cohortHour.substring(0, 7); // '2025-10'
2090
-
2091
- // Roll up to day
2092
- await this._rollupPeriod('day', cohortDate, cohortDate);
2093
-
2094
- // Roll up to month
2095
- await this._rollupPeriod('month', cohortMonth, cohortMonth);
2096
- }
2097
-
2098
- /**
2099
- * Roll up analytics for a specific period
2100
- * @private
2101
- */
2102
- async _rollupPeriod(period, cohort, sourcePrefix) {
2103
- // Get all source analytics (e.g., all hours for a day)
2104
- const sourcePeriod = period === 'day' ? 'hour' : 'day';
2105
-
2106
- const [ok, err, allAnalytics] = await tryFn(() =>
2107
- this.analyticsResource.list()
2108
- );
2109
-
2110
- if (!ok || !allAnalytics) return;
2111
-
2112
- // Filter to matching cohorts
2113
- const sourceAnalytics = allAnalytics.filter(a =>
2114
- a.period === sourcePeriod && a.cohort.startsWith(sourcePrefix)
2115
- );
2116
-
2117
- if (sourceAnalytics.length === 0) return;
2118
-
2119
- // Aggregate metrics
2120
- const transactionCount = sourceAnalytics.reduce((sum, a) => sum + a.transactionCount, 0);
2121
- const totalValue = sourceAnalytics.reduce((sum, a) => sum + a.totalValue, 0);
2122
- const avgValue = totalValue / transactionCount;
2123
- const minValue = Math.min(...sourceAnalytics.map(a => a.minValue));
2124
- const maxValue = Math.max(...sourceAnalytics.map(a => a.maxValue));
2125
-
2126
- // Merge operation breakdown
2127
- const operations = {};
2128
- for (const analytics of sourceAnalytics) {
2129
- for (const [op, stats] of Object.entries(analytics.operations || {})) {
2130
- if (!operations[op]) {
2131
- operations[op] = { count: 0, sum: 0 };
2132
- }
2133
- operations[op].count += stats.count;
2134
- operations[op].sum += stats.sum;
2135
- }
2136
- }
2137
-
2138
- // Approximate record count (max of all periods)
2139
- const recordCount = Math.max(...sourceAnalytics.map(a => a.recordCount));
2140
-
2141
- const id = `${period}-${cohort}`;
2142
- const now = new Date().toISOString();
2143
-
2144
- // Upsert rolled-up analytics
2145
- const [existingOk, existingErr, existing] = await tryFn(() =>
2146
- this.analyticsResource.get(id)
2147
- );
2148
-
2149
- if (existingOk && existing) {
2150
- await tryFn(() =>
2151
- this.analyticsResource.update(id, {
2152
- transactionCount,
2153
- totalValue,
2154
- avgValue,
2155
- minValue,
2156
- maxValue,
2157
- operations,
2158
- recordCount,
2159
- updatedAt: now
2160
- })
2161
- );
2162
- } else {
2163
- await tryFn(() =>
2164
- this.analyticsResource.insert({
2165
- id,
2166
- period,
2167
- cohort,
2168
- transactionCount,
2169
- totalValue,
2170
- avgValue,
2171
- minValue,
2172
- maxValue,
2173
- operations,
2174
- recordCount,
2175
- consolidatedAt: now,
2176
- updatedAt: now
2177
- })
2178
- );
2179
- }
2180
- }
2181
-
2182
- /**
2183
- * Get analytics for a specific period
2184
- * @param {string} resourceName - Resource name
2185
- * @param {string} field - Field name
2186
- * @param {Object} options - Query options
2187
- * @returns {Promise<Array>} Analytics data
2188
- */
2189
- async getAnalytics(resourceName, field, options = {}) {
2190
- // Get handler for this resource/field combination
2191
- const fieldHandlers = this.fieldHandlers.get(resourceName);
2192
- if (!fieldHandlers) {
2193
- throw new Error(`No eventual consistency configured for resource: ${resourceName}`);
2194
- }
2195
-
2196
- const handler = fieldHandlers.get(field);
2197
- if (!handler) {
2198
- throw new Error(`No eventual consistency configured for field: ${resourceName}.${field}`);
2199
- }
2200
-
2201
- if (!handler.analyticsResource) {
2202
- throw new Error('Analytics not enabled for this plugin');
2203
- }
2204
-
2205
- const { period = 'day', date, startDate, endDate, month, year, breakdown = false } = options;
2206
-
2207
- const [ok, err, allAnalytics] = await tryFn(() =>
2208
- handler.analyticsResource.list()
2209
- );
2210
-
2211
- if (!ok || !allAnalytics) {
2212
- return [];
2213
- }
2214
-
2215
- // Filter by period
2216
- let filtered = allAnalytics.filter(a => a.period === period);
2217
-
2218
- // Filter by date/range
2219
- if (date) {
2220
- if (period === 'hour') {
2221
- // Match all hours of the date
2222
- filtered = filtered.filter(a => a.cohort.startsWith(date));
2223
- } else {
2224
- filtered = filtered.filter(a => a.cohort === date);
2225
- }
2226
- } else if (startDate && endDate) {
2227
- filtered = filtered.filter(a => a.cohort >= startDate && a.cohort <= endDate);
2228
- } else if (month) {
2229
- filtered = filtered.filter(a => a.cohort.startsWith(month));
2230
- } else if (year) {
2231
- filtered = filtered.filter(a => a.cohort.startsWith(String(year)));
2232
- }
2233
-
2234
- // Sort by cohort
2235
- filtered.sort((a, b) => a.cohort.localeCompare(b.cohort));
2236
-
2237
- // Return with or without breakdown
2238
- if (breakdown === 'operations') {
2239
- return filtered.map(a => ({
2240
- cohort: a.cohort,
2241
- ...a.operations
2242
- }));
2243
- }
2244
-
2245
- return filtered.map(a => ({
2246
- cohort: a.cohort,
2247
- count: a.transactionCount,
2248
- sum: a.totalValue,
2249
- avg: a.avgValue,
2250
- min: a.minValue,
2251
- max: a.maxValue,
2252
- operations: a.operations,
2253
- recordCount: a.recordCount
2254
- }));
2255
- }
2256
-
2257
- /**
2258
- * Fill gaps in analytics data with zeros for continuous time series
2259
- * @private
2260
- * @param {Array} data - Sparse analytics data
2261
- * @param {string} period - Period type ('hour', 'day', 'month')
2262
- * @param {string} startDate - Start date (ISO format)
2263
- * @param {string} endDate - End date (ISO format)
2264
- * @returns {Array} Complete time series with gaps filled
2265
- */
2266
- _fillGaps(data, period, startDate, endDate) {
2267
- if (!data || data.length === 0) {
2268
- // If no data, still generate empty series
2269
- data = [];
2270
- }
2271
-
2272
- // Create a map of existing data by cohort
2273
- const dataMap = new Map();
2274
- data.forEach(item => {
2275
- dataMap.set(item.cohort, item);
2276
- });
2277
-
2278
- const result = [];
2279
- const emptyRecord = {
2280
- count: 0,
2281
- sum: 0,
2282
- avg: 0,
2283
- min: 0,
2284
- max: 0,
2285
- recordCount: 0
2286
- };
2287
-
2288
- if (period === 'hour') {
2289
- // Generate all hours between startDate and endDate
2290
- const start = new Date(startDate + 'T00:00:00Z');
2291
- const end = new Date(endDate + 'T23:59:59Z');
2292
-
2293
- for (let dt = new Date(start); dt <= end; dt.setHours(dt.getHours() + 1)) {
2294
- const cohort = dt.toISOString().substring(0, 13); // YYYY-MM-DDTHH
2295
- result.push(dataMap.get(cohort) || { cohort, ...emptyRecord });
2296
- }
2297
- } else if (period === 'day') {
2298
- // Generate all days between startDate and endDate
2299
- const start = new Date(startDate);
2300
- const end = new Date(endDate);
2301
-
2302
- for (let dt = new Date(start); dt <= end; dt.setDate(dt.getDate() + 1)) {
2303
- const cohort = dt.toISOString().substring(0, 10); // YYYY-MM-DD
2304
- result.push(dataMap.get(cohort) || { cohort, ...emptyRecord });
2305
- }
2306
- } else if (period === 'month') {
2307
- // Generate all months between startDate and endDate
2308
- const startYear = parseInt(startDate.substring(0, 4));
2309
- const startMonth = parseInt(startDate.substring(5, 7));
2310
- const endYear = parseInt(endDate.substring(0, 4));
2311
- const endMonth = parseInt(endDate.substring(5, 7));
2312
-
2313
- for (let year = startYear; year <= endYear; year++) {
2314
- const firstMonth = (year === startYear) ? startMonth : 1;
2315
- const lastMonth = (year === endYear) ? endMonth : 12;
2316
-
2317
- for (let month = firstMonth; month <= lastMonth; month++) {
2318
- const cohort = `${year}-${month.toString().padStart(2, '0')}`;
2319
- result.push(dataMap.get(cohort) || { cohort, ...emptyRecord });
2320
- }
2321
- }
2322
- }
2323
-
2324
- return result;
2325
- }
2326
-
2327
- /**
2328
- * Get analytics for entire month, broken down by days
2329
- * @param {string} resourceName - Resource name
2330
- * @param {string} field - Field name
2331
- * @param {string} month - Month in YYYY-MM format
2332
- * @param {Object} options - Options
2333
- * @param {boolean} options.fillGaps - Fill missing days with zeros (default: false)
2334
- * @returns {Promise<Array>} Daily analytics for the month
2335
- */
2336
- async getMonthByDay(resourceName, field, month, options = {}) {
2337
- // month format: '2025-10'
2338
- const year = parseInt(month.substring(0, 4));
2339
- const monthNum = parseInt(month.substring(5, 7));
2340
-
2341
- // Get first and last day of month
2342
- const firstDay = new Date(year, monthNum - 1, 1);
2343
- const lastDay = new Date(year, monthNum, 0);
2344
-
2345
- const startDate = firstDay.toISOString().substring(0, 10);
2346
- const endDate = lastDay.toISOString().substring(0, 10);
2347
-
2348
- const data = await this.getAnalytics(resourceName, field, {
2349
- period: 'day',
2350
- startDate,
2351
- endDate
2352
- });
2353
-
2354
- if (options.fillGaps) {
2355
- return this._fillGaps(data, 'day', startDate, endDate);
2356
- }
2357
-
2358
- return data;
2359
- }
2360
-
2361
- /**
2362
- * Get analytics for entire day, broken down by hours
2363
- * @param {string} resourceName - Resource name
2364
- * @param {string} field - Field name
2365
- * @param {string} date - Date in YYYY-MM-DD format
2366
- * @param {Object} options - Options
2367
- * @param {boolean} options.fillGaps - Fill missing hours with zeros (default: false)
2368
- * @returns {Promise<Array>} Hourly analytics for the day
2369
- */
2370
- async getDayByHour(resourceName, field, date, options = {}) {
2371
- // date format: '2025-10-09'
2372
- const data = await this.getAnalytics(resourceName, field, {
2373
- period: 'hour',
2374
- date
2375
- });
2376
-
2377
- if (options.fillGaps) {
2378
- return this._fillGaps(data, 'hour', date, date);
2379
- }
2380
-
2381
- return data;
2382
- }
2383
-
2384
- /**
2385
- * Get analytics for last N days, broken down by days
2386
- * @param {string} resourceName - Resource name
2387
- * @param {string} field - Field name
2388
- * @param {number} days - Number of days to look back (default: 7)
2389
- * @param {Object} options - Options
2390
- * @param {boolean} options.fillGaps - Fill missing days with zeros (default: false)
2391
- * @returns {Promise<Array>} Daily analytics
2392
- */
2393
- async getLastNDays(resourceName, field, days = 7, options = {}) {
2394
- const dates = Array.from({ length: days }, (_, i) => {
2395
- const date = new Date();
2396
- date.setDate(date.getDate() - i);
2397
- return date.toISOString().substring(0, 10);
2398
- }).reverse();
2399
-
2400
- const data = await this.getAnalytics(resourceName, field, {
2401
- period: 'day',
2402
- startDate: dates[0],
2403
- endDate: dates[dates.length - 1]
2404
- });
2405
-
2406
- if (options.fillGaps) {
2407
- return this._fillGaps(data, 'day', dates[0], dates[dates.length - 1]);
2408
- }
2409
-
2410
- return data;
2411
- }
2412
-
2413
- /**
2414
- * Get analytics for entire year, broken down by months
2415
- * @param {string} resourceName - Resource name
2416
- * @param {string} field - Field name
2417
- * @param {number} year - Year (e.g., 2025)
2418
- * @param {Object} options - Options
2419
- * @param {boolean} options.fillGaps - Fill missing months with zeros (default: false)
2420
- * @returns {Promise<Array>} Monthly analytics for the year
2421
- */
2422
- async getYearByMonth(resourceName, field, year, options = {}) {
2423
- const data = await this.getAnalytics(resourceName, field, {
2424
- period: 'month',
2425
- year
2426
- });
2427
-
2428
- if (options.fillGaps) {
2429
- const startDate = `${year}-01`;
2430
- const endDate = `${year}-12`;
2431
- return this._fillGaps(data, 'month', startDate, endDate);
2432
- }
2433
-
2434
- return data;
2435
- }
2436
-
2437
- /**
2438
- * Get analytics for entire month, broken down by hours
2439
- * @param {string} resourceName - Resource name
2440
- * @param {string} field - Field name
2441
- * @param {string} month - Month in YYYY-MM format (or 'last' for previous month)
2442
- * @param {Object} options - Options
2443
- * @param {boolean} options.fillGaps - Fill missing hours with zeros (default: false)
2444
- * @returns {Promise<Array>} Hourly analytics for the month (up to 24*31=744 records)
2445
- */
2446
- async getMonthByHour(resourceName, field, month, options = {}) {
2447
- // month format: '2025-10' or 'last'
2448
- let year, monthNum;
2449
-
2450
- if (month === 'last') {
2451
- const now = new Date();
2452
- now.setMonth(now.getMonth() - 1);
2453
- year = now.getFullYear();
2454
- monthNum = now.getMonth() + 1;
2455
- } else {
2456
- year = parseInt(month.substring(0, 4));
2457
- monthNum = parseInt(month.substring(5, 7));
2458
- }
2459
-
2460
- // Get first and last day of month
2461
- const firstDay = new Date(year, monthNum - 1, 1);
2462
- const lastDay = new Date(year, monthNum, 0);
2463
-
2464
- const startDate = firstDay.toISOString().substring(0, 10);
2465
- const endDate = lastDay.toISOString().substring(0, 10);
2466
-
2467
- const data = await this.getAnalytics(resourceName, field, {
2468
- period: 'hour',
2469
- startDate,
2470
- endDate
2471
- });
2472
-
2473
- if (options.fillGaps) {
2474
- return this._fillGaps(data, 'hour', startDate, endDate);
2475
- }
2476
-
2477
- return data;
2478
- }
2479
-
2480
- /**
2481
- * Get top records by volume
2482
- * @param {string} resourceName - Resource name
2483
- * @param {string} field - Field name
2484
- * @param {Object} options - Query options
2485
- * @returns {Promise<Array>} Top records
2486
- */
2487
- async getTopRecords(resourceName, field, options = {}) {
2488
- // Get handler for this resource/field combination
2489
- const fieldHandlers = this.fieldHandlers.get(resourceName);
2490
- if (!fieldHandlers) {
2491
- throw new Error(`No eventual consistency configured for resource: ${resourceName}`);
2492
- }
2493
-
2494
- const handler = fieldHandlers.get(field);
2495
- if (!handler) {
2496
- throw new Error(`No eventual consistency configured for field: ${resourceName}.${field}`);
2497
- }
2498
-
2499
- if (!handler.transactionResource) {
2500
- throw new Error('Transaction resource not initialized');
2501
- }
2502
-
2503
- const { period = 'day', date, metric = 'transactionCount', limit = 10 } = options;
2504
-
2505
- // Get all transactions for the period
2506
- const [ok, err, transactions] = await tryFn(() =>
2507
- handler.transactionResource.list()
2508
- );
2509
-
2510
- if (!ok || !transactions) {
2511
- return [];
2512
- }
2513
-
2514
- // Filter by date
2515
- let filtered = transactions;
2516
- if (date) {
2517
- if (period === 'hour') {
2518
- filtered = transactions.filter(t => t.cohortHour && t.cohortHour.startsWith(date));
2519
- } else if (period === 'day') {
2520
- filtered = transactions.filter(t => t.cohortDate === date);
2521
- } else if (period === 'month') {
2522
- filtered = transactions.filter(t => t.cohortMonth && t.cohortMonth.startsWith(date));
2523
- }
2524
- }
2525
-
2526
- // Group by originalId
2527
- const byRecord = {};
2528
- for (const txn of filtered) {
2529
- const recordId = txn.originalId;
2530
- if (!byRecord[recordId]) {
2531
- byRecord[recordId] = { count: 0, sum: 0 };
2532
- }
2533
- byRecord[recordId].count++;
2534
- byRecord[recordId].sum += txn.value;
2535
- }
2536
-
2537
- // Convert to array and sort
2538
- const records = Object.entries(byRecord).map(([recordId, stats]) => ({
2539
- recordId,
2540
- count: stats.count,
2541
- sum: stats.sum
2542
- }));
2543
-
2544
- // Sort by metric
2545
- records.sort((a, b) => {
2546
- if (metric === 'transactionCount') {
2547
- return b.count - a.count;
2548
- } else if (metric === 'totalValue') {
2549
- return b.sum - a.sum;
2550
- }
2551
- return 0;
2552
- });
2553
-
2554
- // Limit results
2555
- return records.slice(0, limit);
2556
- }
2557
- }
2558
-
2559
- export default EventualConsistencyPlugin;