s3db.js 10.0.12 → 10.0.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/s3db.cjs.js +346 -41
- package/dist/s3db.cjs.js.map +1 -1
- package/dist/s3db.es.js +346 -41
- package/dist/s3db.es.js.map +1 -1
- package/package.json +5 -5
- package/src/plugins/eventual-consistency.plugin.js +467 -56
|
@@ -317,10 +317,20 @@ export class EventualConsistencyPlugin extends Plugin {
|
|
|
317
317
|
}
|
|
318
318
|
|
|
319
319
|
async onStart() {
|
|
320
|
-
//
|
|
320
|
+
// Start timers and emit events for all field handlers
|
|
321
321
|
for (const [resourceName, fieldHandlers] of this.fieldHandlers) {
|
|
322
322
|
for (const [fieldName, handler] of fieldHandlers) {
|
|
323
323
|
if (!handler.deferredSetup) {
|
|
324
|
+
// Start auto-consolidation timer if enabled
|
|
325
|
+
if (this.config.autoConsolidate && this.config.mode === 'async') {
|
|
326
|
+
this.startConsolidationTimerForHandler(handler, resourceName, fieldName);
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
// Start garbage collection timer
|
|
330
|
+
if (this.config.transactionRetention && this.config.transactionRetention > 0) {
|
|
331
|
+
this.startGarbageCollectionTimerForHandler(handler, resourceName, fieldName);
|
|
332
|
+
}
|
|
333
|
+
|
|
324
334
|
this.emit('eventual-consistency.started', {
|
|
325
335
|
resource: resourceName,
|
|
326
336
|
field: fieldName,
|
|
@@ -467,20 +477,57 @@ export class EventualConsistencyPlugin extends Plugin {
|
|
|
467
477
|
// Add method to set value (replaces current value)
|
|
468
478
|
// Signature: set(id, field, value)
|
|
469
479
|
resource.set = async (id, field, value) => {
|
|
470
|
-
const { plugin:
|
|
480
|
+
const { plugin: handler } =
|
|
471
481
|
plugin._resolveFieldAndPlugin(resource, field, value);
|
|
472
482
|
|
|
473
|
-
// Create
|
|
474
|
-
|
|
483
|
+
// Create transaction inline
|
|
484
|
+
const now = new Date();
|
|
485
|
+
const cohortInfo = plugin.getCohortInfo(now);
|
|
486
|
+
|
|
487
|
+
const transaction = {
|
|
488
|
+
id: idGenerator(),
|
|
475
489
|
originalId: id,
|
|
476
|
-
|
|
490
|
+
field: handler.field,
|
|
477
491
|
value: value,
|
|
478
|
-
|
|
479
|
-
|
|
492
|
+
operation: 'set',
|
|
493
|
+
timestamp: now.toISOString(),
|
|
494
|
+
cohortDate: cohortInfo.date,
|
|
495
|
+
cohortHour: cohortInfo.hour,
|
|
496
|
+
cohortMonth: cohortInfo.month,
|
|
497
|
+
source: 'set',
|
|
498
|
+
applied: false
|
|
499
|
+
};
|
|
500
|
+
|
|
501
|
+
await handler.transactionResource.insert(transaction);
|
|
502
|
+
|
|
503
|
+
// In sync mode, immediately consolidate
|
|
504
|
+
if (plugin.config.mode === 'sync') {
|
|
505
|
+
// Temporarily set config for legacy methods
|
|
506
|
+
const oldResource = plugin.config.resource;
|
|
507
|
+
const oldField = plugin.config.field;
|
|
508
|
+
const oldTransactionResource = plugin.transactionResource;
|
|
509
|
+
const oldTargetResource = plugin.targetResource;
|
|
510
|
+
const oldLockResource = plugin.lockResource;
|
|
511
|
+
const oldAnalyticsResource = plugin.analyticsResource;
|
|
480
512
|
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
513
|
+
plugin.config.resource = handler.resource;
|
|
514
|
+
plugin.config.field = handler.field;
|
|
515
|
+
plugin.transactionResource = handler.transactionResource;
|
|
516
|
+
plugin.targetResource = handler.targetResource;
|
|
517
|
+
plugin.lockResource = handler.lockResource;
|
|
518
|
+
plugin.analyticsResource = handler.analyticsResource;
|
|
519
|
+
|
|
520
|
+
const result = await plugin._syncModeConsolidate(id, field);
|
|
521
|
+
|
|
522
|
+
// Restore
|
|
523
|
+
plugin.config.resource = oldResource;
|
|
524
|
+
plugin.config.field = oldField;
|
|
525
|
+
plugin.transactionResource = oldTransactionResource;
|
|
526
|
+
plugin.targetResource = oldTargetResource;
|
|
527
|
+
plugin.lockResource = oldLockResource;
|
|
528
|
+
plugin.analyticsResource = oldAnalyticsResource;
|
|
529
|
+
|
|
530
|
+
return result;
|
|
484
531
|
}
|
|
485
532
|
|
|
486
533
|
return value;
|
|
@@ -489,48 +536,120 @@ export class EventualConsistencyPlugin extends Plugin {
|
|
|
489
536
|
// Add method to increment value
|
|
490
537
|
// Signature: add(id, field, amount)
|
|
491
538
|
resource.add = async (id, field, amount) => {
|
|
492
|
-
const { plugin:
|
|
539
|
+
const { plugin: handler } =
|
|
493
540
|
plugin._resolveFieldAndPlugin(resource, field, amount);
|
|
494
541
|
|
|
495
|
-
// Create
|
|
496
|
-
|
|
542
|
+
// Create transaction inline
|
|
543
|
+
const now = new Date();
|
|
544
|
+
const cohortInfo = plugin.getCohortInfo(now);
|
|
545
|
+
|
|
546
|
+
const transaction = {
|
|
547
|
+
id: idGenerator(),
|
|
497
548
|
originalId: id,
|
|
498
|
-
|
|
549
|
+
field: handler.field,
|
|
499
550
|
value: amount,
|
|
500
|
-
|
|
501
|
-
|
|
551
|
+
operation: 'add',
|
|
552
|
+
timestamp: now.toISOString(),
|
|
553
|
+
cohortDate: cohortInfo.date,
|
|
554
|
+
cohortHour: cohortInfo.hour,
|
|
555
|
+
cohortMonth: cohortInfo.month,
|
|
556
|
+
source: 'add',
|
|
557
|
+
applied: false
|
|
558
|
+
};
|
|
502
559
|
|
|
503
|
-
|
|
504
|
-
if (fieldPlugin.config.mode === 'sync') {
|
|
505
|
-
return await fieldPlugin._syncModeConsolidate(fieldPlugin, id, field);
|
|
506
|
-
}
|
|
560
|
+
await handler.transactionResource.insert(transaction);
|
|
507
561
|
|
|
508
|
-
// In
|
|
509
|
-
|
|
562
|
+
// In sync mode, immediately consolidate
|
|
563
|
+
if (plugin.config.mode === 'sync') {
|
|
564
|
+
const oldResource = plugin.config.resource;
|
|
565
|
+
const oldField = plugin.config.field;
|
|
566
|
+
const oldTransactionResource = plugin.transactionResource;
|
|
567
|
+
const oldTargetResource = plugin.targetResource;
|
|
568
|
+
const oldLockResource = plugin.lockResource;
|
|
569
|
+
const oldAnalyticsResource = plugin.analyticsResource;
|
|
570
|
+
|
|
571
|
+
plugin.config.resource = handler.resource;
|
|
572
|
+
plugin.config.field = handler.field;
|
|
573
|
+
plugin.transactionResource = handler.transactionResource;
|
|
574
|
+
plugin.targetResource = handler.targetResource;
|
|
575
|
+
plugin.lockResource = handler.lockResource;
|
|
576
|
+
plugin.analyticsResource = handler.analyticsResource;
|
|
577
|
+
|
|
578
|
+
const result = await plugin._syncModeConsolidate(id, field);
|
|
579
|
+
|
|
580
|
+
plugin.config.resource = oldResource;
|
|
581
|
+
plugin.config.field = oldField;
|
|
582
|
+
plugin.transactionResource = oldTransactionResource;
|
|
583
|
+
plugin.targetResource = oldTargetResource;
|
|
584
|
+
plugin.lockResource = oldLockResource;
|
|
585
|
+
plugin.analyticsResource = oldAnalyticsResource;
|
|
586
|
+
|
|
587
|
+
return result;
|
|
588
|
+
}
|
|
589
|
+
|
|
590
|
+
// Async mode - return current value (optimistic)
|
|
591
|
+
const [ok, err, record] = await tryFn(() => handler.targetResource.get(id));
|
|
592
|
+
const currentValue = (ok && record) ? (record[field] || 0) : 0;
|
|
510
593
|
return currentValue + amount;
|
|
511
594
|
};
|
|
512
595
|
|
|
513
596
|
// Add method to decrement value
|
|
514
597
|
// Signature: sub(id, field, amount)
|
|
515
598
|
resource.sub = async (id, field, amount) => {
|
|
516
|
-
const { plugin:
|
|
599
|
+
const { plugin: handler } =
|
|
517
600
|
plugin._resolveFieldAndPlugin(resource, field, amount);
|
|
518
601
|
|
|
519
|
-
// Create
|
|
520
|
-
|
|
602
|
+
// Create transaction inline
|
|
603
|
+
const now = new Date();
|
|
604
|
+
const cohortInfo = plugin.getCohortInfo(now);
|
|
605
|
+
|
|
606
|
+
const transaction = {
|
|
607
|
+
id: idGenerator(),
|
|
521
608
|
originalId: id,
|
|
522
|
-
|
|
609
|
+
field: handler.field,
|
|
523
610
|
value: amount,
|
|
524
|
-
|
|
525
|
-
|
|
611
|
+
operation: 'sub',
|
|
612
|
+
timestamp: now.toISOString(),
|
|
613
|
+
cohortDate: cohortInfo.date,
|
|
614
|
+
cohortHour: cohortInfo.hour,
|
|
615
|
+
cohortMonth: cohortInfo.month,
|
|
616
|
+
source: 'sub',
|
|
617
|
+
applied: false
|
|
618
|
+
};
|
|
526
619
|
|
|
527
|
-
|
|
528
|
-
if (fieldPlugin.config.mode === 'sync') {
|
|
529
|
-
return await fieldPlugin._syncModeConsolidate(fieldPlugin, id, field);
|
|
530
|
-
}
|
|
620
|
+
await handler.transactionResource.insert(transaction);
|
|
531
621
|
|
|
532
|
-
// In
|
|
533
|
-
|
|
622
|
+
// In sync mode, immediately consolidate
|
|
623
|
+
if (plugin.config.mode === 'sync') {
|
|
624
|
+
const oldResource = plugin.config.resource;
|
|
625
|
+
const oldField = plugin.config.field;
|
|
626
|
+
const oldTransactionResource = plugin.transactionResource;
|
|
627
|
+
const oldTargetResource = plugin.targetResource;
|
|
628
|
+
const oldLockResource = plugin.lockResource;
|
|
629
|
+
const oldAnalyticsResource = plugin.analyticsResource;
|
|
630
|
+
|
|
631
|
+
plugin.config.resource = handler.resource;
|
|
632
|
+
plugin.config.field = handler.field;
|
|
633
|
+
plugin.transactionResource = handler.transactionResource;
|
|
634
|
+
plugin.targetResource = handler.targetResource;
|
|
635
|
+
plugin.lockResource = handler.lockResource;
|
|
636
|
+
plugin.analyticsResource = handler.analyticsResource;
|
|
637
|
+
|
|
638
|
+
const result = await plugin._syncModeConsolidate(id, field);
|
|
639
|
+
|
|
640
|
+
plugin.config.resource = oldResource;
|
|
641
|
+
plugin.config.field = oldField;
|
|
642
|
+
plugin.transactionResource = oldTransactionResource;
|
|
643
|
+
plugin.targetResource = oldTargetResource;
|
|
644
|
+
plugin.lockResource = oldLockResource;
|
|
645
|
+
plugin.analyticsResource = oldAnalyticsResource;
|
|
646
|
+
|
|
647
|
+
return result;
|
|
648
|
+
}
|
|
649
|
+
|
|
650
|
+
// Async mode - return current value (optimistic)
|
|
651
|
+
const [ok, err, record] = await tryFn(() => handler.targetResource.get(id));
|
|
652
|
+
const currentValue = (ok && record) ? (record[field] || 0) : 0;
|
|
534
653
|
return currentValue - amount;
|
|
535
654
|
};
|
|
536
655
|
|
|
@@ -541,9 +660,9 @@ export class EventualConsistencyPlugin extends Plugin {
|
|
|
541
660
|
throw new Error(`Field parameter is required: consolidate(id, field)`);
|
|
542
661
|
}
|
|
543
662
|
|
|
544
|
-
const
|
|
663
|
+
const handler = resource._eventualConsistencyPlugins[field];
|
|
545
664
|
|
|
546
|
-
if (!
|
|
665
|
+
if (!handler) {
|
|
547
666
|
const availableFields = Object.keys(resource._eventualConsistencyPlugins).join(', ');
|
|
548
667
|
throw new Error(
|
|
549
668
|
`No eventual consistency plugin found for field "${field}". ` +
|
|
@@ -551,15 +670,39 @@ export class EventualConsistencyPlugin extends Plugin {
|
|
|
551
670
|
);
|
|
552
671
|
}
|
|
553
672
|
|
|
554
|
-
|
|
673
|
+
// Temporarily set config for legacy methods
|
|
674
|
+
const oldResource = plugin.config.resource;
|
|
675
|
+
const oldField = plugin.config.field;
|
|
676
|
+
const oldTransactionResource = plugin.transactionResource;
|
|
677
|
+
const oldTargetResource = plugin.targetResource;
|
|
678
|
+
const oldLockResource = plugin.lockResource;
|
|
679
|
+
const oldAnalyticsResource = plugin.analyticsResource;
|
|
680
|
+
|
|
681
|
+
plugin.config.resource = handler.resource;
|
|
682
|
+
plugin.config.field = handler.field;
|
|
683
|
+
plugin.transactionResource = handler.transactionResource;
|
|
684
|
+
plugin.targetResource = handler.targetResource;
|
|
685
|
+
plugin.lockResource = handler.lockResource;
|
|
686
|
+
plugin.analyticsResource = handler.analyticsResource;
|
|
687
|
+
|
|
688
|
+
const result = await plugin.consolidateRecord(id);
|
|
689
|
+
|
|
690
|
+
plugin.config.resource = oldResource;
|
|
691
|
+
plugin.config.field = oldField;
|
|
692
|
+
plugin.transactionResource = oldTransactionResource;
|
|
693
|
+
plugin.targetResource = oldTargetResource;
|
|
694
|
+
plugin.lockResource = oldLockResource;
|
|
695
|
+
plugin.analyticsResource = oldAnalyticsResource;
|
|
696
|
+
|
|
697
|
+
return result;
|
|
555
698
|
};
|
|
556
699
|
|
|
557
700
|
// Add method to get consolidated value without applying
|
|
558
701
|
// Signature: getConsolidatedValue(id, field, options)
|
|
559
702
|
resource.getConsolidatedValue = async (id, field, options = {}) => {
|
|
560
|
-
const
|
|
703
|
+
const handler = resource._eventualConsistencyPlugins[field];
|
|
561
704
|
|
|
562
|
-
if (!
|
|
705
|
+
if (!handler) {
|
|
563
706
|
const availableFields = Object.keys(resource._eventualConsistencyPlugins).join(', ');
|
|
564
707
|
throw new Error(
|
|
565
708
|
`No eventual consistency plugin found for field "${field}". ` +
|
|
@@ -567,7 +710,25 @@ export class EventualConsistencyPlugin extends Plugin {
|
|
|
567
710
|
);
|
|
568
711
|
}
|
|
569
712
|
|
|
570
|
-
|
|
713
|
+
// Temporarily set config for legacy methods
|
|
714
|
+
const oldResource = plugin.config.resource;
|
|
715
|
+
const oldField = plugin.config.field;
|
|
716
|
+
const oldTransactionResource = plugin.transactionResource;
|
|
717
|
+
const oldTargetResource = plugin.targetResource;
|
|
718
|
+
|
|
719
|
+
plugin.config.resource = handler.resource;
|
|
720
|
+
plugin.config.field = handler.field;
|
|
721
|
+
plugin.transactionResource = handler.transactionResource;
|
|
722
|
+
plugin.targetResource = handler.targetResource;
|
|
723
|
+
|
|
724
|
+
const result = await plugin.getConsolidatedValue(id, options);
|
|
725
|
+
|
|
726
|
+
plugin.config.resource = oldResource;
|
|
727
|
+
plugin.config.field = oldField;
|
|
728
|
+
plugin.transactionResource = oldTransactionResource;
|
|
729
|
+
plugin.targetResource = oldTargetResource;
|
|
730
|
+
|
|
731
|
+
return result;
|
|
571
732
|
};
|
|
572
733
|
}
|
|
573
734
|
|
|
@@ -746,6 +907,52 @@ export class EventualConsistencyPlugin extends Plugin {
|
|
|
746
907
|
}, intervalMs);
|
|
747
908
|
}
|
|
748
909
|
|
|
910
|
+
startConsolidationTimerForHandler(handler, resourceName, fieldName) {
|
|
911
|
+
const intervalMs = this.config.consolidationInterval * 1000; // Convert seconds to ms
|
|
912
|
+
|
|
913
|
+
if (this.config.verbose) {
|
|
914
|
+
const nextRun = new Date(Date.now() + intervalMs);
|
|
915
|
+
console.log(
|
|
916
|
+
`[EventualConsistency] ${resourceName}.${fieldName} - ` +
|
|
917
|
+
`Consolidation timer started. Next run at ${nextRun.toISOString()} ` +
|
|
918
|
+
`(every ${this.config.consolidationInterval}s)`
|
|
919
|
+
);
|
|
920
|
+
}
|
|
921
|
+
|
|
922
|
+
handler.consolidationTimer = setInterval(async () => {
|
|
923
|
+
await this.runConsolidationForHandler(handler, resourceName, fieldName);
|
|
924
|
+
}, intervalMs);
|
|
925
|
+
}
|
|
926
|
+
|
|
927
|
+
async runConsolidationForHandler(handler, resourceName, fieldName) {
|
|
928
|
+
// Temporarily swap config to use this handler
|
|
929
|
+
const oldResource = this.config.resource;
|
|
930
|
+
const oldField = this.config.field;
|
|
931
|
+
const oldTransactionResource = this.transactionResource;
|
|
932
|
+
const oldTargetResource = this.targetResource;
|
|
933
|
+
const oldLockResource = this.lockResource;
|
|
934
|
+
const oldAnalyticsResource = this.analyticsResource;
|
|
935
|
+
|
|
936
|
+
this.config.resource = resourceName;
|
|
937
|
+
this.config.field = fieldName;
|
|
938
|
+
this.transactionResource = handler.transactionResource;
|
|
939
|
+
this.targetResource = handler.targetResource;
|
|
940
|
+
this.lockResource = handler.lockResource;
|
|
941
|
+
this.analyticsResource = handler.analyticsResource;
|
|
942
|
+
|
|
943
|
+
try {
|
|
944
|
+
await this.runConsolidation();
|
|
945
|
+
} finally {
|
|
946
|
+
// Restore
|
|
947
|
+
this.config.resource = oldResource;
|
|
948
|
+
this.config.field = oldField;
|
|
949
|
+
this.transactionResource = oldTransactionResource;
|
|
950
|
+
this.targetResource = oldTargetResource;
|
|
951
|
+
this.lockResource = oldLockResource;
|
|
952
|
+
this.analyticsResource = oldAnalyticsResource;
|
|
953
|
+
}
|
|
954
|
+
}
|
|
955
|
+
|
|
749
956
|
async runConsolidation() {
|
|
750
957
|
const startTime = Date.now();
|
|
751
958
|
|
|
@@ -885,14 +1092,7 @@ export class EventualConsistencyPlugin extends Plugin {
|
|
|
885
1092
|
}
|
|
886
1093
|
|
|
887
1094
|
try {
|
|
888
|
-
// Get
|
|
889
|
-
const [recordOk, recordErr, record] = await tryFn(() =>
|
|
890
|
-
this.targetResource.get(originalId)
|
|
891
|
-
);
|
|
892
|
-
|
|
893
|
-
const currentValue = (recordOk && record) ? (record[this.config.field] || 0) : 0;
|
|
894
|
-
|
|
895
|
-
// Get all transactions for this record
|
|
1095
|
+
// Get all unapplied transactions for this record
|
|
896
1096
|
const [ok, err, transactions] = await tryFn(() =>
|
|
897
1097
|
this.transactionResource.query({
|
|
898
1098
|
originalId,
|
|
@@ -901,6 +1101,12 @@ export class EventualConsistencyPlugin extends Plugin {
|
|
|
901
1101
|
);
|
|
902
1102
|
|
|
903
1103
|
if (!ok || !transactions || transactions.length === 0) {
|
|
1104
|
+
// No pending transactions - try to get current value from record
|
|
1105
|
+
const [recordOk, recordErr, record] = await tryFn(() =>
|
|
1106
|
+
this.targetResource.get(originalId)
|
|
1107
|
+
);
|
|
1108
|
+
const currentValue = (recordOk && record) ? (record[this.config.field] || 0) : 0;
|
|
1109
|
+
|
|
904
1110
|
if (this.config.verbose) {
|
|
905
1111
|
console.log(
|
|
906
1112
|
`[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
|
|
@@ -910,20 +1116,169 @@ export class EventualConsistencyPlugin extends Plugin {
|
|
|
910
1116
|
return currentValue;
|
|
911
1117
|
}
|
|
912
1118
|
|
|
1119
|
+
// Get the LAST APPLIED VALUE from transactions (not from record - avoids S3 eventual consistency issues)
|
|
1120
|
+
// This is the source of truth for the current value
|
|
1121
|
+
const [appliedOk, appliedErr, appliedTransactions] = await tryFn(() =>
|
|
1122
|
+
this.transactionResource.query({
|
|
1123
|
+
originalId,
|
|
1124
|
+
applied: true
|
|
1125
|
+
})
|
|
1126
|
+
);
|
|
1127
|
+
|
|
1128
|
+
let currentValue = 0;
|
|
1129
|
+
|
|
1130
|
+
if (appliedOk && appliedTransactions && appliedTransactions.length > 0) {
|
|
1131
|
+
// Check if record exists - if deleted, ignore old applied transactions
|
|
1132
|
+
const [recordExistsOk, recordExistsErr, recordExists] = await tryFn(() =>
|
|
1133
|
+
this.targetResource.get(originalId)
|
|
1134
|
+
);
|
|
1135
|
+
|
|
1136
|
+
if (!recordExistsOk || !recordExists) {
|
|
1137
|
+
// Record was deleted - ignore applied transactions and start fresh
|
|
1138
|
+
// This prevents old values from being carried over after deletion
|
|
1139
|
+
if (this.config.verbose) {
|
|
1140
|
+
console.log(
|
|
1141
|
+
`[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
|
|
1142
|
+
`Record ${originalId} doesn't exist, deleting ${appliedTransactions.length} old applied transactions`
|
|
1143
|
+
);
|
|
1144
|
+
}
|
|
1145
|
+
|
|
1146
|
+
// Delete old applied transactions to prevent them from being used when record is recreated
|
|
1147
|
+
const { results, errors } = await PromisePool
|
|
1148
|
+
.for(appliedTransactions)
|
|
1149
|
+
.withConcurrency(10)
|
|
1150
|
+
.process(async (txn) => {
|
|
1151
|
+
const [deleted] = await tryFn(() => this.transactionResource.delete(txn.id));
|
|
1152
|
+
return deleted;
|
|
1153
|
+
});
|
|
1154
|
+
|
|
1155
|
+
if (this.config.verbose && errors && errors.length > 0) {
|
|
1156
|
+
console.warn(
|
|
1157
|
+
`[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
|
|
1158
|
+
`Failed to delete ${errors.length} old applied transactions`
|
|
1159
|
+
);
|
|
1160
|
+
}
|
|
1161
|
+
|
|
1162
|
+
currentValue = 0;
|
|
1163
|
+
} else {
|
|
1164
|
+
// Record exists - use applied transactions to calculate current value
|
|
1165
|
+
// Sort by timestamp to get chronological order
|
|
1166
|
+
appliedTransactions.sort((a, b) =>
|
|
1167
|
+
new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()
|
|
1168
|
+
);
|
|
1169
|
+
|
|
1170
|
+
// Check if there's a 'set' operation in applied transactions
|
|
1171
|
+
const hasSetInApplied = appliedTransactions.some(t => t.operation === 'set');
|
|
1172
|
+
|
|
1173
|
+
if (!hasSetInApplied) {
|
|
1174
|
+
// No 'set' operation in applied transactions means we're missing the base value
|
|
1175
|
+
// This can only happen if:
|
|
1176
|
+
// 1. Record had an initial value before first transaction
|
|
1177
|
+
// 2. First consolidation didn't create an anchor transaction (legacy behavior)
|
|
1178
|
+
// Solution: Get the current record value and create an anchor transaction now
|
|
1179
|
+
const recordValue = recordExists[this.config.field] || 0;
|
|
1180
|
+
|
|
1181
|
+
// Calculate what the base value was by subtracting all applied deltas
|
|
1182
|
+
let appliedDelta = 0;
|
|
1183
|
+
for (const t of appliedTransactions) {
|
|
1184
|
+
if (t.operation === 'add') appliedDelta += t.value;
|
|
1185
|
+
else if (t.operation === 'sub') appliedDelta -= t.value;
|
|
1186
|
+
}
|
|
1187
|
+
|
|
1188
|
+
const baseValue = recordValue - appliedDelta;
|
|
1189
|
+
|
|
1190
|
+
// Create and save anchor transaction with the base value
|
|
1191
|
+
// Only create if baseValue is non-zero AND we don't already have an anchor transaction
|
|
1192
|
+
const hasExistingAnchor = appliedTransactions.some(t => t.source === 'anchor');
|
|
1193
|
+
if (baseValue !== 0 && !hasExistingAnchor) {
|
|
1194
|
+
// Use the timestamp of the first applied transaction for cohort info
|
|
1195
|
+
const firstTransactionDate = new Date(appliedTransactions[0].timestamp);
|
|
1196
|
+
const cohortInfo = this.getCohortInfo(firstTransactionDate);
|
|
1197
|
+
const anchorTransaction = {
|
|
1198
|
+
id: idGenerator(),
|
|
1199
|
+
originalId: originalId,
|
|
1200
|
+
field: this.config.field,
|
|
1201
|
+
value: baseValue,
|
|
1202
|
+
operation: 'set',
|
|
1203
|
+
timestamp: new Date(firstTransactionDate.getTime() - 1).toISOString(), // 1ms before first txn to ensure it's first
|
|
1204
|
+
cohortDate: cohortInfo.date,
|
|
1205
|
+
cohortHour: cohortInfo.hour,
|
|
1206
|
+
cohortMonth: cohortInfo.month,
|
|
1207
|
+
source: 'anchor',
|
|
1208
|
+
applied: true
|
|
1209
|
+
};
|
|
1210
|
+
|
|
1211
|
+
await this.transactionResource.insert(anchorTransaction);
|
|
1212
|
+
|
|
1213
|
+
// Prepend to applied transactions for this consolidation
|
|
1214
|
+
appliedTransactions.unshift(anchorTransaction);
|
|
1215
|
+
}
|
|
1216
|
+
}
|
|
1217
|
+
|
|
1218
|
+
// Apply reducer to get the last consolidated value
|
|
1219
|
+
currentValue = this.config.reducer(appliedTransactions);
|
|
1220
|
+
}
|
|
1221
|
+
} else {
|
|
1222
|
+
// No applied transactions - this is the FIRST consolidation
|
|
1223
|
+
// Try to get initial value from record
|
|
1224
|
+
const [recordOk, recordErr, record] = await tryFn(() =>
|
|
1225
|
+
this.targetResource.get(originalId)
|
|
1226
|
+
);
|
|
1227
|
+
currentValue = (recordOk && record) ? (record[this.config.field] || 0) : 0;
|
|
1228
|
+
|
|
1229
|
+
// If there's an initial value, create and save an anchor transaction
|
|
1230
|
+
// This ensures all future consolidations have a reliable base value
|
|
1231
|
+
if (currentValue !== 0) {
|
|
1232
|
+
// Use timestamp of the first pending transaction (or current time if none)
|
|
1233
|
+
let anchorTimestamp;
|
|
1234
|
+
if (transactions && transactions.length > 0) {
|
|
1235
|
+
const firstPendingDate = new Date(transactions[0].timestamp);
|
|
1236
|
+
anchorTimestamp = new Date(firstPendingDate.getTime() - 1).toISOString();
|
|
1237
|
+
} else {
|
|
1238
|
+
anchorTimestamp = new Date().toISOString();
|
|
1239
|
+
}
|
|
1240
|
+
|
|
1241
|
+
const cohortInfo = this.getCohortInfo(new Date(anchorTimestamp));
|
|
1242
|
+
const anchorTransaction = {
|
|
1243
|
+
id: idGenerator(),
|
|
1244
|
+
originalId: originalId,
|
|
1245
|
+
field: this.config.field,
|
|
1246
|
+
value: currentValue,
|
|
1247
|
+
operation: 'set',
|
|
1248
|
+
timestamp: anchorTimestamp,
|
|
1249
|
+
cohortDate: cohortInfo.date,
|
|
1250
|
+
cohortHour: cohortInfo.hour,
|
|
1251
|
+
cohortMonth: cohortInfo.month,
|
|
1252
|
+
source: 'anchor',
|
|
1253
|
+
applied: true
|
|
1254
|
+
};
|
|
1255
|
+
|
|
1256
|
+
await this.transactionResource.insert(anchorTransaction);
|
|
1257
|
+
|
|
1258
|
+
if (this.config.verbose) {
|
|
1259
|
+
console.log(
|
|
1260
|
+
`[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
|
|
1261
|
+
`Created anchor transaction for ${originalId} with base value ${currentValue}`
|
|
1262
|
+
);
|
|
1263
|
+
}
|
|
1264
|
+
}
|
|
1265
|
+
}
|
|
1266
|
+
|
|
913
1267
|
if (this.config.verbose) {
|
|
914
1268
|
console.log(
|
|
915
1269
|
`[EventualConsistency] ${this.config.resource}.${this.config.field} - ` +
|
|
916
1270
|
`Consolidating ${originalId}: ${transactions.length} pending transactions ` +
|
|
917
|
-
`(current: ${currentValue})`
|
|
1271
|
+
`(current: ${currentValue} from ${appliedOk && appliedTransactions?.length > 0 ? 'applied transactions' : 'record'})`
|
|
918
1272
|
);
|
|
919
1273
|
}
|
|
920
1274
|
|
|
921
|
-
// Sort transactions by timestamp
|
|
1275
|
+
// Sort pending transactions by timestamp
|
|
922
1276
|
transactions.sort((a, b) =>
|
|
923
1277
|
new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()
|
|
924
1278
|
);
|
|
925
1279
|
|
|
926
|
-
// If there's a current value and no 'set' operations
|
|
1280
|
+
// If there's a current value and no 'set' operations in pending transactions,
|
|
1281
|
+
// prepend a synthetic set transaction to preserve the current value
|
|
927
1282
|
const hasSetOperation = transactions.some(t => t.operation === 'set');
|
|
928
1283
|
if (currentValue !== 0 && !hasSetOperation) {
|
|
929
1284
|
transactions.unshift(this._createSyntheticSetTransaction(currentValue));
|
|
@@ -1035,7 +1390,7 @@ export class EventualConsistencyPlugin extends Plugin {
|
|
|
1035
1390
|
}
|
|
1036
1391
|
|
|
1037
1392
|
// Invalidate cache for this record after consolidation
|
|
1038
|
-
if (this.targetResource.cache && typeof this.targetResource.cache.delete === 'function') {
|
|
1393
|
+
if (this.targetResource && this.targetResource.cache && typeof this.targetResource.cache.delete === 'function') {
|
|
1039
1394
|
try {
|
|
1040
1395
|
const cacheKey = await this.targetResource.cacheKeyFor({ id: originalId });
|
|
1041
1396
|
await this.targetResource.cache.delete(cacheKey);
|
|
@@ -1244,6 +1599,40 @@ export class EventualConsistencyPlugin extends Plugin {
|
|
|
1244
1599
|
}, gcIntervalMs);
|
|
1245
1600
|
}
|
|
1246
1601
|
|
|
1602
|
+
startGarbageCollectionTimerForHandler(handler, resourceName, fieldName) {
|
|
1603
|
+
const gcIntervalMs = this.config.gcInterval * 1000; // Convert seconds to ms
|
|
1604
|
+
|
|
1605
|
+
handler.gcTimer = setInterval(async () => {
|
|
1606
|
+
await this.runGarbageCollectionForHandler(handler, resourceName, fieldName);
|
|
1607
|
+
}, gcIntervalMs);
|
|
1608
|
+
}
|
|
1609
|
+
|
|
1610
|
+
async runGarbageCollectionForHandler(handler, resourceName, fieldName) {
|
|
1611
|
+
// Temporarily swap config to use this handler
|
|
1612
|
+
const oldResource = this.config.resource;
|
|
1613
|
+
const oldField = this.config.field;
|
|
1614
|
+
const oldTransactionResource = this.transactionResource;
|
|
1615
|
+
const oldTargetResource = this.targetResource;
|
|
1616
|
+
const oldLockResource = this.lockResource;
|
|
1617
|
+
|
|
1618
|
+
this.config.resource = resourceName;
|
|
1619
|
+
this.config.field = fieldName;
|
|
1620
|
+
this.transactionResource = handler.transactionResource;
|
|
1621
|
+
this.targetResource = handler.targetResource;
|
|
1622
|
+
this.lockResource = handler.lockResource;
|
|
1623
|
+
|
|
1624
|
+
try {
|
|
1625
|
+
await this.runGarbageCollection();
|
|
1626
|
+
} finally {
|
|
1627
|
+
// Restore
|
|
1628
|
+
this.config.resource = oldResource;
|
|
1629
|
+
this.config.field = oldField;
|
|
1630
|
+
this.transactionResource = oldTransactionResource;
|
|
1631
|
+
this.targetResource = oldTargetResource;
|
|
1632
|
+
this.lockResource = oldLockResource;
|
|
1633
|
+
}
|
|
1634
|
+
}
|
|
1635
|
+
|
|
1247
1636
|
/**
|
|
1248
1637
|
* Delete old applied transactions based on retention policy
|
|
1249
1638
|
* Uses distributed locking to prevent multiple containers from running GC simultaneously
|
|
@@ -1634,14 +2023,25 @@ export class EventualConsistencyPlugin extends Plugin {
|
|
|
1634
2023
|
* @returns {Promise<Array>} Analytics data
|
|
1635
2024
|
*/
|
|
1636
2025
|
async getAnalytics(resourceName, field, options = {}) {
|
|
1637
|
-
|
|
2026
|
+
// Get handler for this resource/field combination
|
|
2027
|
+
const fieldHandlers = this.fieldHandlers.get(resourceName);
|
|
2028
|
+
if (!fieldHandlers) {
|
|
2029
|
+
throw new Error(`No eventual consistency configured for resource: ${resourceName}`);
|
|
2030
|
+
}
|
|
2031
|
+
|
|
2032
|
+
const handler = fieldHandlers.get(field);
|
|
2033
|
+
if (!handler) {
|
|
2034
|
+
throw new Error(`No eventual consistency configured for field: ${resourceName}.${field}`);
|
|
2035
|
+
}
|
|
2036
|
+
|
|
2037
|
+
if (!handler.analyticsResource) {
|
|
1638
2038
|
throw new Error('Analytics not enabled for this plugin');
|
|
1639
2039
|
}
|
|
1640
2040
|
|
|
1641
2041
|
const { period = 'day', date, startDate, endDate, month, year, breakdown = false } = options;
|
|
1642
2042
|
|
|
1643
2043
|
const [ok, err, allAnalytics] = await tryFn(() =>
|
|
1644
|
-
|
|
2044
|
+
handler.analyticsResource.list()
|
|
1645
2045
|
);
|
|
1646
2046
|
|
|
1647
2047
|
if (!ok || !allAnalytics) {
|
|
@@ -1921,7 +2321,18 @@ export class EventualConsistencyPlugin extends Plugin {
|
|
|
1921
2321
|
* @returns {Promise<Array>} Top records
|
|
1922
2322
|
*/
|
|
1923
2323
|
async getTopRecords(resourceName, field, options = {}) {
|
|
1924
|
-
|
|
2324
|
+
// Get handler for this resource/field combination
|
|
2325
|
+
const fieldHandlers = this.fieldHandlers.get(resourceName);
|
|
2326
|
+
if (!fieldHandlers) {
|
|
2327
|
+
throw new Error(`No eventual consistency configured for resource: ${resourceName}`);
|
|
2328
|
+
}
|
|
2329
|
+
|
|
2330
|
+
const handler = fieldHandlers.get(field);
|
|
2331
|
+
if (!handler) {
|
|
2332
|
+
throw new Error(`No eventual consistency configured for field: ${resourceName}.${field}`);
|
|
2333
|
+
}
|
|
2334
|
+
|
|
2335
|
+
if (!handler.transactionResource) {
|
|
1925
2336
|
throw new Error('Transaction resource not initialized');
|
|
1926
2337
|
}
|
|
1927
2338
|
|
|
@@ -1929,7 +2340,7 @@ export class EventualConsistencyPlugin extends Plugin {
|
|
|
1929
2340
|
|
|
1930
2341
|
// Get all transactions for the period
|
|
1931
2342
|
const [ok, err, transactions] = await tryFn(() =>
|
|
1932
|
-
|
|
2343
|
+
handler.transactionResource.list()
|
|
1933
2344
|
);
|
|
1934
2345
|
|
|
1935
2346
|
if (!ok || !transactions) {
|