s3db.js 11.0.5 → 11.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +59 -2
- package/SECURITY.md +76 -0
- package/dist/s3db.cjs.js +446 -86
- package/dist/s3db.cjs.js.map +1 -1
- package/dist/s3db.es.js +446 -86
- package/dist/s3db.es.js.map +1 -1
- package/package.json +3 -1
- package/src/concerns/crypto.js +7 -14
- package/src/plugins/eventual-consistency/analytics.js +164 -2
- package/src/plugins/eventual-consistency/consolidation.js +228 -80
- package/src/plugins/eventual-consistency/helpers.js +24 -8
- package/src/plugins/eventual-consistency/install.js +2 -1
- package/src/plugins/eventual-consistency/utils.js +218 -4
- package/src/concerns/advanced-metadata-encoding.js +0 -440
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "s3db.js",
|
|
3
|
-
"version": "11.0
|
|
3
|
+
"version": "11.2.0",
|
|
4
4
|
"description": "Use AWS S3, the world's most reliable document storage, as a database with this ORM.",
|
|
5
5
|
"main": "dist/s3db.cjs.js",
|
|
6
6
|
"module": "dist/s3db.es.js",
|
|
@@ -55,6 +55,7 @@
|
|
|
55
55
|
"mcp/server.js",
|
|
56
56
|
"README.md",
|
|
57
57
|
"PLUGINS.md",
|
|
58
|
+
"SECURITY.md",
|
|
58
59
|
"UNLICENSE"
|
|
59
60
|
],
|
|
60
61
|
"dependencies": {
|
|
@@ -140,6 +141,7 @@
|
|
|
140
141
|
"test:plugins": "node --no-warnings --experimental-vm-modules node_modules/jest/bin/jest.js tests/plugins/ --testTimeout=60000",
|
|
141
142
|
"test:full": "pnpm run test:js && pnpm run test:ts",
|
|
142
143
|
"benchmark": "node benchmark-compression.js",
|
|
144
|
+
"benchmark:partitions": "node docs/benchmarks/partitions-matrix.js",
|
|
143
145
|
"version": "echo 'Use pnpm run release v<version> instead of npm version'",
|
|
144
146
|
"release:check": "./scripts/pre-release-check.sh",
|
|
145
147
|
"release:prepare": "pnpm run build:binaries && echo 'Binaries ready for GitHub release'",
|
package/src/concerns/crypto.js
CHANGED
|
@@ -1,19 +1,13 @@
|
|
|
1
1
|
import { CryptoError } from "../errors.js";
|
|
2
2
|
import tryFn, { tryFnSync } from "./try-fn.js";
|
|
3
|
+
import crypto from 'crypto';
|
|
3
4
|
|
|
4
5
|
async function dynamicCrypto() {
|
|
5
6
|
let lib;
|
|
6
7
|
|
|
7
8
|
if (typeof process !== 'undefined') {
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
return webcrypto;
|
|
11
|
-
});
|
|
12
|
-
if (ok) {
|
|
13
|
-
lib = result;
|
|
14
|
-
} else {
|
|
15
|
-
throw new CryptoError('Crypto API not available', { original: err, context: 'dynamicCrypto' });
|
|
16
|
-
}
|
|
9
|
+
// Use the static import instead of dynamic import
|
|
10
|
+
lib = crypto.webcrypto;
|
|
17
11
|
} else if (typeof window !== 'undefined') {
|
|
18
12
|
lib = window.crypto;
|
|
19
13
|
}
|
|
@@ -86,16 +80,15 @@ export async function md5(data) {
|
|
|
86
80
|
if (typeof process === 'undefined') {
|
|
87
81
|
throw new CryptoError('MD5 hashing is only available in Node.js environment', { context: 'md5' });
|
|
88
82
|
}
|
|
89
|
-
|
|
83
|
+
|
|
90
84
|
const [ok, err, result] = await tryFn(async () => {
|
|
91
|
-
|
|
92
|
-
return createHash('md5').update(data).digest('base64');
|
|
85
|
+
return crypto.createHash('md5').update(data).digest('base64');
|
|
93
86
|
});
|
|
94
|
-
|
|
87
|
+
|
|
95
88
|
if (!ok) {
|
|
96
89
|
throw new CryptoError('MD5 hashing failed', { original: err, data });
|
|
97
90
|
}
|
|
98
|
-
|
|
91
|
+
|
|
99
92
|
return result;
|
|
100
93
|
}
|
|
101
94
|
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
*/
|
|
5
5
|
|
|
6
6
|
import tryFn from "../../concerns/try-fn.js";
|
|
7
|
-
import { groupByCohort } from "./utils.js";
|
|
7
|
+
import { groupByCohort, ensureCohortHours } from "./utils.js";
|
|
8
8
|
|
|
9
9
|
/**
|
|
10
10
|
* Update analytics with consolidated transactions
|
|
@@ -459,8 +459,14 @@ export async function getAnalytics(resourceName, field, options, fieldHandlers)
|
|
|
459
459
|
throw new Error('Analytics not enabled for this plugin');
|
|
460
460
|
}
|
|
461
461
|
|
|
462
|
-
const { period = 'day', date, startDate, endDate, month, year, breakdown = false } = options;
|
|
462
|
+
const { period = 'day', date, startDate, endDate, month, year, breakdown = false, recordId } = options;
|
|
463
463
|
|
|
464
|
+
// ✅ FIX BUG #1: If recordId is specified, fetch from transactions directly
|
|
465
|
+
if (recordId) {
|
|
466
|
+
return await getAnalyticsForRecord(resourceName, field, recordId, options, handler);
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
// Original behavior: global analytics from pre-aggregated data
|
|
464
470
|
const [ok, err, allAnalytics] = await tryFn(() =>
|
|
465
471
|
handler.analyticsResource.list()
|
|
466
472
|
);
|
|
@@ -511,6 +517,156 @@ export async function getAnalytics(resourceName, field, options, fieldHandlers)
|
|
|
511
517
|
}));
|
|
512
518
|
}
|
|
513
519
|
|
|
520
|
+
/**
|
|
521
|
+
* Get analytics for a specific record from transactions
|
|
522
|
+
* ✅ FIX BUG #1: Filter by recordId
|
|
523
|
+
*
|
|
524
|
+
* @param {string} resourceName - Resource name
|
|
525
|
+
* @param {string} field - Field name
|
|
526
|
+
* @param {string} recordId - Record ID to filter by
|
|
527
|
+
* @param {Object} options - Query options
|
|
528
|
+
* @param {Object} handler - Field handler
|
|
529
|
+
* @returns {Promise<Array>} Analytics data for specific record
|
|
530
|
+
* @private
|
|
531
|
+
*/
|
|
532
|
+
async function getAnalyticsForRecord(resourceName, field, recordId, options, handler) {
|
|
533
|
+
const { period = 'day', date, startDate, endDate, month, year } = options;
|
|
534
|
+
|
|
535
|
+
// Fetch transactions for this specific record
|
|
536
|
+
// Note: We need both applied: true and applied: false transactions for analytics
|
|
537
|
+
// Same format as used in consolidation.js - the query() method auto-selects the partition
|
|
538
|
+
const [okTrue, errTrue, appliedTransactions] = await tryFn(() =>
|
|
539
|
+
handler.transactionResource.query({
|
|
540
|
+
originalId: recordId,
|
|
541
|
+
applied: true
|
|
542
|
+
})
|
|
543
|
+
);
|
|
544
|
+
|
|
545
|
+
const [okFalse, errFalse, pendingTransactions] = await tryFn(() =>
|
|
546
|
+
handler.transactionResource.query({
|
|
547
|
+
originalId: recordId,
|
|
548
|
+
applied: false
|
|
549
|
+
})
|
|
550
|
+
);
|
|
551
|
+
|
|
552
|
+
// Combine both applied and pending transactions
|
|
553
|
+
let allTransactions = [
|
|
554
|
+
...(okTrue && appliedTransactions ? appliedTransactions : []),
|
|
555
|
+
...(okFalse && pendingTransactions ? pendingTransactions : [])
|
|
556
|
+
];
|
|
557
|
+
|
|
558
|
+
if (allTransactions.length === 0) {
|
|
559
|
+
return [];
|
|
560
|
+
}
|
|
561
|
+
|
|
562
|
+
// ✅ FIX BUG #2: Ensure all transactions have cohortHour calculated
|
|
563
|
+
// This handles legacy data that may be missing cohortHour
|
|
564
|
+
allTransactions = ensureCohortHours(allTransactions, handler.config?.cohort?.timezone || 'UTC', false);
|
|
565
|
+
|
|
566
|
+
// Filter transactions by temporal range
|
|
567
|
+
let filtered = allTransactions;
|
|
568
|
+
|
|
569
|
+
if (date) {
|
|
570
|
+
if (period === 'hour') {
|
|
571
|
+
// Match all hours of the date
|
|
572
|
+
filtered = filtered.filter(t => t.cohortHour && t.cohortHour.startsWith(date));
|
|
573
|
+
} else if (period === 'day') {
|
|
574
|
+
filtered = filtered.filter(t => t.cohortDate === date);
|
|
575
|
+
} else if (period === 'month') {
|
|
576
|
+
filtered = filtered.filter(t => t.cohortMonth && t.cohortMonth.startsWith(date));
|
|
577
|
+
}
|
|
578
|
+
} else if (startDate && endDate) {
|
|
579
|
+
if (period === 'hour') {
|
|
580
|
+
filtered = filtered.filter(t => t.cohortHour && t.cohortHour >= startDate && t.cohortHour <= endDate);
|
|
581
|
+
} else if (period === 'day') {
|
|
582
|
+
filtered = filtered.filter(t => t.cohortDate && t.cohortDate >= startDate && t.cohortDate <= endDate);
|
|
583
|
+
} else if (period === 'month') {
|
|
584
|
+
filtered = filtered.filter(t => t.cohortMonth && t.cohortMonth >= startDate && t.cohortMonth <= endDate);
|
|
585
|
+
}
|
|
586
|
+
} else if (month) {
|
|
587
|
+
if (period === 'hour') {
|
|
588
|
+
filtered = filtered.filter(t => t.cohortHour && t.cohortHour.startsWith(month));
|
|
589
|
+
} else if (period === 'day') {
|
|
590
|
+
filtered = filtered.filter(t => t.cohortDate && t.cohortDate.startsWith(month));
|
|
591
|
+
}
|
|
592
|
+
} else if (year) {
|
|
593
|
+
if (period === 'hour') {
|
|
594
|
+
filtered = filtered.filter(t => t.cohortHour && t.cohortHour.startsWith(String(year)));
|
|
595
|
+
} else if (period === 'day') {
|
|
596
|
+
filtered = filtered.filter(t => t.cohortDate && t.cohortDate.startsWith(String(year)));
|
|
597
|
+
} else if (period === 'month') {
|
|
598
|
+
filtered = filtered.filter(t => t.cohortMonth && t.cohortMonth.startsWith(String(year)));
|
|
599
|
+
}
|
|
600
|
+
}
|
|
601
|
+
|
|
602
|
+
// Aggregate transactions by cohort
|
|
603
|
+
const cohortField = period === 'hour' ? 'cohortHour' : period === 'day' ? 'cohortDate' : 'cohortMonth';
|
|
604
|
+
const aggregated = aggregateTransactionsByCohort(filtered, cohortField);
|
|
605
|
+
|
|
606
|
+
return aggregated;
|
|
607
|
+
}
|
|
608
|
+
|
|
609
|
+
/**
|
|
610
|
+
* Aggregate transactions by cohort field
|
|
611
|
+
* ✅ Helper for BUG #1 fix
|
|
612
|
+
*
|
|
613
|
+
* @param {Array} transactions - Transactions to aggregate
|
|
614
|
+
* @param {string} cohortField - Cohort field name ('cohortHour', 'cohortDate', 'cohortMonth')
|
|
615
|
+
* @returns {Array} Aggregated analytics
|
|
616
|
+
* @private
|
|
617
|
+
*/
|
|
618
|
+
function aggregateTransactionsByCohort(transactions, cohortField) {
|
|
619
|
+
const groups = {};
|
|
620
|
+
|
|
621
|
+
for (const txn of transactions) {
|
|
622
|
+
const cohort = txn[cohortField];
|
|
623
|
+
if (!cohort) continue;
|
|
624
|
+
|
|
625
|
+
if (!groups[cohort]) {
|
|
626
|
+
groups[cohort] = {
|
|
627
|
+
cohort,
|
|
628
|
+
count: 0,
|
|
629
|
+
sum: 0,
|
|
630
|
+
min: Infinity,
|
|
631
|
+
max: -Infinity,
|
|
632
|
+
recordCount: new Set(),
|
|
633
|
+
operations: {}
|
|
634
|
+
};
|
|
635
|
+
}
|
|
636
|
+
|
|
637
|
+
const group = groups[cohort];
|
|
638
|
+
const signedValue = txn.operation === 'sub' ? -txn.value : txn.value;
|
|
639
|
+
|
|
640
|
+
group.count++;
|
|
641
|
+
group.sum += signedValue;
|
|
642
|
+
group.min = Math.min(group.min, signedValue);
|
|
643
|
+
group.max = Math.max(group.max, signedValue);
|
|
644
|
+
group.recordCount.add(txn.originalId);
|
|
645
|
+
|
|
646
|
+
// Track operation breakdown
|
|
647
|
+
const op = txn.operation;
|
|
648
|
+
if (!group.operations[op]) {
|
|
649
|
+
group.operations[op] = { count: 0, sum: 0 };
|
|
650
|
+
}
|
|
651
|
+
group.operations[op].count++;
|
|
652
|
+
group.operations[op].sum += signedValue;
|
|
653
|
+
}
|
|
654
|
+
|
|
655
|
+
// Convert to array and finalize
|
|
656
|
+
return Object.values(groups)
|
|
657
|
+
.map(g => ({
|
|
658
|
+
cohort: g.cohort,
|
|
659
|
+
count: g.count,
|
|
660
|
+
sum: g.sum,
|
|
661
|
+
avg: g.sum / g.count,
|
|
662
|
+
min: g.min === Infinity ? 0 : g.min,
|
|
663
|
+
max: g.max === -Infinity ? 0 : g.max,
|
|
664
|
+
recordCount: g.recordCount.size,
|
|
665
|
+
operations: g.operations
|
|
666
|
+
}))
|
|
667
|
+
.sort((a, b) => a.cohort.localeCompare(b.cohort));
|
|
668
|
+
}
|
|
669
|
+
|
|
514
670
|
/**
|
|
515
671
|
* Get analytics for entire month, broken down by days
|
|
516
672
|
*
|
|
@@ -587,7 +743,9 @@ export async function getLastNDays(resourceName, field, days, options, fieldHand
|
|
|
587
743
|
return date.toISOString().substring(0, 10);
|
|
588
744
|
}).reverse();
|
|
589
745
|
|
|
746
|
+
// ✅ FIX BUG #1: Pass through recordId and other options
|
|
590
747
|
const data = await getAnalytics(resourceName, field, {
|
|
748
|
+
...options, // ✅ Include all options (recordId, etc.)
|
|
591
749
|
period: 'day',
|
|
592
750
|
startDate: dates[0],
|
|
593
751
|
endDate: dates[dates.length - 1]
|
|
@@ -942,7 +1100,9 @@ export async function getLastNHours(resourceName, field, hours = 24, options, fi
|
|
|
942
1100
|
const startHour = hoursAgo.toISOString().substring(0, 13); // YYYY-MM-DDTHH
|
|
943
1101
|
const endHour = now.toISOString().substring(0, 13);
|
|
944
1102
|
|
|
1103
|
+
// ✅ FIX BUG #1: Pass through recordId and other options
|
|
945
1104
|
const data = await getAnalytics(resourceName, field, {
|
|
1105
|
+
...options, // ✅ Include all options (recordId, etc.)
|
|
946
1106
|
period: 'hour',
|
|
947
1107
|
startDate: startHour,
|
|
948
1108
|
endDate: endHour
|
|
@@ -1023,7 +1183,9 @@ export async function getLastNMonths(resourceName, field, months = 12, options,
|
|
|
1023
1183
|
const startDate = monthsAgo.toISOString().substring(0, 7); // YYYY-MM
|
|
1024
1184
|
const endDate = now.toISOString().substring(0, 7);
|
|
1025
1185
|
|
|
1186
|
+
// ✅ FIX BUG #1: Pass through recordId and other options
|
|
1026
1187
|
const data = await getAnalytics(resourceName, field, {
|
|
1188
|
+
...options, // ✅ Include all options (recordId, etc.)
|
|
1027
1189
|
period: 'month',
|
|
1028
1190
|
startDate,
|
|
1029
1191
|
endDate
|
|
@@ -272,6 +272,8 @@ export async function consolidateRecord(
|
|
|
272
272
|
}
|
|
273
273
|
|
|
274
274
|
currentValue = 0;
|
|
275
|
+
// Clear the applied transactions array since we deleted them
|
|
276
|
+
appliedTransactions.length = 0;
|
|
275
277
|
} else {
|
|
276
278
|
// Record exists - use applied transactions to calculate current value
|
|
277
279
|
// Sort by timestamp to get chronological order
|
|
@@ -290,40 +292,44 @@ export async function consolidateRecord(
|
|
|
290
292
|
// Solution: Get the current record value and create an anchor transaction now
|
|
291
293
|
const recordValue = recordExists[config.field] || 0;
|
|
292
294
|
|
|
293
|
-
//
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
295
|
+
// Only create anchor if recordValue is a number (not object/array for nested fields)
|
|
296
|
+
if (typeof recordValue === 'number') {
|
|
297
|
+
// Calculate what the base value was by subtracting all applied deltas
|
|
298
|
+
let appliedDelta = 0;
|
|
299
|
+
for (const t of appliedTransactions) {
|
|
300
|
+
if (t.operation === 'add') appliedDelta += t.value;
|
|
301
|
+
else if (t.operation === 'sub') appliedDelta -= t.value;
|
|
302
|
+
}
|
|
299
303
|
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
304
|
+
const baseValue = recordValue - appliedDelta;
|
|
305
|
+
|
|
306
|
+
// Create and save anchor transaction with the base value
|
|
307
|
+
// Only create if baseValue is non-zero AND we don't already have an anchor transaction
|
|
308
|
+
const hasExistingAnchor = appliedTransactions.some(t => t.source === 'anchor');
|
|
309
|
+
if (baseValue !== 0 && typeof baseValue === 'number' && !hasExistingAnchor) {
|
|
310
|
+
// Use the timestamp of the first applied transaction for cohort info
|
|
311
|
+
const firstTransactionDate = new Date(appliedTransactions[0].timestamp);
|
|
312
|
+
const cohortInfo = getCohortInfo(firstTransactionDate, config.cohort.timezone, config.verbose);
|
|
313
|
+
const anchorTransaction = {
|
|
314
|
+
id: idGenerator(),
|
|
315
|
+
originalId: originalId,
|
|
316
|
+
field: config.field,
|
|
317
|
+
fieldPath: config.field, // Add fieldPath for consistency
|
|
318
|
+
value: baseValue,
|
|
319
|
+
operation: 'set',
|
|
320
|
+
timestamp: new Date(firstTransactionDate.getTime() - 1).toISOString(), // 1ms before first txn to ensure it's first
|
|
321
|
+
cohortDate: cohortInfo.date,
|
|
322
|
+
cohortHour: cohortInfo.hour,
|
|
323
|
+
cohortMonth: cohortInfo.month,
|
|
324
|
+
source: 'anchor',
|
|
325
|
+
applied: true
|
|
326
|
+
};
|
|
327
|
+
|
|
328
|
+
await transactionResource.insert(anchorTransaction);
|
|
329
|
+
|
|
330
|
+
// Prepend to applied transactions for this consolidation
|
|
331
|
+
appliedTransactions.unshift(anchorTransaction);
|
|
332
|
+
}
|
|
327
333
|
}
|
|
328
334
|
}
|
|
329
335
|
|
|
@@ -340,7 +346,8 @@ export async function consolidateRecord(
|
|
|
340
346
|
|
|
341
347
|
// If there's an initial value, create and save an anchor transaction
|
|
342
348
|
// This ensures all future consolidations have a reliable base value
|
|
343
|
-
if
|
|
349
|
+
// IMPORTANT: Only create anchor if currentValue is a number (not object/array for nested fields)
|
|
350
|
+
if (currentValue !== 0 && typeof currentValue === 'number') {
|
|
344
351
|
// Use timestamp of the first pending transaction (or current time if none)
|
|
345
352
|
let anchorTimestamp;
|
|
346
353
|
if (transactions && transactions.length > 0) {
|
|
@@ -355,6 +362,7 @@ export async function consolidateRecord(
|
|
|
355
362
|
id: idGenerator(),
|
|
356
363
|
originalId: originalId,
|
|
357
364
|
field: config.field,
|
|
365
|
+
fieldPath: config.field, // Add fieldPath for consistency
|
|
358
366
|
value: currentValue,
|
|
359
367
|
operation: 'set',
|
|
360
368
|
timestamp: anchorTimestamp,
|
|
@@ -389,22 +397,75 @@ export async function consolidateRecord(
|
|
|
389
397
|
new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()
|
|
390
398
|
);
|
|
391
399
|
|
|
392
|
-
//
|
|
393
|
-
|
|
394
|
-
const
|
|
395
|
-
|
|
396
|
-
|
|
400
|
+
// Group PENDING transactions by fieldPath to support nested fields
|
|
401
|
+
const transactionsByPath = {};
|
|
402
|
+
for (const txn of transactions) {
|
|
403
|
+
const path = txn.fieldPath || txn.field || config.field;
|
|
404
|
+
if (!transactionsByPath[path]) {
|
|
405
|
+
transactionsByPath[path] = [];
|
|
406
|
+
}
|
|
407
|
+
transactionsByPath[path].push(txn);
|
|
397
408
|
}
|
|
398
409
|
|
|
399
|
-
//
|
|
400
|
-
|
|
410
|
+
// For each fieldPath, we need the currentValue from applied transactions
|
|
411
|
+
// Group APPLIED transactions by fieldPath
|
|
412
|
+
const appliedByPath = {};
|
|
413
|
+
if (appliedOk && appliedTransactions && appliedTransactions.length > 0) {
|
|
414
|
+
for (const txn of appliedTransactions) {
|
|
415
|
+
const path = txn.fieldPath || txn.field || config.field;
|
|
416
|
+
if (!appliedByPath[path]) {
|
|
417
|
+
appliedByPath[path] = [];
|
|
418
|
+
}
|
|
419
|
+
appliedByPath[path].push(txn);
|
|
420
|
+
}
|
|
421
|
+
}
|
|
401
422
|
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
423
|
+
// Consolidate each fieldPath group separately
|
|
424
|
+
const consolidatedValues = {};
|
|
425
|
+
const lodash = await import('lodash-es');
|
|
426
|
+
|
|
427
|
+
// Get current record to extract existing values for nested paths
|
|
428
|
+
const [currentRecordOk, currentRecordErr, currentRecord] = await tryFn(() =>
|
|
429
|
+
targetResource.get(originalId)
|
|
430
|
+
);
|
|
431
|
+
|
|
432
|
+
for (const [fieldPath, pathTransactions] of Object.entries(transactionsByPath)) {
|
|
433
|
+
// Calculate current value for this path from applied transactions
|
|
434
|
+
let pathCurrentValue = 0;
|
|
435
|
+
if (appliedByPath[fieldPath] && appliedByPath[fieldPath].length > 0) {
|
|
436
|
+
// Sort applied transactions by timestamp
|
|
437
|
+
appliedByPath[fieldPath].sort((a, b) =>
|
|
438
|
+
new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()
|
|
439
|
+
);
|
|
440
|
+
// Apply reducer to get current value from applied transactions
|
|
441
|
+
pathCurrentValue = config.reducer(appliedByPath[fieldPath]);
|
|
442
|
+
} else {
|
|
443
|
+
// No applied transactions yet - use value from record (first consolidation)
|
|
444
|
+
// This happens when there's an initial value in the record before any consolidation
|
|
445
|
+
if (currentRecordOk && currentRecord) {
|
|
446
|
+
const recordValue = lodash.get(currentRecord, fieldPath, 0);
|
|
447
|
+
if (typeof recordValue === 'number') {
|
|
448
|
+
pathCurrentValue = recordValue;
|
|
449
|
+
}
|
|
450
|
+
}
|
|
451
|
+
}
|
|
452
|
+
|
|
453
|
+
// Prepend synthetic set transaction with current value
|
|
454
|
+
if (pathCurrentValue !== 0) {
|
|
455
|
+
pathTransactions.unshift(createSyntheticSetTransaction(pathCurrentValue));
|
|
456
|
+
}
|
|
457
|
+
|
|
458
|
+
// Apply reducer to get consolidated value for this path
|
|
459
|
+
const pathConsolidatedValue = config.reducer(pathTransactions);
|
|
460
|
+
consolidatedValues[fieldPath] = pathConsolidatedValue;
|
|
461
|
+
|
|
462
|
+
if (config.verbose) {
|
|
463
|
+
console.log(
|
|
464
|
+
`[EventualConsistency] ${config.resource}.${fieldPath} - ` +
|
|
465
|
+
`${originalId}: ${pathCurrentValue} → ${pathConsolidatedValue} ` +
|
|
466
|
+
`(${pathTransactions.length - (pathCurrentValue !== 0 ? 1 : 0)} pending txns)`
|
|
467
|
+
);
|
|
468
|
+
}
|
|
408
469
|
}
|
|
409
470
|
|
|
410
471
|
// 🔥 DEBUG: Log BEFORE update
|
|
@@ -412,63 +473,105 @@ export async function consolidateRecord(
|
|
|
412
473
|
console.log(
|
|
413
474
|
`🔥 [DEBUG] BEFORE targetResource.update() {` +
|
|
414
475
|
`\n originalId: '${originalId}',` +
|
|
415
|
-
`\n
|
|
416
|
-
`\n consolidatedValue: ${consolidatedValue},` +
|
|
417
|
-
`\n currentValue: ${currentValue}` +
|
|
476
|
+
`\n consolidatedValues: ${JSON.stringify(consolidatedValues, null, 2)}` +
|
|
418
477
|
`\n}`
|
|
419
478
|
);
|
|
420
479
|
}
|
|
421
480
|
|
|
422
|
-
//
|
|
423
|
-
//
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
// 3. Transactions will remain pending until the record is created
|
|
427
|
-
const [updateOk, updateErr, updateResult] = await tryFn(() =>
|
|
428
|
-
targetResource.update(originalId, {
|
|
429
|
-
[config.field]: consolidatedValue
|
|
430
|
-
})
|
|
481
|
+
// Build update object using lodash.set for nested paths
|
|
482
|
+
// Get fresh record to avoid overwriting other fields
|
|
483
|
+
const [recordOk, recordErr, record] = await tryFn(() =>
|
|
484
|
+
targetResource.get(originalId)
|
|
431
485
|
);
|
|
432
486
|
|
|
487
|
+
let updateOk, updateErr, updateResult;
|
|
488
|
+
|
|
489
|
+
if (!recordOk || !record) {
|
|
490
|
+
// Record doesn't exist - we'll let the update fail and handle it below
|
|
491
|
+
// This ensures transactions remain pending until record is created
|
|
492
|
+
if (config.verbose) {
|
|
493
|
+
console.log(
|
|
494
|
+
`[EventualConsistency] ${config.resource}.${config.field} - ` +
|
|
495
|
+
`Record ${originalId} doesn't exist yet. Will attempt update anyway (expected to fail).`
|
|
496
|
+
);
|
|
497
|
+
}
|
|
498
|
+
|
|
499
|
+
// Create a minimal record object with just our field
|
|
500
|
+
const minimalRecord = { id: originalId };
|
|
501
|
+
for (const [fieldPath, value] of Object.entries(consolidatedValues)) {
|
|
502
|
+
lodash.set(minimalRecord, fieldPath, value);
|
|
503
|
+
}
|
|
504
|
+
|
|
505
|
+
// Try to update (will fail, handled below)
|
|
506
|
+
const result = await tryFn(() =>
|
|
507
|
+
targetResource.update(originalId, minimalRecord)
|
|
508
|
+
);
|
|
509
|
+
updateOk = result[0];
|
|
510
|
+
updateErr = result[1];
|
|
511
|
+
updateResult = result[2];
|
|
512
|
+
} else {
|
|
513
|
+
// Record exists - apply all consolidated values using lodash.set
|
|
514
|
+
for (const [fieldPath, value] of Object.entries(consolidatedValues)) {
|
|
515
|
+
lodash.set(record, fieldPath, value);
|
|
516
|
+
}
|
|
517
|
+
|
|
518
|
+
// Update the original record with all changes
|
|
519
|
+
// NOTE: We update the entire record to preserve nested structures
|
|
520
|
+
const result = await tryFn(() =>
|
|
521
|
+
targetResource.update(originalId, record)
|
|
522
|
+
);
|
|
523
|
+
updateOk = result[0];
|
|
524
|
+
updateErr = result[1];
|
|
525
|
+
updateResult = result[2];
|
|
526
|
+
}
|
|
527
|
+
|
|
528
|
+
// For backward compatibility, return the value of the main field
|
|
529
|
+
const consolidatedValue = consolidatedValues[config.field] ||
|
|
530
|
+
(record ? lodash.get(record, config.field, 0) : 0);
|
|
531
|
+
|
|
433
532
|
// 🔥 DEBUG: Log AFTER update
|
|
434
533
|
if (config.verbose) {
|
|
435
534
|
console.log(
|
|
436
535
|
`🔥 [DEBUG] AFTER targetResource.update() {` +
|
|
437
536
|
`\n updateOk: ${updateOk},` +
|
|
438
537
|
`\n updateErr: ${updateErr?.message || 'undefined'},` +
|
|
439
|
-
`\n
|
|
440
|
-
`\n hasField: ${updateResult?.[config.field]}` +
|
|
538
|
+
`\n consolidatedValue (main field): ${consolidatedValue}` +
|
|
441
539
|
`\n}`
|
|
442
540
|
);
|
|
443
541
|
}
|
|
444
542
|
|
|
445
|
-
// 🔥 VERIFY: Check if update actually persisted
|
|
543
|
+
// 🔥 VERIFY: Check if update actually persisted for all fieldPaths
|
|
446
544
|
if (updateOk && config.verbose) {
|
|
447
545
|
// Bypass cache to get fresh data
|
|
448
546
|
const [verifyOk, verifyErr, verifiedRecord] = await tryFn(() =>
|
|
449
547
|
targetResource.get(originalId, { skipCache: true })
|
|
450
548
|
);
|
|
451
549
|
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
`\n expectedValue: ${consolidatedValue},` +
|
|
457
|
-
`\n ✅ MATCH: ${verifiedRecord?.[config.field] === consolidatedValue}` +
|
|
458
|
-
`\n}`
|
|
459
|
-
);
|
|
550
|
+
// Verify each fieldPath
|
|
551
|
+
for (const [fieldPath, expectedValue] of Object.entries(consolidatedValues)) {
|
|
552
|
+
const actualValue = lodash.get(verifiedRecord, fieldPath);
|
|
553
|
+
const match = actualValue === expectedValue;
|
|
460
554
|
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
`\n
|
|
466
|
-
`\n
|
|
467
|
-
`\n Record ID: ${originalId}` +
|
|
468
|
-
`\n Expected: ${consolidatedValue}` +
|
|
469
|
-
`\n Actually got: ${verifiedRecord?.[config.field]}` +
|
|
470
|
-
`\n This indicates a bug in s3db.js resource.update()`
|
|
555
|
+
console.log(
|
|
556
|
+
`🔥 [DEBUG] VERIFICATION ${fieldPath} {` +
|
|
557
|
+
`\n expectedValue: ${expectedValue},` +
|
|
558
|
+
`\n actualValue: ${actualValue},` +
|
|
559
|
+
`\n ${match ? '✅ MATCH' : '❌ MISMATCH'}` +
|
|
560
|
+
`\n}`
|
|
471
561
|
);
|
|
562
|
+
|
|
563
|
+
// If verification fails, this is a critical bug
|
|
564
|
+
if (!match) {
|
|
565
|
+
console.error(
|
|
566
|
+
`❌ [CRITICAL BUG] Update reported success but value not persisted!` +
|
|
567
|
+
`\n Resource: ${config.resource}` +
|
|
568
|
+
`\n FieldPath: ${fieldPath}` +
|
|
569
|
+
`\n Record ID: ${originalId}` +
|
|
570
|
+
`\n Expected: ${expectedValue}` +
|
|
571
|
+
`\n Actually got: ${actualValue}` +
|
|
572
|
+
`\n This indicates a bug in s3db.js resource.update()`
|
|
573
|
+
);
|
|
574
|
+
}
|
|
472
575
|
}
|
|
473
576
|
}
|
|
474
577
|
|
|
@@ -765,6 +868,51 @@ export async function recalculateRecord(
|
|
|
765
868
|
);
|
|
766
869
|
}
|
|
767
870
|
|
|
871
|
+
// Check if there's an anchor transaction
|
|
872
|
+
const hasAnchor = allTransactions.some(txn => txn.source === 'anchor');
|
|
873
|
+
|
|
874
|
+
// If no anchor exists, create one with value 0 to serve as the baseline
|
|
875
|
+
// This ensures recalculate is idempotent - running it multiple times produces same result
|
|
876
|
+
if (!hasAnchor) {
|
|
877
|
+
const now = new Date();
|
|
878
|
+
const cohortInfo = getCohortInfo(now, config.cohort.timezone, config.verbose);
|
|
879
|
+
|
|
880
|
+
// Create anchor transaction with timestamp before all other transactions
|
|
881
|
+
const oldestTransaction = allTransactions.sort((a, b) =>
|
|
882
|
+
new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()
|
|
883
|
+
)[0];
|
|
884
|
+
|
|
885
|
+
const anchorTimestamp = oldestTransaction
|
|
886
|
+
? new Date(new Date(oldestTransaction.timestamp).getTime() - 1).toISOString()
|
|
887
|
+
: now.toISOString();
|
|
888
|
+
|
|
889
|
+
const anchorCohortInfo = getCohortInfo(new Date(anchorTimestamp), config.cohort.timezone, config.verbose);
|
|
890
|
+
|
|
891
|
+
const anchorTransaction = {
|
|
892
|
+
id: idGenerator(),
|
|
893
|
+
originalId: originalId,
|
|
894
|
+
field: config.field,
|
|
895
|
+
fieldPath: config.field,
|
|
896
|
+
value: 0, // Always 0 for recalculate - we start from scratch
|
|
897
|
+
operation: 'set',
|
|
898
|
+
timestamp: anchorTimestamp,
|
|
899
|
+
cohortDate: anchorCohortInfo.date,
|
|
900
|
+
cohortHour: anchorCohortInfo.hour,
|
|
901
|
+
cohortMonth: anchorCohortInfo.month,
|
|
902
|
+
source: 'anchor',
|
|
903
|
+
applied: true // Anchor is always applied
|
|
904
|
+
};
|
|
905
|
+
|
|
906
|
+
await transactionResource.insert(anchorTransaction);
|
|
907
|
+
|
|
908
|
+
if (config.verbose) {
|
|
909
|
+
console.log(
|
|
910
|
+
`[EventualConsistency] ${config.resource}.${config.field} - ` +
|
|
911
|
+
`Created anchor transaction for ${originalId} with value 0`
|
|
912
|
+
);
|
|
913
|
+
}
|
|
914
|
+
}
|
|
915
|
+
|
|
768
916
|
// Mark ALL transactions as pending (applied: false)
|
|
769
917
|
// Exclude anchor transactions (they should always be applied)
|
|
770
918
|
const transactionsToReset = allTransactions.filter(txn => txn.source !== 'anchor');
|