s3db.js 11.0.2 → 11.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/s3db.cjs.js +612 -308
- package/dist/s3db.cjs.js.map +1 -1
- package/dist/s3db.es.js +612 -308
- package/dist/s3db.es.js.map +1 -1
- package/package.json +1 -1
- package/src/concerns/plugin-storage.js +274 -9
- package/src/plugins/audit.plugin.js +94 -18
- package/src/plugins/eventual-consistency/analytics.js +131 -15
- package/src/plugins/eventual-consistency/config.js +3 -0
- package/src/plugins/eventual-consistency/consolidation.js +32 -36
- package/src/plugins/eventual-consistency/garbage-collection.js +11 -13
- package/src/plugins/eventual-consistency/index.js +28 -19
- package/src/plugins/eventual-consistency/install.js +9 -26
- package/src/plugins/eventual-consistency/partitions.js +5 -0
- package/src/plugins/eventual-consistency/transactions.js +1 -0
- package/src/plugins/eventual-consistency/utils.js +36 -1
- package/src/plugins/fulltext.plugin.js +76 -22
- package/src/plugins/metrics.plugin.js +70 -20
- package/src/plugins/s3-queue.plugin.js +21 -120
- package/src/plugins/scheduler.plugin.js +11 -37
|
@@ -44,14 +44,16 @@ export async function updateAnalytics(transactions, analyticsResource, config) {
|
|
|
44
44
|
if (config.verbose) {
|
|
45
45
|
console.log(
|
|
46
46
|
`[EventualConsistency] ${config.resource}.${config.field} - ` +
|
|
47
|
-
`Updating ${cohortCount} hourly analytics cohorts...`
|
|
47
|
+
`Updating ${cohortCount} hourly analytics cohorts IN PARALLEL...`
|
|
48
48
|
);
|
|
49
49
|
}
|
|
50
50
|
|
|
51
|
-
// Update hourly analytics
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
51
|
+
// ✅ OTIMIZAÇÃO: Update hourly analytics EM PARALELO
|
|
52
|
+
await Promise.all(
|
|
53
|
+
Object.entries(byHour).map(([cohort, txns]) =>
|
|
54
|
+
upsertAnalytics('hour', cohort, txns, analyticsResource, config)
|
|
55
|
+
)
|
|
56
|
+
);
|
|
55
57
|
|
|
56
58
|
// Roll up to daily and monthly if configured
|
|
57
59
|
if (config.analyticsConfig.rollupStrategy === 'incremental') {
|
|
@@ -60,13 +62,16 @@ export async function updateAnalytics(transactions, analyticsResource, config) {
|
|
|
60
62
|
if (config.verbose) {
|
|
61
63
|
console.log(
|
|
62
64
|
`[EventualConsistency] ${config.resource}.${config.field} - ` +
|
|
63
|
-
`Rolling up ${uniqueHours.length} hours to daily/monthly analytics...`
|
|
65
|
+
`Rolling up ${uniqueHours.length} hours to daily/weekly/monthly analytics IN PARALLEL...`
|
|
64
66
|
);
|
|
65
67
|
}
|
|
66
68
|
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
69
|
+
// ✅ OTIMIZAÇÃO: Rollup analytics EM PARALELO
|
|
70
|
+
await Promise.all(
|
|
71
|
+
uniqueHours.map(cohortHour =>
|
|
72
|
+
rollupAnalytics(cohortHour, analyticsResource, config)
|
|
73
|
+
)
|
|
74
|
+
);
|
|
70
75
|
}
|
|
71
76
|
|
|
72
77
|
if (config.verbose) {
|
|
@@ -206,7 +211,7 @@ function calculateOperationBreakdown(transactions) {
|
|
|
206
211
|
}
|
|
207
212
|
|
|
208
213
|
/**
|
|
209
|
-
* Roll up hourly analytics to daily and monthly
|
|
214
|
+
* Roll up hourly analytics to daily, weekly, and monthly
|
|
210
215
|
* @private
|
|
211
216
|
*/
|
|
212
217
|
async function rollupAnalytics(cohortHour, analyticsResource, config) {
|
|
@@ -214,20 +219,58 @@ async function rollupAnalytics(cohortHour, analyticsResource, config) {
|
|
|
214
219
|
const cohortDate = cohortHour.substring(0, 10); // '2025-10-09'
|
|
215
220
|
const cohortMonth = cohortHour.substring(0, 7); // '2025-10'
|
|
216
221
|
|
|
222
|
+
// Calculate week cohort (ISO 8601 format)
|
|
223
|
+
const date = new Date(cohortDate);
|
|
224
|
+
const cohortWeek = getCohortWeekFromDate(date);
|
|
225
|
+
|
|
217
226
|
// Roll up to day
|
|
218
227
|
await rollupPeriod('day', cohortDate, cohortDate, analyticsResource, config);
|
|
219
228
|
|
|
229
|
+
// Roll up to week
|
|
230
|
+
await rollupPeriod('week', cohortWeek, cohortWeek, analyticsResource, config);
|
|
231
|
+
|
|
220
232
|
// Roll up to month
|
|
221
233
|
await rollupPeriod('month', cohortMonth, cohortMonth, analyticsResource, config);
|
|
222
234
|
}
|
|
223
235
|
|
|
236
|
+
/**
|
|
237
|
+
* Get cohort week string from a date
|
|
238
|
+
* @private
|
|
239
|
+
*/
|
|
240
|
+
function getCohortWeekFromDate(date) {
|
|
241
|
+
// ISO week calculation (use UTC methods)
|
|
242
|
+
const target = new Date(date.valueOf());
|
|
243
|
+
const dayNr = (date.getUTCDay() + 6) % 7;
|
|
244
|
+
target.setUTCDate(target.getUTCDate() - dayNr + 3);
|
|
245
|
+
|
|
246
|
+
const yearStart = new Date(Date.UTC(target.getUTCFullYear(), 0, 1));
|
|
247
|
+
const firstThursday = new Date(yearStart.valueOf());
|
|
248
|
+
if (yearStart.getUTCDay() !== 4) {
|
|
249
|
+
firstThursday.setUTCDate(yearStart.getUTCDate() + ((4 - yearStart.getUTCDay()) + 7) % 7);
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
const weekNumber = 1 + Math.round((target - firstThursday) / 604800000);
|
|
253
|
+
const weekYear = target.getUTCFullYear();
|
|
254
|
+
|
|
255
|
+
return `${weekYear}-W${String(weekNumber).padStart(2, '0')}`;
|
|
256
|
+
}
|
|
257
|
+
|
|
224
258
|
/**
|
|
225
259
|
* Roll up analytics for a specific period
|
|
226
260
|
* @private
|
|
227
261
|
*/
|
|
228
262
|
async function rollupPeriod(period, cohort, sourcePrefix, analyticsResource, config) {
|
|
229
|
-
// Get all source analytics (e.g., all hours for a day)
|
|
230
|
-
|
|
263
|
+
// Get all source analytics (e.g., all hours for a day, all days for a week)
|
|
264
|
+
let sourcePeriod;
|
|
265
|
+
if (period === 'day') {
|
|
266
|
+
sourcePeriod = 'hour';
|
|
267
|
+
} else if (period === 'week') {
|
|
268
|
+
sourcePeriod = 'day';
|
|
269
|
+
} else if (period === 'month') {
|
|
270
|
+
sourcePeriod = 'week'; // Aggregate weeks to month
|
|
271
|
+
} else {
|
|
272
|
+
sourcePeriod = 'day'; // Fallback
|
|
273
|
+
}
|
|
231
274
|
|
|
232
275
|
const [ok, err, allAnalytics] = await tryFn(() =>
|
|
233
276
|
analyticsResource.list()
|
|
@@ -236,9 +279,22 @@ async function rollupPeriod(period, cohort, sourcePrefix, analyticsResource, con
|
|
|
236
279
|
if (!ok || !allAnalytics) return;
|
|
237
280
|
|
|
238
281
|
// Filter to matching cohorts
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
282
|
+
let sourceAnalytics;
|
|
283
|
+
if (period === 'week') {
|
|
284
|
+
// For week, we need to find all days that belong to this week
|
|
285
|
+
sourceAnalytics = allAnalytics.filter(a => {
|
|
286
|
+
if (a.period !== sourcePeriod) return false;
|
|
287
|
+
// Check if this day's cohort belongs to the target week
|
|
288
|
+
const dayDate = new Date(a.cohort);
|
|
289
|
+
const dayWeek = getCohortWeekFromDate(dayDate);
|
|
290
|
+
return dayWeek === cohort;
|
|
291
|
+
});
|
|
292
|
+
} else {
|
|
293
|
+
// For day and month, simple prefix matching works
|
|
294
|
+
sourceAnalytics = allAnalytics.filter(a =>
|
|
295
|
+
a.period === sourcePeriod && a.cohort.startsWith(sourcePrefix)
|
|
296
|
+
);
|
|
297
|
+
}
|
|
242
298
|
|
|
243
299
|
if (sourceAnalytics.length === 0) return;
|
|
244
300
|
|
|
@@ -567,6 +623,66 @@ export async function getYearByMonth(resourceName, field, year, options, fieldHa
|
|
|
567
623
|
return data;
|
|
568
624
|
}
|
|
569
625
|
|
|
626
|
+
/**
|
|
627
|
+
* Get analytics for entire year, broken down by weeks
|
|
628
|
+
*
|
|
629
|
+
* @param {string} resourceName - Resource name
|
|
630
|
+
* @param {string} field - Field name
|
|
631
|
+
* @param {number} year - Year (e.g., 2025)
|
|
632
|
+
* @param {Object} options - Options
|
|
633
|
+
* @param {Object} fieldHandlers - Field handlers map
|
|
634
|
+
* @returns {Promise<Array>} Weekly analytics for the year (up to 53 weeks)
|
|
635
|
+
*/
|
|
636
|
+
export async function getYearByWeek(resourceName, field, year, options, fieldHandlers) {
|
|
637
|
+
const data = await getAnalytics(resourceName, field, {
|
|
638
|
+
period: 'week',
|
|
639
|
+
year
|
|
640
|
+
}, fieldHandlers);
|
|
641
|
+
|
|
642
|
+
// Week data doesn't need gap filling as much as daily/hourly
|
|
643
|
+
// But we can still provide it if requested
|
|
644
|
+
if (options.fillGaps) {
|
|
645
|
+
// ISO weeks: typically 52-53 weeks per year
|
|
646
|
+
const startWeek = `${year}-W01`;
|
|
647
|
+
const endWeek = `${year}-W53`;
|
|
648
|
+
return fillGaps(data, 'week', startWeek, endWeek);
|
|
649
|
+
}
|
|
650
|
+
|
|
651
|
+
return data;
|
|
652
|
+
}
|
|
653
|
+
|
|
654
|
+
/**
|
|
655
|
+
* Get analytics for entire month, broken down by weeks
|
|
656
|
+
*
|
|
657
|
+
* @param {string} resourceName - Resource name
|
|
658
|
+
* @param {string} field - Field name
|
|
659
|
+
* @param {string} month - Month in YYYY-MM format
|
|
660
|
+
* @param {Object} options - Options
|
|
661
|
+
* @param {Object} fieldHandlers - Field handlers map
|
|
662
|
+
* @returns {Promise<Array>} Weekly analytics for the month
|
|
663
|
+
*/
|
|
664
|
+
export async function getMonthByWeek(resourceName, field, month, options, fieldHandlers) {
|
|
665
|
+
// month format: '2025-10'
|
|
666
|
+
const year = parseInt(month.substring(0, 4));
|
|
667
|
+
const monthNum = parseInt(month.substring(5, 7));
|
|
668
|
+
|
|
669
|
+
// Get first and last day of month
|
|
670
|
+
const firstDay = new Date(year, monthNum - 1, 1);
|
|
671
|
+
const lastDay = new Date(year, monthNum, 0);
|
|
672
|
+
|
|
673
|
+
// Find which weeks this month spans
|
|
674
|
+
const firstWeek = getCohortWeekFromDate(firstDay);
|
|
675
|
+
const lastWeek = getCohortWeekFromDate(lastDay);
|
|
676
|
+
|
|
677
|
+
const data = await getAnalytics(resourceName, field, {
|
|
678
|
+
period: 'week',
|
|
679
|
+
startDate: firstWeek,
|
|
680
|
+
endDate: lastWeek
|
|
681
|
+
}, fieldHandlers);
|
|
682
|
+
|
|
683
|
+
return data;
|
|
684
|
+
}
|
|
685
|
+
|
|
570
686
|
/**
|
|
571
687
|
* Get analytics for entire month, broken down by hours
|
|
572
688
|
*
|
|
@@ -47,6 +47,9 @@ export function createConfig(options, detectedTimezone) {
|
|
|
47
47
|
autoConsolidate: consolidation.auto !== false,
|
|
48
48
|
mode: consolidation.mode || 'async',
|
|
49
49
|
|
|
50
|
+
// ✅ NOVO: Performance tuning - Mark applied concurrency (default 50, antes era 10 hardcoded)
|
|
51
|
+
markAppliedConcurrency: consolidation.markAppliedConcurrency ?? 50,
|
|
52
|
+
|
|
50
53
|
// Late arrivals
|
|
51
54
|
lateArrivalStrategy: lateArrivals.strategy || 'warn',
|
|
52
55
|
|
|
@@ -7,7 +7,6 @@ import tryFn from "../../concerns/try-fn.js";
|
|
|
7
7
|
import { PromisePool } from "@supercharge/promise-pool";
|
|
8
8
|
import { idGenerator } from "../../concerns/id.js";
|
|
9
9
|
import { getCohortInfo, createSyntheticSetTransaction } from "./utils.js";
|
|
10
|
-
import { cleanupStaleLocks } from "./locks.js";
|
|
11
10
|
|
|
12
11
|
/**
|
|
13
12
|
* Start consolidation timer for a handler
|
|
@@ -169,7 +168,7 @@ export async function runConsolidation(transactionResource, consolidateRecordFn,
|
|
|
169
168
|
* @param {string} originalId - ID of the record to consolidate
|
|
170
169
|
* @param {Object} transactionResource - Transaction resource
|
|
171
170
|
* @param {Object} targetResource - Target resource
|
|
172
|
-
* @param {Object}
|
|
171
|
+
* @param {Object} storage - PluginStorage instance for locks
|
|
173
172
|
* @param {Object} analyticsResource - Analytics resource (optional)
|
|
174
173
|
* @param {Function} updateAnalyticsFn - Function to update analytics (optional)
|
|
175
174
|
* @param {Object} config - Plugin configuration
|
|
@@ -179,26 +178,21 @@ export async function consolidateRecord(
|
|
|
179
178
|
originalId,
|
|
180
179
|
transactionResource,
|
|
181
180
|
targetResource,
|
|
182
|
-
|
|
181
|
+
storage,
|
|
183
182
|
analyticsResource,
|
|
184
183
|
updateAnalyticsFn,
|
|
185
184
|
config
|
|
186
185
|
) {
|
|
187
|
-
//
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
id: lockId,
|
|
195
|
-
lockedAt: Date.now(),
|
|
196
|
-
workerId: process.pid ? String(process.pid) : 'unknown'
|
|
197
|
-
})
|
|
198
|
-
);
|
|
186
|
+
// Acquire distributed lock with TTL to prevent concurrent consolidation
|
|
187
|
+
const lockKey = `consolidation-${config.resource}-${config.field}-${originalId}`;
|
|
188
|
+
const lock = await storage.acquireLock(lockKey, {
|
|
189
|
+
ttl: config.lockTimeout || 30,
|
|
190
|
+
timeout: 0, // Don't wait if locked
|
|
191
|
+
workerId: process.pid ? String(process.pid) : 'unknown'
|
|
192
|
+
});
|
|
199
193
|
|
|
200
194
|
// If lock couldn't be acquired, another worker is consolidating
|
|
201
|
-
if (!
|
|
195
|
+
if (!lock) {
|
|
202
196
|
if (config.verbose) {
|
|
203
197
|
console.log(`[EventualConsistency] Lock for ${originalId} already held, skipping`);
|
|
204
198
|
}
|
|
@@ -508,9 +502,12 @@ export async function consolidateRecord(
|
|
|
508
502
|
// Mark transactions as applied (skip synthetic ones) - use PromisePool for controlled concurrency
|
|
509
503
|
const transactionsToUpdate = transactions.filter(txn => txn.id !== '__synthetic__');
|
|
510
504
|
|
|
505
|
+
// ✅ OTIMIZAÇÃO: Usar concurrency do config (default aumentado de 10 para 50)
|
|
506
|
+
const markAppliedConcurrency = config.markAppliedConcurrency || 50;
|
|
507
|
+
|
|
511
508
|
const { results, errors } = await PromisePool
|
|
512
509
|
.for(transactionsToUpdate)
|
|
513
|
-
.withConcurrency(
|
|
510
|
+
.withConcurrency(markAppliedConcurrency) // ✅ Configurável e maior!
|
|
514
511
|
.process(async (txn) => {
|
|
515
512
|
const [ok, err] = await tryFn(() =>
|
|
516
513
|
transactionResource.update(txn.id, { applied: true })
|
|
@@ -576,10 +573,12 @@ export async function consolidateRecord(
|
|
|
576
573
|
return consolidatedValue;
|
|
577
574
|
} finally {
|
|
578
575
|
// Always release the lock
|
|
579
|
-
const [lockReleased, lockReleaseErr] = await tryFn(() =>
|
|
576
|
+
const [lockReleased, lockReleaseErr] = await tryFn(() =>
|
|
577
|
+
storage.releaseLock(lockKey)
|
|
578
|
+
);
|
|
580
579
|
|
|
581
580
|
if (!lockReleased && config.verbose) {
|
|
582
|
-
console.warn(`[EventualConsistency] Failed to release lock ${
|
|
581
|
+
console.warn(`[EventualConsistency] Failed to release lock ${lockKey}:`, lockReleaseErr?.message);
|
|
583
582
|
}
|
|
584
583
|
}
|
|
585
584
|
}
|
|
@@ -705,7 +704,7 @@ export async function getCohortStats(cohortDate, transactionResource) {
|
|
|
705
704
|
* @param {string} originalId - ID of the record to recalculate
|
|
706
705
|
* @param {Object} transactionResource - Transaction resource
|
|
707
706
|
* @param {Object} targetResource - Target resource
|
|
708
|
-
* @param {Object}
|
|
707
|
+
* @param {Object} storage - PluginStorage instance for locks
|
|
709
708
|
* @param {Function} consolidateRecordFn - Function to consolidate the record
|
|
710
709
|
* @param {Object} config - Plugin configuration
|
|
711
710
|
* @returns {Promise<number>} Recalculated value
|
|
@@ -714,25 +713,20 @@ export async function recalculateRecord(
|
|
|
714
713
|
originalId,
|
|
715
714
|
transactionResource,
|
|
716
715
|
targetResource,
|
|
717
|
-
|
|
716
|
+
storage,
|
|
718
717
|
consolidateRecordFn,
|
|
719
718
|
config
|
|
720
719
|
) {
|
|
721
|
-
//
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
id: lockId,
|
|
729
|
-
lockedAt: Date.now(),
|
|
730
|
-
workerId: process.pid ? String(process.pid) : 'unknown'
|
|
731
|
-
})
|
|
732
|
-
);
|
|
720
|
+
// Acquire distributed lock with TTL to prevent concurrent operations
|
|
721
|
+
const lockKey = `recalculate-${config.resource}-${config.field}-${originalId}`;
|
|
722
|
+
const lock = await storage.acquireLock(lockKey, {
|
|
723
|
+
ttl: config.lockTimeout || 30,
|
|
724
|
+
timeout: 0, // Don't wait if locked
|
|
725
|
+
workerId: process.pid ? String(process.pid) : 'unknown'
|
|
726
|
+
});
|
|
733
727
|
|
|
734
728
|
// If lock couldn't be acquired, another worker is operating on this record
|
|
735
|
-
if (!
|
|
729
|
+
if (!lock) {
|
|
736
730
|
if (config.verbose) {
|
|
737
731
|
console.log(`[EventualConsistency] Recalculate lock for ${originalId} already held, skipping`);
|
|
738
732
|
}
|
|
@@ -832,10 +826,12 @@ export async function recalculateRecord(
|
|
|
832
826
|
return consolidatedValue;
|
|
833
827
|
} finally {
|
|
834
828
|
// Always release the lock
|
|
835
|
-
const [lockReleased, lockReleaseErr] = await tryFn(() =>
|
|
829
|
+
const [lockReleased, lockReleaseErr] = await tryFn(() =>
|
|
830
|
+
storage.releaseLock(lockKey)
|
|
831
|
+
);
|
|
836
832
|
|
|
837
833
|
if (!lockReleased && config.verbose) {
|
|
838
|
-
console.warn(`[EventualConsistency] Failed to release recalculate lock ${
|
|
834
|
+
console.warn(`[EventualConsistency] Failed to release recalculate lock ${lockKey}:`, lockReleaseErr?.message);
|
|
839
835
|
}
|
|
840
836
|
}
|
|
841
837
|
}
|
|
@@ -31,24 +31,22 @@ export function startGarbageCollectionTimer(handler, resourceName, fieldName, ru
|
|
|
31
31
|
* Uses distributed locking to prevent multiple containers from running GC simultaneously
|
|
32
32
|
*
|
|
33
33
|
* @param {Object} transactionResource - Transaction resource
|
|
34
|
-
* @param {Object}
|
|
34
|
+
* @param {Object} storage - PluginStorage instance for locks
|
|
35
35
|
* @param {Object} config - Plugin configuration
|
|
36
36
|
* @param {Function} emitFn - Function to emit events
|
|
37
37
|
* @returns {Promise<void>}
|
|
38
38
|
*/
|
|
39
|
-
export async function runGarbageCollection(transactionResource,
|
|
40
|
-
// Acquire distributed lock for GC operation
|
|
41
|
-
const
|
|
42
|
-
const
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
})
|
|
48
|
-
);
|
|
39
|
+
export async function runGarbageCollection(transactionResource, storage, config, emitFn) {
|
|
40
|
+
// Acquire distributed lock with TTL for GC operation
|
|
41
|
+
const lockKey = `gc-${config.resource}-${config.field}`;
|
|
42
|
+
const lock = await storage.acquireLock(lockKey, {
|
|
43
|
+
ttl: 300, // 5 minutes for GC
|
|
44
|
+
timeout: 0, // Don't wait if locked
|
|
45
|
+
workerId: process.pid ? String(process.pid) : 'unknown'
|
|
46
|
+
});
|
|
49
47
|
|
|
50
48
|
// If another container is already running GC, skip
|
|
51
|
-
if (!
|
|
49
|
+
if (!lock) {
|
|
52
50
|
if (config.verbose) {
|
|
53
51
|
console.log(`[EventualConsistency] GC already running in another container`);
|
|
54
52
|
}
|
|
@@ -121,6 +119,6 @@ export async function runGarbageCollection(transactionResource, lockResource, co
|
|
|
121
119
|
}
|
|
122
120
|
} finally {
|
|
123
121
|
// Always release GC lock
|
|
124
|
-
await tryFn(() =>
|
|
122
|
+
await tryFn(() => storage.releaseLock(lockKey));
|
|
125
123
|
}
|
|
126
124
|
}
|
|
@@ -17,7 +17,7 @@ import {
|
|
|
17
17
|
runConsolidation
|
|
18
18
|
} from "./consolidation.js";
|
|
19
19
|
import { runGarbageCollection } from "./garbage-collection.js";
|
|
20
|
-
import { updateAnalytics, getAnalytics, getMonthByDay, getDayByHour, getLastNDays, getYearByMonth, getMonthByHour, getTopRecords } from "./analytics.js";
|
|
20
|
+
import { updateAnalytics, getAnalytics, getMonthByDay, getDayByHour, getLastNDays, getYearByMonth, getYearByWeek, getMonthByWeek, getMonthByHour, getTopRecords } from "./analytics.js";
|
|
21
21
|
import { onInstall, onStart, onStop, watchForResource, completeFieldSetup } from "./install.js";
|
|
22
22
|
|
|
23
23
|
export class EventualConsistencyPlugin extends Plugin {
|
|
@@ -125,7 +125,7 @@ export class EventualConsistencyPlugin extends Plugin {
|
|
|
125
125
|
originalId,
|
|
126
126
|
this.transactionResource,
|
|
127
127
|
this.targetResource,
|
|
128
|
-
this.
|
|
128
|
+
this.getStorage(),
|
|
129
129
|
this.analyticsResource,
|
|
130
130
|
(transactions) => this.updateAnalytics(transactions),
|
|
131
131
|
this.config
|
|
@@ -164,7 +164,7 @@ export class EventualConsistencyPlugin extends Plugin {
|
|
|
164
164
|
originalId,
|
|
165
165
|
this.transactionResource,
|
|
166
166
|
this.targetResource,
|
|
167
|
-
this.
|
|
167
|
+
this.getStorage(),
|
|
168
168
|
(id) => this.consolidateRecord(id),
|
|
169
169
|
this.config
|
|
170
170
|
);
|
|
@@ -188,14 +188,12 @@ export class EventualConsistencyPlugin extends Plugin {
|
|
|
188
188
|
const oldField = this.config.field;
|
|
189
189
|
const oldTransactionResource = this.transactionResource;
|
|
190
190
|
const oldTargetResource = this.targetResource;
|
|
191
|
-
const oldLockResource = this.lockResource;
|
|
192
191
|
const oldAnalyticsResource = this.analyticsResource;
|
|
193
192
|
|
|
194
193
|
this.config.resource = handler.resource;
|
|
195
194
|
this.config.field = handler.field;
|
|
196
195
|
this.transactionResource = handler.transactionResource;
|
|
197
196
|
this.targetResource = handler.targetResource;
|
|
198
|
-
this.lockResource = handler.lockResource;
|
|
199
197
|
this.analyticsResource = handler.analyticsResource;
|
|
200
198
|
|
|
201
199
|
const result = await this.consolidateRecord(id);
|
|
@@ -205,7 +203,6 @@ export class EventualConsistencyPlugin extends Plugin {
|
|
|
205
203
|
this.config.field = oldField;
|
|
206
204
|
this.transactionResource = oldTransactionResource;
|
|
207
205
|
this.targetResource = oldTargetResource;
|
|
208
|
-
this.lockResource = oldLockResource;
|
|
209
206
|
this.analyticsResource = oldAnalyticsResource;
|
|
210
207
|
|
|
211
208
|
return result;
|
|
@@ -220,14 +217,12 @@ export class EventualConsistencyPlugin extends Plugin {
|
|
|
220
217
|
const oldField = this.config.field;
|
|
221
218
|
const oldTransactionResource = this.transactionResource;
|
|
222
219
|
const oldTargetResource = this.targetResource;
|
|
223
|
-
const oldLockResource = this.lockResource;
|
|
224
220
|
const oldAnalyticsResource = this.analyticsResource;
|
|
225
221
|
|
|
226
222
|
this.config.resource = handler.resource;
|
|
227
223
|
this.config.field = handler.field;
|
|
228
224
|
this.transactionResource = handler.transactionResource;
|
|
229
225
|
this.targetResource = handler.targetResource;
|
|
230
|
-
this.lockResource = handler.lockResource;
|
|
231
226
|
this.analyticsResource = handler.analyticsResource;
|
|
232
227
|
|
|
233
228
|
const result = await this.consolidateRecord(id);
|
|
@@ -236,7 +231,6 @@ export class EventualConsistencyPlugin extends Plugin {
|
|
|
236
231
|
this.config.field = oldField;
|
|
237
232
|
this.transactionResource = oldTransactionResource;
|
|
238
233
|
this.targetResource = oldTargetResource;
|
|
239
|
-
this.lockResource = oldLockResource;
|
|
240
234
|
this.analyticsResource = oldAnalyticsResource;
|
|
241
235
|
|
|
242
236
|
return result;
|
|
@@ -276,14 +270,12 @@ export class EventualConsistencyPlugin extends Plugin {
|
|
|
276
270
|
const oldField = this.config.field;
|
|
277
271
|
const oldTransactionResource = this.transactionResource;
|
|
278
272
|
const oldTargetResource = this.targetResource;
|
|
279
|
-
const oldLockResource = this.lockResource;
|
|
280
273
|
const oldAnalyticsResource = this.analyticsResource;
|
|
281
274
|
|
|
282
275
|
this.config.resource = handler.resource;
|
|
283
276
|
this.config.field = handler.field;
|
|
284
277
|
this.transactionResource = handler.transactionResource;
|
|
285
278
|
this.targetResource = handler.targetResource;
|
|
286
|
-
this.lockResource = handler.lockResource;
|
|
287
279
|
this.analyticsResource = handler.analyticsResource;
|
|
288
280
|
|
|
289
281
|
const result = await this.recalculateRecord(id);
|
|
@@ -292,7 +284,6 @@ export class EventualConsistencyPlugin extends Plugin {
|
|
|
292
284
|
this.config.field = oldField;
|
|
293
285
|
this.transactionResource = oldTransactionResource;
|
|
294
286
|
this.targetResource = oldTargetResource;
|
|
295
|
-
this.lockResource = oldLockResource;
|
|
296
287
|
this.analyticsResource = oldAnalyticsResource;
|
|
297
288
|
|
|
298
289
|
return result;
|
|
@@ -307,14 +298,12 @@ export class EventualConsistencyPlugin extends Plugin {
|
|
|
307
298
|
const oldField = this.config.field;
|
|
308
299
|
const oldTransactionResource = this.transactionResource;
|
|
309
300
|
const oldTargetResource = this.targetResource;
|
|
310
|
-
const oldLockResource = this.lockResource;
|
|
311
301
|
const oldAnalyticsResource = this.analyticsResource;
|
|
312
302
|
|
|
313
303
|
this.config.resource = resourceName;
|
|
314
304
|
this.config.field = fieldName;
|
|
315
305
|
this.transactionResource = handler.transactionResource;
|
|
316
306
|
this.targetResource = handler.targetResource;
|
|
317
|
-
this.lockResource = handler.lockResource;
|
|
318
307
|
this.analyticsResource = handler.analyticsResource;
|
|
319
308
|
|
|
320
309
|
try {
|
|
@@ -329,7 +318,6 @@ export class EventualConsistencyPlugin extends Plugin {
|
|
|
329
318
|
this.config.field = oldField;
|
|
330
319
|
this.transactionResource = oldTransactionResource;
|
|
331
320
|
this.targetResource = oldTargetResource;
|
|
332
|
-
this.lockResource = oldLockResource;
|
|
333
321
|
this.analyticsResource = oldAnalyticsResource;
|
|
334
322
|
}
|
|
335
323
|
}
|
|
@@ -343,18 +331,16 @@ export class EventualConsistencyPlugin extends Plugin {
|
|
|
343
331
|
const oldField = this.config.field;
|
|
344
332
|
const oldTransactionResource = this.transactionResource;
|
|
345
333
|
const oldTargetResource = this.targetResource;
|
|
346
|
-
const oldLockResource = this.lockResource;
|
|
347
334
|
|
|
348
335
|
this.config.resource = resourceName;
|
|
349
336
|
this.config.field = fieldName;
|
|
350
337
|
this.transactionResource = handler.transactionResource;
|
|
351
338
|
this.targetResource = handler.targetResource;
|
|
352
|
-
this.lockResource = handler.lockResource;
|
|
353
339
|
|
|
354
340
|
try {
|
|
355
341
|
await runGarbageCollection(
|
|
356
342
|
this.transactionResource,
|
|
357
|
-
this.
|
|
343
|
+
this.getStorage(),
|
|
358
344
|
this.config,
|
|
359
345
|
(event, data) => this.emit(event, data)
|
|
360
346
|
);
|
|
@@ -363,7 +349,6 @@ export class EventualConsistencyPlugin extends Plugin {
|
|
|
363
349
|
this.config.field = oldField;
|
|
364
350
|
this.transactionResource = oldTransactionResource;
|
|
365
351
|
this.targetResource = oldTargetResource;
|
|
366
|
-
this.lockResource = oldLockResource;
|
|
367
352
|
}
|
|
368
353
|
}
|
|
369
354
|
|
|
@@ -440,6 +425,30 @@ export class EventualConsistencyPlugin extends Plugin {
|
|
|
440
425
|
return await getMonthByHour(resourceName, field, month, options, this.fieldHandlers);
|
|
441
426
|
}
|
|
442
427
|
|
|
428
|
+
/**
|
|
429
|
+
* Get analytics for entire year, broken down by weeks
|
|
430
|
+
* @param {string} resourceName - Resource name
|
|
431
|
+
* @param {string} field - Field name
|
|
432
|
+
* @param {number} year - Year (e.g., 2025)
|
|
433
|
+
* @param {Object} options - Options
|
|
434
|
+
* @returns {Promise<Array>} Weekly analytics for the year (up to 53 weeks)
|
|
435
|
+
*/
|
|
436
|
+
async getYearByWeek(resourceName, field, year, options = {}) {
|
|
437
|
+
return await getYearByWeek(resourceName, field, year, options, this.fieldHandlers);
|
|
438
|
+
}
|
|
439
|
+
|
|
440
|
+
/**
|
|
441
|
+
* Get analytics for entire month, broken down by weeks
|
|
442
|
+
* @param {string} resourceName - Resource name
|
|
443
|
+
* @param {string} field - Field name
|
|
444
|
+
* @param {string} month - Month in YYYY-MM format
|
|
445
|
+
* @param {Object} options - Options
|
|
446
|
+
* @returns {Promise<Array>} Weekly analytics for the month
|
|
447
|
+
*/
|
|
448
|
+
async getMonthByWeek(resourceName, field, month, options = {}) {
|
|
449
|
+
return await getMonthByWeek(resourceName, field, month, options, this.fieldHandlers);
|
|
450
|
+
}
|
|
451
|
+
|
|
443
452
|
/**
|
|
444
453
|
* Get top records by volume
|
|
445
454
|
* @param {string} resourceName - Resource name
|
|
@@ -84,8 +84,8 @@ export async function completeFieldSetup(handler, database, config, plugin) {
|
|
|
84
84
|
const resourceName = handler.resource;
|
|
85
85
|
const fieldName = handler.field;
|
|
86
86
|
|
|
87
|
-
// Create transaction resource with partitions
|
|
88
|
-
const transactionResourceName =
|
|
87
|
+
// Create transaction resource with partitions (plg_ prefix for plugin resources)
|
|
88
|
+
const transactionResourceName = `plg_${resourceName}_tx_${fieldName}`;
|
|
89
89
|
const partitionConfig = createPartitionConfig();
|
|
90
90
|
|
|
91
91
|
const [ok, err, transactionResource] = await tryFn(() =>
|
|
@@ -100,6 +100,7 @@ export async function completeFieldSetup(handler, database, config, plugin) {
|
|
|
100
100
|
timestamp: 'string|required',
|
|
101
101
|
cohortDate: 'string|required',
|
|
102
102
|
cohortHour: 'string|required',
|
|
103
|
+
cohortWeek: 'string|optional',
|
|
103
104
|
cohortMonth: 'string|optional',
|
|
104
105
|
source: 'string|optional',
|
|
105
106
|
applied: 'boolean|optional'
|
|
@@ -118,27 +119,8 @@ export async function completeFieldSetup(handler, database, config, plugin) {
|
|
|
118
119
|
|
|
119
120
|
handler.transactionResource = ok ? transactionResource : database.resources[transactionResourceName];
|
|
120
121
|
|
|
121
|
-
//
|
|
122
|
-
|
|
123
|
-
const [lockOk, lockErr, lockResource] = await tryFn(() =>
|
|
124
|
-
database.createResource({
|
|
125
|
-
name: lockResourceName,
|
|
126
|
-
attributes: {
|
|
127
|
-
id: 'string|required',
|
|
128
|
-
lockedAt: 'number|required',
|
|
129
|
-
workerId: 'string|optional'
|
|
130
|
-
},
|
|
131
|
-
behavior: 'body-only',
|
|
132
|
-
timestamps: false,
|
|
133
|
-
createdBy: 'EventualConsistencyPlugin'
|
|
134
|
-
})
|
|
135
|
-
);
|
|
136
|
-
|
|
137
|
-
if (!lockOk && !database.resources[lockResourceName]) {
|
|
138
|
-
throw new Error(`Failed to create lock resource for ${resourceName}.${fieldName}: ${lockErr?.message}`);
|
|
139
|
-
}
|
|
140
|
-
|
|
141
|
-
handler.lockResource = lockOk ? lockResource : database.resources[lockResourceName];
|
|
122
|
+
// Locks are now managed by PluginStorage with TTL - no Resource needed
|
|
123
|
+
// Lock acquisition is handled via storage.acquireLock() with automatic expiration
|
|
142
124
|
|
|
143
125
|
// Create analytics resource if enabled
|
|
144
126
|
if (config.enableAnalytics) {
|
|
@@ -151,8 +133,9 @@ export async function completeFieldSetup(handler, database, config, plugin) {
|
|
|
151
133
|
if (config.verbose) {
|
|
152
134
|
console.log(
|
|
153
135
|
`[EventualConsistency] ${resourceName}.${fieldName} - ` +
|
|
154
|
-
`Setup complete. Resources: ${transactionResourceName}
|
|
155
|
-
`${config.enableAnalytics ? `, ${resourceName}
|
|
136
|
+
`Setup complete. Resources: ${transactionResourceName}` +
|
|
137
|
+
`${config.enableAnalytics ? `, plg_${resourceName}_an_${fieldName}` : ''}` +
|
|
138
|
+
` (locks via PluginStorage TTL)`
|
|
156
139
|
);
|
|
157
140
|
}
|
|
158
141
|
}
|
|
@@ -167,7 +150,7 @@ export async function completeFieldSetup(handler, database, config, plugin) {
|
|
|
167
150
|
* @returns {Promise<void>}
|
|
168
151
|
*/
|
|
169
152
|
async function createAnalyticsResource(handler, database, resourceName, fieldName) {
|
|
170
|
-
const analyticsResourceName =
|
|
153
|
+
const analyticsResourceName = `plg_${resourceName}_an_${fieldName}`;
|
|
171
154
|
|
|
172
155
|
const [ok, err, analyticsResource] = await tryFn(() =>
|
|
173
156
|
database.createResource({
|
|
@@ -54,6 +54,7 @@ export async function createTransaction(handler, data, config) {
|
|
|
54
54
|
timestamp: now.toISOString(),
|
|
55
55
|
cohortDate: cohortInfo.date,
|
|
56
56
|
cohortHour: cohortInfo.hour,
|
|
57
|
+
cohortWeek: cohortInfo.week,
|
|
57
58
|
cohortMonth: cohortInfo.month,
|
|
58
59
|
source: data.source || 'unknown',
|
|
59
60
|
applied: false
|