s3db.js 10.0.16 → 10.0.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/s3db.cjs.js +1766 -1549
- package/dist/s3db.cjs.js.map +1 -1
- package/dist/s3db.es.js +1766 -1549
- package/dist/s3db.es.js.map +1 -1
- package/package.json +1 -1
- package/src/plugins/eventual-consistency/analytics.js +668 -0
- package/src/plugins/eventual-consistency/config.js +156 -0
- package/src/plugins/eventual-consistency/consolidation.js +770 -0
- package/src/plugins/eventual-consistency/garbage-collection.js +126 -0
- package/src/plugins/eventual-consistency/helpers.js +179 -0
- package/src/plugins/eventual-consistency/index.js +455 -0
- package/src/plugins/eventual-consistency/locks.js +77 -0
- package/src/plugins/eventual-consistency/partitions.js +45 -0
- package/src/plugins/eventual-consistency/setup.js +298 -0
- package/src/plugins/eventual-consistency/transactions.js +119 -0
- package/src/plugins/eventual-consistency/utils.js +172 -0
- package/src/plugins/eventual-consistency.plugin.js +195 -2
- package/src/plugins/index.js +1 -1
package/dist/s3db.cjs.js
CHANGED
|
@@ -4307,1466 +4307,1907 @@ const CostsPlugin = {
|
|
|
4307
4307
|
}
|
|
4308
4308
|
};
|
|
4309
4309
|
|
|
4310
|
-
|
|
4311
|
-
|
|
4312
|
-
|
|
4313
|
-
|
|
4310
|
+
function createConfig(options, detectedTimezone) {
|
|
4311
|
+
const consolidation = options.consolidation || {};
|
|
4312
|
+
const locks = options.locks || {};
|
|
4313
|
+
const gc = options.garbageCollection || {};
|
|
4314
|
+
const analytics = options.analytics || {};
|
|
4315
|
+
const batch = options.batch || {};
|
|
4316
|
+
const lateArrivals = options.lateArrivals || {};
|
|
4317
|
+
const checkpoints = options.checkpoints || {};
|
|
4318
|
+
return {
|
|
4319
|
+
// Cohort (timezone)
|
|
4320
|
+
cohort: {
|
|
4321
|
+
timezone: options.cohort?.timezone || detectedTimezone
|
|
4322
|
+
},
|
|
4323
|
+
// Reducer function
|
|
4324
|
+
reducer: options.reducer || ((transactions) => {
|
|
4325
|
+
let baseValue = 0;
|
|
4326
|
+
for (const t of transactions) {
|
|
4327
|
+
if (t.operation === "set") {
|
|
4328
|
+
baseValue = t.value;
|
|
4329
|
+
} else if (t.operation === "add") {
|
|
4330
|
+
baseValue += t.value;
|
|
4331
|
+
} else if (t.operation === "sub") {
|
|
4332
|
+
baseValue -= t.value;
|
|
4333
|
+
}
|
|
4334
|
+
}
|
|
4335
|
+
return baseValue;
|
|
4336
|
+
}),
|
|
4337
|
+
// Consolidation settings
|
|
4338
|
+
consolidationInterval: consolidation.interval ?? 300,
|
|
4339
|
+
consolidationConcurrency: consolidation.concurrency ?? 5,
|
|
4340
|
+
consolidationWindow: consolidation.window ?? 24,
|
|
4341
|
+
autoConsolidate: consolidation.auto !== false,
|
|
4342
|
+
mode: consolidation.mode || "async",
|
|
4343
|
+
// Late arrivals
|
|
4344
|
+
lateArrivalStrategy: lateArrivals.strategy || "warn",
|
|
4345
|
+
// Batch transactions
|
|
4346
|
+
batchTransactions: batch.enabled || false,
|
|
4347
|
+
batchSize: batch.size || 100,
|
|
4348
|
+
// Locks
|
|
4349
|
+
lockTimeout: locks.timeout || 300,
|
|
4350
|
+
// Garbage collection
|
|
4351
|
+
transactionRetention: gc.retention ?? 30,
|
|
4352
|
+
gcInterval: gc.interval ?? 86400,
|
|
4353
|
+
// Analytics
|
|
4354
|
+
enableAnalytics: analytics.enabled || false,
|
|
4355
|
+
analyticsConfig: {
|
|
4356
|
+
periods: analytics.periods || ["hour", "day", "month"],
|
|
4357
|
+
metrics: analytics.metrics || ["count", "sum", "avg", "min", "max"],
|
|
4358
|
+
rollupStrategy: analytics.rollupStrategy || "incremental",
|
|
4359
|
+
retentionDays: analytics.retentionDays ?? 365
|
|
4360
|
+
},
|
|
4361
|
+
// Checkpoints
|
|
4362
|
+
enableCheckpoints: checkpoints.enabled !== false,
|
|
4363
|
+
checkpointStrategy: checkpoints.strategy || "hourly",
|
|
4364
|
+
checkpointRetention: checkpoints.retention ?? 90,
|
|
4365
|
+
checkpointThreshold: checkpoints.threshold ?? 1e3,
|
|
4366
|
+
deleteConsolidatedTransactions: checkpoints.deleteConsolidated !== false,
|
|
4367
|
+
autoCheckpoint: checkpoints.auto !== false,
|
|
4368
|
+
// Debug
|
|
4369
|
+
verbose: options.verbose || false
|
|
4370
|
+
};
|
|
4371
|
+
}
|
|
4372
|
+
function validateResourcesConfig(resources) {
|
|
4373
|
+
if (!resources || typeof resources !== "object") {
|
|
4374
|
+
throw new Error(
|
|
4375
|
+
"EventualConsistencyPlugin requires 'resources' option.\nExample: { resources: { urls: ['clicks', 'views'], posts: ['likes'] } }"
|
|
4376
|
+
);
|
|
4377
|
+
}
|
|
4378
|
+
for (const [resourceName, fields] of Object.entries(resources)) {
|
|
4379
|
+
if (!Array.isArray(fields)) {
|
|
4314
4380
|
throw new Error(
|
|
4315
|
-
|
|
4381
|
+
`EventualConsistencyPlugin resources.${resourceName} must be an array of field names`
|
|
4316
4382
|
);
|
|
4317
4383
|
}
|
|
4318
|
-
|
|
4319
|
-
|
|
4320
|
-
|
|
4321
|
-
|
|
4322
|
-
|
|
4323
|
-
|
|
4324
|
-
|
|
4325
|
-
|
|
4326
|
-
|
|
4327
|
-
|
|
4328
|
-
|
|
4329
|
-
|
|
4330
|
-
|
|
4331
|
-
|
|
4332
|
-
|
|
4333
|
-
|
|
4334
|
-
|
|
4335
|
-
|
|
4336
|
-
|
|
4337
|
-
|
|
4338
|
-
|
|
4339
|
-
|
|
4340
|
-
|
|
4341
|
-
|
|
4342
|
-
|
|
4343
|
-
|
|
4344
|
-
|
|
4345
|
-
|
|
4346
|
-
|
|
4347
|
-
|
|
4348
|
-
|
|
4349
|
-
|
|
4350
|
-
|
|
4351
|
-
|
|
4352
|
-
|
|
4353
|
-
|
|
4354
|
-
|
|
4384
|
+
}
|
|
4385
|
+
}
|
|
4386
|
+
function logConfigWarnings(config) {
|
|
4387
|
+
if (config.batchTransactions && !config.verbose) {
|
|
4388
|
+
console.warn(
|
|
4389
|
+
`[EventualConsistency] WARNING: batch.enabled is true. This stores transactions in memory and will lose data if container crashes. Not recommended for distributed/production environments.`
|
|
4390
|
+
);
|
|
4391
|
+
}
|
|
4392
|
+
if (!config.enableCheckpoints && !config.verbose) {
|
|
4393
|
+
console.warn(
|
|
4394
|
+
`[EventualConsistency] INFO: checkpoints.enabled is false. Checkpoints improve performance in high-volume scenarios by creating snapshots. Consider enabling for production use.`
|
|
4395
|
+
);
|
|
4396
|
+
}
|
|
4397
|
+
}
|
|
4398
|
+
function logInitialization(config, fieldHandlers, timezoneAutoDetected) {
|
|
4399
|
+
if (!config.verbose) return;
|
|
4400
|
+
const totalFields = Array.from(fieldHandlers.values()).reduce((sum, handlers) => sum + handlers.size, 0);
|
|
4401
|
+
console.log(
|
|
4402
|
+
`[EventualConsistency] Initialized with ${fieldHandlers.size} resource(s), ${totalFields} field(s) total`
|
|
4403
|
+
);
|
|
4404
|
+
if (timezoneAutoDetected) {
|
|
4405
|
+
console.log(
|
|
4406
|
+
`[EventualConsistency] Using timezone: ${config.cohort.timezone} (${process.env.TZ ? "from TZ env var" : "default UTC"})`
|
|
4407
|
+
);
|
|
4408
|
+
}
|
|
4409
|
+
}
|
|
4410
|
+
|
|
4411
|
+
function detectTimezone() {
|
|
4412
|
+
if (process.env.TZ) {
|
|
4413
|
+
return process.env.TZ;
|
|
4414
|
+
}
|
|
4415
|
+
return "UTC";
|
|
4416
|
+
}
|
|
4417
|
+
function getTimezoneOffset(timezone, verbose = false) {
|
|
4418
|
+
try {
|
|
4419
|
+
const now = /* @__PURE__ */ new Date();
|
|
4420
|
+
const utcDate = new Date(now.toLocaleString("en-US", { timeZone: "UTC" }));
|
|
4421
|
+
const tzDate = new Date(now.toLocaleString("en-US", { timeZone: timezone }));
|
|
4422
|
+
return tzDate.getTime() - utcDate.getTime();
|
|
4423
|
+
} catch (err) {
|
|
4424
|
+
const offsets = {
|
|
4425
|
+
"UTC": 0,
|
|
4426
|
+
"America/New_York": -5 * 36e5,
|
|
4427
|
+
"America/Chicago": -6 * 36e5,
|
|
4428
|
+
"America/Denver": -7 * 36e5,
|
|
4429
|
+
"America/Los_Angeles": -8 * 36e5,
|
|
4430
|
+
"America/Sao_Paulo": -3 * 36e5,
|
|
4431
|
+
"Europe/London": 0,
|
|
4432
|
+
"Europe/Paris": 1 * 36e5,
|
|
4433
|
+
"Europe/Berlin": 1 * 36e5,
|
|
4434
|
+
"Asia/Tokyo": 9 * 36e5,
|
|
4435
|
+
"Asia/Shanghai": 8 * 36e5,
|
|
4436
|
+
"Australia/Sydney": 10 * 36e5
|
|
4355
4437
|
};
|
|
4356
|
-
|
|
4357
|
-
|
|
4358
|
-
|
|
4359
|
-
|
|
4360
|
-
|
|
4361
|
-
|
|
4438
|
+
if (verbose && !offsets[timezone]) {
|
|
4439
|
+
console.warn(
|
|
4440
|
+
`[EventualConsistency] Unknown timezone '${timezone}', using UTC. Consider using a valid IANA timezone (e.g., 'America/New_York')`
|
|
4441
|
+
);
|
|
4442
|
+
}
|
|
4443
|
+
return offsets[timezone] || 0;
|
|
4444
|
+
}
|
|
4445
|
+
}
|
|
4446
|
+
function getCohortInfo(date, timezone, verbose = false) {
|
|
4447
|
+
const offset = getTimezoneOffset(timezone, verbose);
|
|
4448
|
+
const localDate = new Date(date.getTime() + offset);
|
|
4449
|
+
const year = localDate.getFullYear();
|
|
4450
|
+
const month = String(localDate.getMonth() + 1).padStart(2, "0");
|
|
4451
|
+
const day = String(localDate.getDate()).padStart(2, "0");
|
|
4452
|
+
const hour = String(localDate.getHours()).padStart(2, "0");
|
|
4453
|
+
return {
|
|
4454
|
+
date: `${year}-${month}-${day}`,
|
|
4455
|
+
hour: `${year}-${month}-${day}T${hour}`,
|
|
4456
|
+
// ISO-like format for hour partition
|
|
4457
|
+
month: `${year}-${month}`
|
|
4458
|
+
};
|
|
4459
|
+
}
|
|
4460
|
+
function createSyntheticSetTransaction(currentValue) {
|
|
4461
|
+
return {
|
|
4462
|
+
id: "__synthetic__",
|
|
4463
|
+
operation: "set",
|
|
4464
|
+
value: currentValue,
|
|
4465
|
+
timestamp: (/* @__PURE__ */ new Date(0)).toISOString(),
|
|
4466
|
+
synthetic: true
|
|
4467
|
+
};
|
|
4468
|
+
}
|
|
4469
|
+
function createFieldHandler(resourceName, fieldName) {
|
|
4470
|
+
return {
|
|
4471
|
+
resource: resourceName,
|
|
4472
|
+
field: fieldName,
|
|
4473
|
+
transactionResource: null,
|
|
4474
|
+
targetResource: null,
|
|
4475
|
+
analyticsResource: null,
|
|
4476
|
+
lockResource: null,
|
|
4477
|
+
checkpointResource: null,
|
|
4478
|
+
consolidationTimer: null,
|
|
4479
|
+
gcTimer: null,
|
|
4480
|
+
pendingTransactions: /* @__PURE__ */ new Map(),
|
|
4481
|
+
deferredSetup: false
|
|
4482
|
+
};
|
|
4483
|
+
}
|
|
4484
|
+
function resolveFieldAndPlugin(resource, field, value) {
|
|
4485
|
+
if (!resource._eventualConsistencyPlugins) {
|
|
4486
|
+
throw new Error(`No eventual consistency plugins configured for this resource`);
|
|
4487
|
+
}
|
|
4488
|
+
const fieldPlugin = resource._eventualConsistencyPlugins[field];
|
|
4489
|
+
if (!fieldPlugin) {
|
|
4490
|
+
const availableFields = Object.keys(resource._eventualConsistencyPlugins).join(", ");
|
|
4491
|
+
throw new Error(
|
|
4492
|
+
`No eventual consistency plugin found for field "${field}". Available fields: ${availableFields}`
|
|
4493
|
+
);
|
|
4494
|
+
}
|
|
4495
|
+
return { field, value, plugin: fieldPlugin };
|
|
4496
|
+
}
|
|
4497
|
+
function groupByCohort(transactions, cohortField) {
|
|
4498
|
+
const groups = {};
|
|
4499
|
+
for (const txn of transactions) {
|
|
4500
|
+
const cohort = txn[cohortField];
|
|
4501
|
+
if (!cohort) continue;
|
|
4502
|
+
if (!groups[cohort]) {
|
|
4503
|
+
groups[cohort] = [];
|
|
4504
|
+
}
|
|
4505
|
+
groups[cohort].push(txn);
|
|
4506
|
+
}
|
|
4507
|
+
return groups;
|
|
4508
|
+
}
|
|
4509
|
+
|
|
4510
|
+
function createPartitionConfig() {
|
|
4511
|
+
const partitions = {
|
|
4512
|
+
// Composite partition by originalId + applied status
|
|
4513
|
+
// This is THE MOST CRITICAL optimization for consolidation!
|
|
4514
|
+
// Why: Consolidation always queries { originalId, applied: false }
|
|
4515
|
+
// Without this: Reads ALL transactions (applied + pending) and filters manually
|
|
4516
|
+
// With this: Reads ONLY pending transactions - can be 1000x faster!
|
|
4517
|
+
byOriginalIdAndApplied: {
|
|
4518
|
+
fields: {
|
|
4519
|
+
originalId: "string",
|
|
4520
|
+
applied: "boolean"
|
|
4362
4521
|
}
|
|
4363
|
-
|
|
4364
|
-
|
|
4365
|
-
|
|
4522
|
+
},
|
|
4523
|
+
// Partition by time cohorts for batch consolidation across many records
|
|
4524
|
+
byHour: {
|
|
4525
|
+
fields: {
|
|
4526
|
+
cohortHour: "string"
|
|
4527
|
+
}
|
|
4528
|
+
},
|
|
4529
|
+
byDay: {
|
|
4530
|
+
fields: {
|
|
4531
|
+
cohortDate: "string"
|
|
4532
|
+
}
|
|
4533
|
+
},
|
|
4534
|
+
byMonth: {
|
|
4535
|
+
fields: {
|
|
4536
|
+
cohortMonth: "string"
|
|
4366
4537
|
}
|
|
4367
|
-
this.fieldHandlers.set(resourceName, resourceHandlers);
|
|
4368
4538
|
}
|
|
4369
|
-
|
|
4539
|
+
};
|
|
4540
|
+
return partitions;
|
|
4541
|
+
}
|
|
4542
|
+
|
|
4543
|
+
async function createTransaction(handler, data, config) {
|
|
4544
|
+
const now = /* @__PURE__ */ new Date();
|
|
4545
|
+
const cohortInfo = getCohortInfo(now, config.cohort.timezone, config.verbose);
|
|
4546
|
+
const watermarkMs = config.consolidationWindow * 60 * 60 * 1e3;
|
|
4547
|
+
const watermarkTime = now.getTime() - watermarkMs;
|
|
4548
|
+
const cohortHourDate = /* @__PURE__ */ new Date(cohortInfo.hour + ":00:00Z");
|
|
4549
|
+
if (cohortHourDate.getTime() < watermarkTime) {
|
|
4550
|
+
const hoursLate = Math.floor((now.getTime() - cohortHourDate.getTime()) / (60 * 60 * 1e3));
|
|
4551
|
+
if (config.lateArrivalStrategy === "ignore") {
|
|
4552
|
+
if (config.verbose) {
|
|
4553
|
+
console.warn(
|
|
4554
|
+
`[EventualConsistency] Late arrival ignored: transaction for ${cohortInfo.hour} is ${hoursLate}h late (watermark: ${config.consolidationWindow}h)`
|
|
4555
|
+
);
|
|
4556
|
+
}
|
|
4557
|
+
return null;
|
|
4558
|
+
} else if (config.lateArrivalStrategy === "warn") {
|
|
4370
4559
|
console.warn(
|
|
4371
|
-
`[EventualConsistency]
|
|
4560
|
+
`[EventualConsistency] Late arrival detected: transaction for ${cohortInfo.hour} is ${hoursLate}h late (watermark: ${config.consolidationWindow}h). Processing anyway, but consolidation may not pick it up.`
|
|
4372
4561
|
);
|
|
4373
4562
|
}
|
|
4374
|
-
|
|
4375
|
-
|
|
4563
|
+
}
|
|
4564
|
+
const transaction = {
|
|
4565
|
+
id: idGenerator(),
|
|
4566
|
+
originalId: data.originalId,
|
|
4567
|
+
field: handler.field,
|
|
4568
|
+
value: data.value || 0,
|
|
4569
|
+
operation: data.operation || "set",
|
|
4570
|
+
timestamp: now.toISOString(),
|
|
4571
|
+
cohortDate: cohortInfo.date,
|
|
4572
|
+
cohortHour: cohortInfo.hour,
|
|
4573
|
+
cohortMonth: cohortInfo.month,
|
|
4574
|
+
source: data.source || "unknown",
|
|
4575
|
+
applied: false
|
|
4576
|
+
};
|
|
4577
|
+
if (config.batchTransactions) {
|
|
4578
|
+
handler.pendingTransactions.set(transaction.id, transaction);
|
|
4579
|
+
if (config.verbose) {
|
|
4376
4580
|
console.log(
|
|
4377
|
-
`[EventualConsistency]
|
|
4581
|
+
`[EventualConsistency] ${handler.resource}.${handler.field} - Transaction batched: ${data.operation} ${data.value} for ${data.originalId} (batch: ${handler.pendingTransactions.size}/${config.batchSize})`
|
|
4582
|
+
);
|
|
4583
|
+
}
|
|
4584
|
+
if (handler.pendingTransactions.size >= config.batchSize) {
|
|
4585
|
+
await flushPendingTransactions(handler);
|
|
4586
|
+
}
|
|
4587
|
+
} else {
|
|
4588
|
+
await handler.transactionResource.insert(transaction);
|
|
4589
|
+
if (config.verbose) {
|
|
4590
|
+
console.log(
|
|
4591
|
+
`[EventualConsistency] ${handler.resource}.${handler.field} - Transaction created: ${data.operation} ${data.value} for ${data.originalId} (cohort: ${cohortInfo.hour}, applied: false)`
|
|
4378
4592
|
);
|
|
4379
|
-
if (!options.cohort?.timezone) {
|
|
4380
|
-
console.log(
|
|
4381
|
-
`[EventualConsistency] Auto-detected timezone: ${this.config.cohort.timezone} (from ${process.env.TZ ? "TZ env var" : "system Intl API"})`
|
|
4382
|
-
);
|
|
4383
|
-
}
|
|
4384
4593
|
}
|
|
4385
4594
|
}
|
|
4386
|
-
|
|
4387
|
-
|
|
4388
|
-
|
|
4389
|
-
|
|
4390
|
-
|
|
4391
|
-
|
|
4392
|
-
|
|
4393
|
-
|
|
4394
|
-
|
|
4395
|
-
|
|
4396
|
-
|
|
4397
|
-
|
|
4398
|
-
|
|
4399
|
-
|
|
4400
|
-
|
|
4401
|
-
deferredSetup: false
|
|
4402
|
-
};
|
|
4595
|
+
return transaction;
|
|
4596
|
+
}
|
|
4597
|
+
async function flushPendingTransactions(handler) {
|
|
4598
|
+
if (handler.pendingTransactions.size === 0) return;
|
|
4599
|
+
const transactions = Array.from(handler.pendingTransactions.values());
|
|
4600
|
+
try {
|
|
4601
|
+
await Promise.all(
|
|
4602
|
+
transactions.map(
|
|
4603
|
+
(transaction) => handler.transactionResource.insert(transaction)
|
|
4604
|
+
)
|
|
4605
|
+
);
|
|
4606
|
+
handler.pendingTransactions.clear();
|
|
4607
|
+
} catch (error) {
|
|
4608
|
+
console.error("Failed to flush pending transactions:", error);
|
|
4609
|
+
throw error;
|
|
4403
4610
|
}
|
|
4404
|
-
|
|
4405
|
-
|
|
4406
|
-
|
|
4407
|
-
|
|
4408
|
-
|
|
4409
|
-
|
|
4410
|
-
|
|
4411
|
-
|
|
4412
|
-
|
|
4413
|
-
|
|
4414
|
-
|
|
4415
|
-
|
|
4416
|
-
|
|
4417
|
-
|
|
4611
|
+
}
|
|
4612
|
+
|
|
4613
|
+
async function cleanupStaleLocks(lockResource, config) {
|
|
4614
|
+
const now = Date.now();
|
|
4615
|
+
const lockTimeoutMs = config.lockTimeout * 1e3;
|
|
4616
|
+
const cutoffTime = now - lockTimeoutMs;
|
|
4617
|
+
const cleanupLockId = `lock-cleanup-${config.resource}-${config.field}`;
|
|
4618
|
+
const [lockAcquired] = await tryFn(
|
|
4619
|
+
() => lockResource.insert({
|
|
4620
|
+
id: cleanupLockId,
|
|
4621
|
+
lockedAt: Date.now(),
|
|
4622
|
+
workerId: process.pid ? String(process.pid) : "unknown"
|
|
4623
|
+
})
|
|
4624
|
+
);
|
|
4625
|
+
if (!lockAcquired) {
|
|
4626
|
+
if (config.verbose) {
|
|
4627
|
+
console.log(`[EventualConsistency] Lock cleanup already running in another container`);
|
|
4418
4628
|
}
|
|
4629
|
+
return;
|
|
4419
4630
|
}
|
|
4420
|
-
|
|
4421
|
-
|
|
4422
|
-
|
|
4423
|
-
|
|
4424
|
-
|
|
4425
|
-
|
|
4426
|
-
|
|
4427
|
-
|
|
4428
|
-
|
|
4429
|
-
|
|
4430
|
-
|
|
4431
|
-
|
|
4432
|
-
|
|
4433
|
-
|
|
4434
|
-
|
|
4435
|
-
|
|
4436
|
-
|
|
4437
|
-
|
|
4438
|
-
|
|
4631
|
+
try {
|
|
4632
|
+
const [ok, err, locks] = await tryFn(() => lockResource.list());
|
|
4633
|
+
if (!ok || !locks || locks.length === 0) return;
|
|
4634
|
+
const staleLocks = locks.filter(
|
|
4635
|
+
(lock) => lock.id !== cleanupLockId && lock.lockedAt < cutoffTime
|
|
4636
|
+
);
|
|
4637
|
+
if (staleLocks.length === 0) return;
|
|
4638
|
+
if (config.verbose) {
|
|
4639
|
+
console.log(`[EventualConsistency] Cleaning up ${staleLocks.length} stale locks`);
|
|
4640
|
+
}
|
|
4641
|
+
const { results, errors } = await promisePool.PromisePool.for(staleLocks).withConcurrency(5).process(async (lock) => {
|
|
4642
|
+
const [deleted] = await tryFn(() => lockResource.delete(lock.id));
|
|
4643
|
+
return deleted;
|
|
4644
|
+
});
|
|
4645
|
+
if (errors && errors.length > 0 && config.verbose) {
|
|
4646
|
+
console.warn(`[EventualConsistency] ${errors.length} stale locks failed to delete`);
|
|
4647
|
+
}
|
|
4648
|
+
} catch (error) {
|
|
4649
|
+
if (config.verbose) {
|
|
4650
|
+
console.warn(`[EventualConsistency] Error cleaning up stale locks:`, error.message);
|
|
4651
|
+
}
|
|
4652
|
+
} finally {
|
|
4653
|
+
await tryFn(() => lockResource.delete(cleanupLockId));
|
|
4439
4654
|
}
|
|
4440
|
-
|
|
4441
|
-
|
|
4442
|
-
|
|
4443
|
-
|
|
4444
|
-
|
|
4445
|
-
|
|
4446
|
-
|
|
4447
|
-
|
|
4448
|
-
|
|
4449
|
-
|
|
4450
|
-
|
|
4451
|
-
|
|
4452
|
-
|
|
4453
|
-
|
|
4454
|
-
|
|
4455
|
-
|
|
4456
|
-
|
|
4457
|
-
|
|
4458
|
-
|
|
4459
|
-
|
|
4460
|
-
timestamp: "string|required",
|
|
4461
|
-
cohortDate: "string|required",
|
|
4462
|
-
cohortHour: "string|required",
|
|
4463
|
-
cohortMonth: "string|optional",
|
|
4464
|
-
source: "string|optional",
|
|
4465
|
-
applied: "boolean|optional"
|
|
4466
|
-
},
|
|
4467
|
-
behavior: "body-overflow",
|
|
4468
|
-
timestamps: true,
|
|
4469
|
-
partitions: partitionConfig,
|
|
4470
|
-
asyncPartitions: true,
|
|
4471
|
-
createdBy: "EventualConsistencyPlugin"
|
|
4472
|
-
})
|
|
4655
|
+
}
|
|
4656
|
+
|
|
4657
|
+
function startConsolidationTimer(handler, resourceName, fieldName, runConsolidationCallback, config) {
|
|
4658
|
+
const intervalMs = config.consolidationInterval * 1e3;
|
|
4659
|
+
if (config.verbose) {
|
|
4660
|
+
const nextRun = new Date(Date.now() + intervalMs);
|
|
4661
|
+
console.log(
|
|
4662
|
+
`[EventualConsistency] ${resourceName}.${fieldName} - Consolidation timer started. Next run at ${nextRun.toISOString()} (every ${config.consolidationInterval}s)`
|
|
4663
|
+
);
|
|
4664
|
+
}
|
|
4665
|
+
handler.consolidationTimer = setInterval(async () => {
|
|
4666
|
+
await runConsolidationCallback(handler, resourceName, fieldName);
|
|
4667
|
+
}, intervalMs);
|
|
4668
|
+
return handler.consolidationTimer;
|
|
4669
|
+
}
|
|
4670
|
+
async function runConsolidation(transactionResource, consolidateRecordFn, emitFn, config) {
|
|
4671
|
+
const startTime = Date.now();
|
|
4672
|
+
if (config.verbose) {
|
|
4673
|
+
console.log(
|
|
4674
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Starting consolidation run at ${(/* @__PURE__ */ new Date()).toISOString()}`
|
|
4473
4675
|
);
|
|
4474
|
-
|
|
4475
|
-
|
|
4676
|
+
}
|
|
4677
|
+
try {
|
|
4678
|
+
const now = /* @__PURE__ */ new Date();
|
|
4679
|
+
const hoursToCheck = config.consolidationWindow || 24;
|
|
4680
|
+
const cohortHours = [];
|
|
4681
|
+
for (let i = 0; i < hoursToCheck; i++) {
|
|
4682
|
+
const date = new Date(now.getTime() - i * 60 * 60 * 1e3);
|
|
4683
|
+
const cohortInfo = getCohortInfo(date, config.cohort.timezone, config.verbose);
|
|
4684
|
+
cohortHours.push(cohortInfo.hour);
|
|
4476
4685
|
}
|
|
4477
|
-
|
|
4478
|
-
|
|
4479
|
-
|
|
4480
|
-
|
|
4481
|
-
|
|
4482
|
-
|
|
4483
|
-
|
|
4484
|
-
|
|
4485
|
-
|
|
4486
|
-
|
|
4487
|
-
|
|
4488
|
-
|
|
4489
|
-
|
|
4686
|
+
if (config.verbose) {
|
|
4687
|
+
console.log(
|
|
4688
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Querying ${hoursToCheck} hour partitions for pending transactions...`
|
|
4689
|
+
);
|
|
4690
|
+
}
|
|
4691
|
+
const transactionsByHour = await Promise.all(
|
|
4692
|
+
cohortHours.map(async (cohortHour) => {
|
|
4693
|
+
const [ok, err, txns] = await tryFn(
|
|
4694
|
+
() => transactionResource.query({
|
|
4695
|
+
cohortHour,
|
|
4696
|
+
applied: false
|
|
4697
|
+
})
|
|
4698
|
+
);
|
|
4699
|
+
return ok ? txns : [];
|
|
4490
4700
|
})
|
|
4491
4701
|
);
|
|
4492
|
-
|
|
4493
|
-
|
|
4702
|
+
const transactions = transactionsByHour.flat();
|
|
4703
|
+
if (transactions.length === 0) {
|
|
4704
|
+
if (config.verbose) {
|
|
4705
|
+
console.log(
|
|
4706
|
+
`[EventualConsistency] ${config.resource}.${config.field} - No pending transactions found. Next run in ${config.consolidationInterval}s`
|
|
4707
|
+
);
|
|
4708
|
+
}
|
|
4709
|
+
return;
|
|
4710
|
+
}
|
|
4711
|
+
const uniqueIds = [...new Set(transactions.map((t) => t.originalId))];
|
|
4712
|
+
if (config.verbose) {
|
|
4713
|
+
console.log(
|
|
4714
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Found ${transactions.length} pending transactions for ${uniqueIds.length} records. Consolidating with concurrency=${config.consolidationConcurrency}...`
|
|
4715
|
+
);
|
|
4494
4716
|
}
|
|
4495
|
-
|
|
4496
|
-
|
|
4497
|
-
|
|
4717
|
+
const { results, errors } = await promisePool.PromisePool.for(uniqueIds).withConcurrency(config.consolidationConcurrency).process(async (id) => {
|
|
4718
|
+
return await consolidateRecordFn(id);
|
|
4719
|
+
});
|
|
4720
|
+
const duration = Date.now() - startTime;
|
|
4721
|
+
if (errors && errors.length > 0) {
|
|
4722
|
+
console.error(
|
|
4723
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Consolidation completed with ${errors.length} errors in ${duration}ms:`,
|
|
4724
|
+
errors
|
|
4725
|
+
);
|
|
4498
4726
|
}
|
|
4499
|
-
this._addHelperMethodsForHandler(handler);
|
|
4500
4727
|
if (config.verbose) {
|
|
4501
4728
|
console.log(
|
|
4502
|
-
`[EventualConsistency] ${
|
|
4729
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Consolidation complete: ${results.length} records consolidated in ${duration}ms (${errors.length} errors). Next run in ${config.consolidationInterval}s`
|
|
4503
4730
|
);
|
|
4504
4731
|
}
|
|
4505
|
-
|
|
4506
|
-
|
|
4507
|
-
|
|
4508
|
-
|
|
4509
|
-
|
|
4510
|
-
|
|
4511
|
-
|
|
4512
|
-
|
|
4513
|
-
|
|
4514
|
-
|
|
4515
|
-
|
|
4516
|
-
|
|
4517
|
-
|
|
4518
|
-
|
|
4519
|
-
|
|
4520
|
-
cohort: "string|required",
|
|
4521
|
-
transactionCount: "number|required",
|
|
4522
|
-
totalValue: "number|required",
|
|
4523
|
-
avgValue: "number|required",
|
|
4524
|
-
minValue: "number|required",
|
|
4525
|
-
maxValue: "number|required",
|
|
4526
|
-
operations: "object|optional",
|
|
4527
|
-
recordCount: "number|required",
|
|
4528
|
-
consolidatedAt: "string|required",
|
|
4529
|
-
updatedAt: "string|required"
|
|
4530
|
-
},
|
|
4531
|
-
behavior: "body-overflow",
|
|
4532
|
-
timestamps: false,
|
|
4533
|
-
createdBy: "EventualConsistencyPlugin"
|
|
4534
|
-
})
|
|
4732
|
+
if (emitFn) {
|
|
4733
|
+
emitFn("eventual-consistency.consolidated", {
|
|
4734
|
+
resource: config.resource,
|
|
4735
|
+
field: config.field,
|
|
4736
|
+
recordCount: uniqueIds.length,
|
|
4737
|
+
successCount: results.length,
|
|
4738
|
+
errorCount: errors.length,
|
|
4739
|
+
duration
|
|
4740
|
+
});
|
|
4741
|
+
}
|
|
4742
|
+
} catch (error) {
|
|
4743
|
+
const duration = Date.now() - startTime;
|
|
4744
|
+
console.error(
|
|
4745
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Consolidation error after ${duration}ms:`,
|
|
4746
|
+
error
|
|
4535
4747
|
);
|
|
4536
|
-
if (
|
|
4537
|
-
|
|
4748
|
+
if (emitFn) {
|
|
4749
|
+
emitFn("eventual-consistency.consolidation-error", error);
|
|
4538
4750
|
}
|
|
4539
|
-
handler.analyticsResource = ok ? analyticsResource : this.database.resources[analyticsResourceName];
|
|
4540
4751
|
}
|
|
4541
|
-
|
|
4542
|
-
|
|
4543
|
-
|
|
4544
|
-
|
|
4545
|
-
|
|
4546
|
-
|
|
4547
|
-
|
|
4548
|
-
|
|
4549
|
-
|
|
4550
|
-
}
|
|
4551
|
-
|
|
4552
|
-
|
|
4553
|
-
|
|
4752
|
+
}
|
|
4753
|
+
async function consolidateRecord(originalId, transactionResource, targetResource, lockResource, analyticsResource, updateAnalyticsFn, config) {
|
|
4754
|
+
await cleanupStaleLocks(lockResource, config);
|
|
4755
|
+
const lockId = `lock-${originalId}`;
|
|
4756
|
+
const [lockAcquired, lockErr, lock] = await tryFn(
|
|
4757
|
+
() => lockResource.insert({
|
|
4758
|
+
id: lockId,
|
|
4759
|
+
lockedAt: Date.now(),
|
|
4760
|
+
workerId: process.pid ? String(process.pid) : "unknown"
|
|
4761
|
+
})
|
|
4762
|
+
);
|
|
4763
|
+
if (!lockAcquired) {
|
|
4764
|
+
if (config.verbose) {
|
|
4765
|
+
console.log(`[EventualConsistency] Lock for ${originalId} already held, skipping`);
|
|
4554
4766
|
}
|
|
4767
|
+
const [recordOk, recordErr, record] = await tryFn(
|
|
4768
|
+
() => targetResource.get(originalId)
|
|
4769
|
+
);
|
|
4770
|
+
return recordOk && record ? record[config.field] || 0 : 0;
|
|
4555
4771
|
}
|
|
4556
|
-
|
|
4557
|
-
|
|
4558
|
-
|
|
4559
|
-
|
|
4560
|
-
|
|
4561
|
-
|
|
4562
|
-
|
|
4563
|
-
|
|
4564
|
-
|
|
4565
|
-
|
|
4566
|
-
|
|
4567
|
-
|
|
4568
|
-
|
|
4569
|
-
|
|
4570
|
-
}
|
|
4571
|
-
|
|
4772
|
+
try {
|
|
4773
|
+
const [ok, err, transactions] = await tryFn(
|
|
4774
|
+
() => transactionResource.query({
|
|
4775
|
+
originalId,
|
|
4776
|
+
applied: false
|
|
4777
|
+
})
|
|
4778
|
+
);
|
|
4779
|
+
if (!ok || !transactions || transactions.length === 0) {
|
|
4780
|
+
const [recordOk, recordErr, record] = await tryFn(
|
|
4781
|
+
() => targetResource.get(originalId)
|
|
4782
|
+
);
|
|
4783
|
+
const currentValue2 = recordOk && record ? record[config.field] || 0 : 0;
|
|
4784
|
+
if (config.verbose) {
|
|
4785
|
+
console.log(
|
|
4786
|
+
`[EventualConsistency] ${config.resource}.${config.field} - No pending transactions for ${originalId}, skipping`
|
|
4787
|
+
);
|
|
4572
4788
|
}
|
|
4789
|
+
return currentValue2;
|
|
4573
4790
|
}
|
|
4574
|
-
|
|
4575
|
-
|
|
4576
|
-
|
|
4577
|
-
|
|
4578
|
-
|
|
4579
|
-
|
|
4580
|
-
|
|
4791
|
+
const [appliedOk, appliedErr, appliedTransactions] = await tryFn(
|
|
4792
|
+
() => transactionResource.query({
|
|
4793
|
+
originalId,
|
|
4794
|
+
applied: true
|
|
4795
|
+
})
|
|
4796
|
+
);
|
|
4797
|
+
let currentValue = 0;
|
|
4798
|
+
if (appliedOk && appliedTransactions && appliedTransactions.length > 0) {
|
|
4799
|
+
const [recordExistsOk, recordExistsErr, recordExists] = await tryFn(
|
|
4800
|
+
() => targetResource.get(originalId)
|
|
4801
|
+
);
|
|
4802
|
+
if (!recordExistsOk || !recordExists) {
|
|
4803
|
+
if (config.verbose) {
|
|
4804
|
+
console.log(
|
|
4805
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Record ${originalId} doesn't exist, deleting ${appliedTransactions.length} old applied transactions`
|
|
4806
|
+
);
|
|
4581
4807
|
}
|
|
4582
|
-
|
|
4583
|
-
|
|
4584
|
-
|
|
4808
|
+
const { results, errors } = await promisePool.PromisePool.for(appliedTransactions).withConcurrency(10).process(async (txn) => {
|
|
4809
|
+
const [deleted] = await tryFn(() => transactionResource.delete(txn.id));
|
|
4810
|
+
return deleted;
|
|
4811
|
+
});
|
|
4812
|
+
if (config.verbose && errors && errors.length > 0) {
|
|
4813
|
+
console.warn(
|
|
4814
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Failed to delete ${errors.length} old applied transactions`
|
|
4815
|
+
);
|
|
4585
4816
|
}
|
|
4586
|
-
|
|
4587
|
-
|
|
4817
|
+
currentValue = 0;
|
|
4818
|
+
} else {
|
|
4819
|
+
appliedTransactions.sort(
|
|
4820
|
+
(a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()
|
|
4821
|
+
);
|
|
4822
|
+
const hasSetInApplied = appliedTransactions.some((t) => t.operation === "set");
|
|
4823
|
+
if (!hasSetInApplied) {
|
|
4824
|
+
const recordValue = recordExists[config.field] || 0;
|
|
4825
|
+
let appliedDelta = 0;
|
|
4826
|
+
for (const t of appliedTransactions) {
|
|
4827
|
+
if (t.operation === "add") appliedDelta += t.value;
|
|
4828
|
+
else if (t.operation === "sub") appliedDelta -= t.value;
|
|
4829
|
+
}
|
|
4830
|
+
const baseValue = recordValue - appliedDelta;
|
|
4831
|
+
const hasExistingAnchor = appliedTransactions.some((t) => t.source === "anchor");
|
|
4832
|
+
if (baseValue !== 0 && !hasExistingAnchor) {
|
|
4833
|
+
const firstTransactionDate = new Date(appliedTransactions[0].timestamp);
|
|
4834
|
+
const cohortInfo = getCohortInfo(firstTransactionDate, config.cohort.timezone, config.verbose);
|
|
4835
|
+
const anchorTransaction = {
|
|
4836
|
+
id: idGenerator(),
|
|
4837
|
+
originalId,
|
|
4838
|
+
field: config.field,
|
|
4839
|
+
value: baseValue,
|
|
4840
|
+
operation: "set",
|
|
4841
|
+
timestamp: new Date(firstTransactionDate.getTime() - 1).toISOString(),
|
|
4842
|
+
// 1ms before first txn to ensure it's first
|
|
4843
|
+
cohortDate: cohortInfo.date,
|
|
4844
|
+
cohortHour: cohortInfo.hour,
|
|
4845
|
+
cohortMonth: cohortInfo.month,
|
|
4846
|
+
source: "anchor",
|
|
4847
|
+
applied: true
|
|
4848
|
+
};
|
|
4849
|
+
await transactionResource.insert(anchorTransaction);
|
|
4850
|
+
appliedTransactions.unshift(anchorTransaction);
|
|
4851
|
+
}
|
|
4588
4852
|
}
|
|
4589
|
-
|
|
4590
|
-
resource: resourceName,
|
|
4591
|
-
field: fieldName
|
|
4592
|
-
});
|
|
4853
|
+
currentValue = config.reducer(appliedTransactions);
|
|
4593
4854
|
}
|
|
4594
|
-
}
|
|
4595
|
-
|
|
4596
|
-
|
|
4597
|
-
|
|
4598
|
-
|
|
4599
|
-
|
|
4600
|
-
|
|
4601
|
-
|
|
4602
|
-
|
|
4603
|
-
|
|
4604
|
-
|
|
4605
|
-
|
|
4855
|
+
} else {
|
|
4856
|
+
const [recordOk, recordErr, record] = await tryFn(
|
|
4857
|
+
() => targetResource.get(originalId)
|
|
4858
|
+
);
|
|
4859
|
+
currentValue = recordOk && record ? record[config.field] || 0 : 0;
|
|
4860
|
+
if (currentValue !== 0) {
|
|
4861
|
+
let anchorTimestamp;
|
|
4862
|
+
if (transactions && transactions.length > 0) {
|
|
4863
|
+
const firstPendingDate = new Date(transactions[0].timestamp);
|
|
4864
|
+
anchorTimestamp = new Date(firstPendingDate.getTime() - 1).toISOString();
|
|
4865
|
+
} else {
|
|
4866
|
+
anchorTimestamp = (/* @__PURE__ */ new Date()).toISOString();
|
|
4606
4867
|
}
|
|
4607
|
-
|
|
4608
|
-
|
|
4609
|
-
|
|
4610
|
-
|
|
4868
|
+
const cohortInfo = getCohortInfo(new Date(anchorTimestamp), config.cohort.timezone, config.verbose);
|
|
4869
|
+
const anchorTransaction = {
|
|
4870
|
+
id: idGenerator(),
|
|
4871
|
+
originalId,
|
|
4872
|
+
field: config.field,
|
|
4873
|
+
value: currentValue,
|
|
4874
|
+
operation: "set",
|
|
4875
|
+
timestamp: anchorTimestamp,
|
|
4876
|
+
cohortDate: cohortInfo.date,
|
|
4877
|
+
cohortHour: cohortInfo.hour,
|
|
4878
|
+
cohortMonth: cohortInfo.month,
|
|
4879
|
+
source: "anchor",
|
|
4880
|
+
applied: true
|
|
4881
|
+
};
|
|
4882
|
+
await transactionResource.insert(anchorTransaction);
|
|
4883
|
+
if (config.verbose) {
|
|
4884
|
+
console.log(
|
|
4885
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Created anchor transaction for ${originalId} with base value ${currentValue}`
|
|
4886
|
+
);
|
|
4611
4887
|
}
|
|
4612
4888
|
}
|
|
4613
|
-
};
|
|
4614
|
-
return partitions;
|
|
4615
|
-
}
|
|
4616
|
-
/**
|
|
4617
|
-
* Auto-detect timezone from environment or system
|
|
4618
|
-
* @private
|
|
4619
|
-
*/
|
|
4620
|
-
_detectTimezone() {
|
|
4621
|
-
if (process.env.TZ) {
|
|
4622
|
-
return process.env.TZ;
|
|
4623
4889
|
}
|
|
4624
|
-
|
|
4625
|
-
|
|
4626
|
-
|
|
4627
|
-
|
|
4628
|
-
}
|
|
4629
|
-
} catch (err) {
|
|
4890
|
+
if (config.verbose) {
|
|
4891
|
+
console.log(
|
|
4892
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Consolidating ${originalId}: ${transactions.length} pending transactions (current: ${currentValue} from ${appliedOk && appliedTransactions?.length > 0 ? "applied transactions" : "record"})`
|
|
4893
|
+
);
|
|
4630
4894
|
}
|
|
4631
|
-
|
|
4632
|
-
|
|
4633
|
-
|
|
4634
|
-
|
|
4635
|
-
|
|
4636
|
-
|
|
4637
|
-
_resolveFieldAndPlugin(resource, field, value) {
|
|
4638
|
-
if (!resource._eventualConsistencyPlugins) {
|
|
4639
|
-
throw new Error(`No eventual consistency plugins configured for this resource`);
|
|
4895
|
+
transactions.sort(
|
|
4896
|
+
(a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()
|
|
4897
|
+
);
|
|
4898
|
+
const hasSetOperation = transactions.some((t) => t.operation === "set");
|
|
4899
|
+
if (currentValue !== 0 && !hasSetOperation) {
|
|
4900
|
+
transactions.unshift(createSyntheticSetTransaction(currentValue));
|
|
4640
4901
|
}
|
|
4641
|
-
const
|
|
4642
|
-
if (
|
|
4643
|
-
|
|
4644
|
-
|
|
4645
|
-
`No eventual consistency plugin found for field "${field}". Available fields: ${availableFields}`
|
|
4902
|
+
const consolidatedValue = config.reducer(transactions);
|
|
4903
|
+
if (config.verbose) {
|
|
4904
|
+
console.log(
|
|
4905
|
+
`[EventualConsistency] ${config.resource}.${config.field} - ${originalId}: ${currentValue} \u2192 ${consolidatedValue} (${consolidatedValue > currentValue ? "+" : ""}${consolidatedValue - currentValue})`
|
|
4646
4906
|
);
|
|
4647
4907
|
}
|
|
4648
|
-
|
|
4649
|
-
|
|
4650
|
-
|
|
4651
|
-
|
|
4652
|
-
|
|
4653
|
-
|
|
4654
|
-
|
|
4655
|
-
|
|
4656
|
-
return consolidatedValue;
|
|
4657
|
-
}
|
|
4658
|
-
/**
|
|
4659
|
-
* Create synthetic 'set' transaction from current value
|
|
4660
|
-
* @private
|
|
4661
|
-
*/
|
|
4662
|
-
_createSyntheticSetTransaction(currentValue) {
|
|
4663
|
-
return {
|
|
4664
|
-
id: "__synthetic__",
|
|
4665
|
-
operation: "set",
|
|
4666
|
-
value: currentValue,
|
|
4667
|
-
timestamp: (/* @__PURE__ */ new Date(0)).toISOString(),
|
|
4668
|
-
synthetic: true
|
|
4669
|
-
};
|
|
4670
|
-
}
|
|
4671
|
-
addHelperMethods() {
|
|
4672
|
-
const firstResource = this.fieldHandlers.values().next().value;
|
|
4673
|
-
if (!firstResource) return;
|
|
4674
|
-
const firstHandler = firstResource.values().next().value;
|
|
4675
|
-
if (!firstHandler || !firstHandler.targetResource) return;
|
|
4676
|
-
const resource = firstHandler.targetResource;
|
|
4677
|
-
const plugin = this;
|
|
4678
|
-
resource.set = async (id, field, value) => {
|
|
4679
|
-
const { plugin: handler } = plugin._resolveFieldAndPlugin(resource, field, value);
|
|
4680
|
-
const now = /* @__PURE__ */ new Date();
|
|
4681
|
-
const cohortInfo = plugin.getCohortInfo(now);
|
|
4682
|
-
const transaction = {
|
|
4683
|
-
id: idGenerator(),
|
|
4684
|
-
originalId: id,
|
|
4685
|
-
field: handler.field,
|
|
4686
|
-
value,
|
|
4687
|
-
operation: "set",
|
|
4688
|
-
timestamp: now.toISOString(),
|
|
4689
|
-
cohortDate: cohortInfo.date,
|
|
4690
|
-
cohortHour: cohortInfo.hour,
|
|
4691
|
-
cohortMonth: cohortInfo.month,
|
|
4692
|
-
source: "set",
|
|
4693
|
-
applied: false
|
|
4694
|
-
};
|
|
4695
|
-
await handler.transactionResource.insert(transaction);
|
|
4696
|
-
if (plugin.config.mode === "sync") {
|
|
4697
|
-
const oldResource = plugin.config.resource;
|
|
4698
|
-
const oldField = plugin.config.field;
|
|
4699
|
-
const oldTransactionResource = plugin.transactionResource;
|
|
4700
|
-
const oldTargetResource = plugin.targetResource;
|
|
4701
|
-
const oldLockResource = plugin.lockResource;
|
|
4702
|
-
const oldAnalyticsResource = plugin.analyticsResource;
|
|
4703
|
-
plugin.config.resource = handler.resource;
|
|
4704
|
-
plugin.config.field = handler.field;
|
|
4705
|
-
plugin.transactionResource = handler.transactionResource;
|
|
4706
|
-
plugin.targetResource = handler.targetResource;
|
|
4707
|
-
plugin.lockResource = handler.lockResource;
|
|
4708
|
-
plugin.analyticsResource = handler.analyticsResource;
|
|
4709
|
-
const result = await plugin._syncModeConsolidate(id, field);
|
|
4710
|
-
plugin.config.resource = oldResource;
|
|
4711
|
-
plugin.config.field = oldField;
|
|
4712
|
-
plugin.transactionResource = oldTransactionResource;
|
|
4713
|
-
plugin.targetResource = oldTargetResource;
|
|
4714
|
-
plugin.lockResource = oldLockResource;
|
|
4715
|
-
plugin.analyticsResource = oldAnalyticsResource;
|
|
4716
|
-
return result;
|
|
4717
|
-
}
|
|
4718
|
-
return value;
|
|
4719
|
-
};
|
|
4720
|
-
resource.add = async (id, field, amount) => {
|
|
4721
|
-
const { plugin: handler } = plugin._resolveFieldAndPlugin(resource, field, amount);
|
|
4722
|
-
const now = /* @__PURE__ */ new Date();
|
|
4723
|
-
const cohortInfo = plugin.getCohortInfo(now);
|
|
4724
|
-
const transaction = {
|
|
4725
|
-
id: idGenerator(),
|
|
4726
|
-
originalId: id,
|
|
4727
|
-
field: handler.field,
|
|
4728
|
-
value: amount,
|
|
4729
|
-
operation: "add",
|
|
4730
|
-
timestamp: now.toISOString(),
|
|
4731
|
-
cohortDate: cohortInfo.date,
|
|
4732
|
-
cohortHour: cohortInfo.hour,
|
|
4733
|
-
cohortMonth: cohortInfo.month,
|
|
4734
|
-
source: "add",
|
|
4735
|
-
applied: false
|
|
4736
|
-
};
|
|
4737
|
-
await handler.transactionResource.insert(transaction);
|
|
4738
|
-
if (plugin.config.mode === "sync") {
|
|
4739
|
-
const oldResource = plugin.config.resource;
|
|
4740
|
-
const oldField = plugin.config.field;
|
|
4741
|
-
const oldTransactionResource = plugin.transactionResource;
|
|
4742
|
-
const oldTargetResource = plugin.targetResource;
|
|
4743
|
-
const oldLockResource = plugin.lockResource;
|
|
4744
|
-
const oldAnalyticsResource = plugin.analyticsResource;
|
|
4745
|
-
plugin.config.resource = handler.resource;
|
|
4746
|
-
plugin.config.field = handler.field;
|
|
4747
|
-
plugin.transactionResource = handler.transactionResource;
|
|
4748
|
-
plugin.targetResource = handler.targetResource;
|
|
4749
|
-
plugin.lockResource = handler.lockResource;
|
|
4750
|
-
plugin.analyticsResource = handler.analyticsResource;
|
|
4751
|
-
const result = await plugin._syncModeConsolidate(id, field);
|
|
4752
|
-
plugin.config.resource = oldResource;
|
|
4753
|
-
plugin.config.field = oldField;
|
|
4754
|
-
plugin.transactionResource = oldTransactionResource;
|
|
4755
|
-
plugin.targetResource = oldTargetResource;
|
|
4756
|
-
plugin.lockResource = oldLockResource;
|
|
4757
|
-
plugin.analyticsResource = oldAnalyticsResource;
|
|
4758
|
-
return result;
|
|
4759
|
-
}
|
|
4760
|
-
const [ok, err, record] = await tryFn(() => handler.targetResource.get(id));
|
|
4761
|
-
const currentValue = ok && record ? record[field] || 0 : 0;
|
|
4762
|
-
return currentValue + amount;
|
|
4763
|
-
};
|
|
4764
|
-
resource.sub = async (id, field, amount) => {
|
|
4765
|
-
const { plugin: handler } = plugin._resolveFieldAndPlugin(resource, field, amount);
|
|
4766
|
-
const now = /* @__PURE__ */ new Date();
|
|
4767
|
-
const cohortInfo = plugin.getCohortInfo(now);
|
|
4768
|
-
const transaction = {
|
|
4769
|
-
id: idGenerator(),
|
|
4770
|
-
originalId: id,
|
|
4771
|
-
field: handler.field,
|
|
4772
|
-
value: amount,
|
|
4773
|
-
operation: "sub",
|
|
4774
|
-
timestamp: now.toISOString(),
|
|
4775
|
-
cohortDate: cohortInfo.date,
|
|
4776
|
-
cohortHour: cohortInfo.hour,
|
|
4777
|
-
cohortMonth: cohortInfo.month,
|
|
4778
|
-
source: "sub",
|
|
4779
|
-
applied: false
|
|
4780
|
-
};
|
|
4781
|
-
await handler.transactionResource.insert(transaction);
|
|
4782
|
-
if (plugin.config.mode === "sync") {
|
|
4783
|
-
const oldResource = plugin.config.resource;
|
|
4784
|
-
const oldField = plugin.config.field;
|
|
4785
|
-
const oldTransactionResource = plugin.transactionResource;
|
|
4786
|
-
const oldTargetResource = plugin.targetResource;
|
|
4787
|
-
const oldLockResource = plugin.lockResource;
|
|
4788
|
-
const oldAnalyticsResource = plugin.analyticsResource;
|
|
4789
|
-
plugin.config.resource = handler.resource;
|
|
4790
|
-
plugin.config.field = handler.field;
|
|
4791
|
-
plugin.transactionResource = handler.transactionResource;
|
|
4792
|
-
plugin.targetResource = handler.targetResource;
|
|
4793
|
-
plugin.lockResource = handler.lockResource;
|
|
4794
|
-
plugin.analyticsResource = handler.analyticsResource;
|
|
4795
|
-
const result = await plugin._syncModeConsolidate(id, field);
|
|
4796
|
-
plugin.config.resource = oldResource;
|
|
4797
|
-
plugin.config.field = oldField;
|
|
4798
|
-
plugin.transactionResource = oldTransactionResource;
|
|
4799
|
-
plugin.targetResource = oldTargetResource;
|
|
4800
|
-
plugin.lockResource = oldLockResource;
|
|
4801
|
-
plugin.analyticsResource = oldAnalyticsResource;
|
|
4802
|
-
return result;
|
|
4803
|
-
}
|
|
4804
|
-
const [ok, err, record] = await tryFn(() => handler.targetResource.get(id));
|
|
4805
|
-
const currentValue = ok && record ? record[field] || 0 : 0;
|
|
4806
|
-
return currentValue - amount;
|
|
4807
|
-
};
|
|
4808
|
-
resource.consolidate = async (id, field) => {
|
|
4809
|
-
if (!field) {
|
|
4810
|
-
throw new Error(`Field parameter is required: consolidate(id, field)`);
|
|
4811
|
-
}
|
|
4812
|
-
const handler = resource._eventualConsistencyPlugins[field];
|
|
4813
|
-
if (!handler) {
|
|
4814
|
-
const availableFields = Object.keys(resource._eventualConsistencyPlugins).join(", ");
|
|
4815
|
-
throw new Error(
|
|
4816
|
-
`No eventual consistency plugin found for field "${field}". Available fields: ${availableFields}`
|
|
4817
|
-
);
|
|
4818
|
-
}
|
|
4819
|
-
const oldResource = plugin.config.resource;
|
|
4820
|
-
const oldField = plugin.config.field;
|
|
4821
|
-
const oldTransactionResource = plugin.transactionResource;
|
|
4822
|
-
const oldTargetResource = plugin.targetResource;
|
|
4823
|
-
const oldLockResource = plugin.lockResource;
|
|
4824
|
-
const oldAnalyticsResource = plugin.analyticsResource;
|
|
4825
|
-
plugin.config.resource = handler.resource;
|
|
4826
|
-
plugin.config.field = handler.field;
|
|
4827
|
-
plugin.transactionResource = handler.transactionResource;
|
|
4828
|
-
plugin.targetResource = handler.targetResource;
|
|
4829
|
-
plugin.lockResource = handler.lockResource;
|
|
4830
|
-
plugin.analyticsResource = handler.analyticsResource;
|
|
4831
|
-
const result = await plugin.consolidateRecord(id);
|
|
4832
|
-
plugin.config.resource = oldResource;
|
|
4833
|
-
plugin.config.field = oldField;
|
|
4834
|
-
plugin.transactionResource = oldTransactionResource;
|
|
4835
|
-
plugin.targetResource = oldTargetResource;
|
|
4836
|
-
plugin.lockResource = oldLockResource;
|
|
4837
|
-
plugin.analyticsResource = oldAnalyticsResource;
|
|
4838
|
-
return result;
|
|
4839
|
-
};
|
|
4840
|
-
resource.getConsolidatedValue = async (id, field, options = {}) => {
|
|
4841
|
-
const handler = resource._eventualConsistencyPlugins[field];
|
|
4842
|
-
if (!handler) {
|
|
4843
|
-
const availableFields = Object.keys(resource._eventualConsistencyPlugins).join(", ");
|
|
4844
|
-
throw new Error(
|
|
4845
|
-
`No eventual consistency plugin found for field "${field}". Available fields: ${availableFields}`
|
|
4846
|
-
);
|
|
4847
|
-
}
|
|
4848
|
-
const oldResource = plugin.config.resource;
|
|
4849
|
-
const oldField = plugin.config.field;
|
|
4850
|
-
const oldTransactionResource = plugin.transactionResource;
|
|
4851
|
-
const oldTargetResource = plugin.targetResource;
|
|
4852
|
-
plugin.config.resource = handler.resource;
|
|
4853
|
-
plugin.config.field = handler.field;
|
|
4854
|
-
plugin.transactionResource = handler.transactionResource;
|
|
4855
|
-
plugin.targetResource = handler.targetResource;
|
|
4856
|
-
const result = await plugin.getConsolidatedValue(id, options);
|
|
4857
|
-
plugin.config.resource = oldResource;
|
|
4858
|
-
plugin.config.field = oldField;
|
|
4859
|
-
plugin.transactionResource = oldTransactionResource;
|
|
4860
|
-
plugin.targetResource = oldTargetResource;
|
|
4861
|
-
return result;
|
|
4862
|
-
};
|
|
4863
|
-
}
|
|
4864
|
-
async createTransaction(handler, data) {
|
|
4865
|
-
const now = /* @__PURE__ */ new Date();
|
|
4866
|
-
const cohortInfo = this.getCohortInfo(now);
|
|
4867
|
-
const watermarkMs = this.config.consolidationWindow * 60 * 60 * 1e3;
|
|
4868
|
-
const watermarkTime = now.getTime() - watermarkMs;
|
|
4869
|
-
const cohortHourDate = /* @__PURE__ */ new Date(cohortInfo.hour + ":00:00Z");
|
|
4870
|
-
if (cohortHourDate.getTime() < watermarkTime) {
|
|
4871
|
-
const hoursLate = Math.floor((now.getTime() - cohortHourDate.getTime()) / (60 * 60 * 1e3));
|
|
4872
|
-
if (this.config.lateArrivalStrategy === "ignore") {
|
|
4873
|
-
if (this.config.verbose) {
|
|
4908
|
+
const [updateOk, updateErr] = await tryFn(
|
|
4909
|
+
() => targetResource.update(originalId, {
|
|
4910
|
+
[config.field]: consolidatedValue
|
|
4911
|
+
})
|
|
4912
|
+
);
|
|
4913
|
+
if (!updateOk) {
|
|
4914
|
+
if (updateErr?.message?.includes("does not exist")) {
|
|
4915
|
+
if (config.verbose) {
|
|
4874
4916
|
console.warn(
|
|
4875
|
-
`[EventualConsistency]
|
|
4917
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Record ${originalId} doesn't exist. Skipping consolidation. ${transactions.length} transactions will remain pending until record is created.`
|
|
4876
4918
|
);
|
|
4877
4919
|
}
|
|
4878
|
-
return
|
|
4879
|
-
} else if (this.config.lateArrivalStrategy === "warn") {
|
|
4880
|
-
console.warn(
|
|
4881
|
-
`[EventualConsistency] Late arrival detected: transaction for ${cohortInfo.hour} is ${hoursLate}h late (watermark: ${this.config.consolidationWindow}h). Processing anyway, but consolidation may not pick it up.`
|
|
4882
|
-
);
|
|
4920
|
+
return consolidatedValue;
|
|
4883
4921
|
}
|
|
4922
|
+
console.error(
|
|
4923
|
+
`[EventualConsistency] ${config.resource}.${config.field} - FAILED to update ${originalId}: ${updateErr?.message || updateErr}`,
|
|
4924
|
+
{ error: updateErr, consolidatedValue, currentValue }
|
|
4925
|
+
);
|
|
4926
|
+
throw updateErr;
|
|
4884
4927
|
}
|
|
4885
|
-
|
|
4886
|
-
|
|
4887
|
-
|
|
4888
|
-
|
|
4889
|
-
|
|
4890
|
-
operation: data.operation || "set",
|
|
4891
|
-
timestamp: now.toISOString(),
|
|
4892
|
-
cohortDate: cohortInfo.date,
|
|
4893
|
-
cohortHour: cohortInfo.hour,
|
|
4894
|
-
cohortMonth: cohortInfo.month,
|
|
4895
|
-
source: data.source || "unknown",
|
|
4896
|
-
applied: false
|
|
4897
|
-
};
|
|
4898
|
-
if (this.config.batchTransactions) {
|
|
4899
|
-
handler.pendingTransactions.set(transaction.id, transaction);
|
|
4900
|
-
if (this.config.verbose) {
|
|
4901
|
-
console.log(
|
|
4902
|
-
`[EventualConsistency] ${handler.resource}.${handler.field} - Transaction batched: ${data.operation} ${data.value} for ${data.originalId} (batch: ${handler.pendingTransactions.size}/${this.config.batchSize})`
|
|
4928
|
+
if (updateOk) {
|
|
4929
|
+
const transactionsToUpdate = transactions.filter((txn) => txn.id !== "__synthetic__");
|
|
4930
|
+
const { results, errors } = await promisePool.PromisePool.for(transactionsToUpdate).withConcurrency(10).process(async (txn) => {
|
|
4931
|
+
const [ok2, err2] = await tryFn(
|
|
4932
|
+
() => transactionResource.update(txn.id, { applied: true })
|
|
4903
4933
|
);
|
|
4934
|
+
if (!ok2 && config.verbose) {
|
|
4935
|
+
console.warn(`[EventualConsistency] Failed to mark transaction ${txn.id} as applied:`, err2?.message);
|
|
4936
|
+
}
|
|
4937
|
+
return ok2;
|
|
4938
|
+
});
|
|
4939
|
+
if (errors && errors.length > 0 && config.verbose) {
|
|
4940
|
+
console.warn(`[EventualConsistency] ${errors.length} transactions failed to mark as applied`);
|
|
4904
4941
|
}
|
|
4905
|
-
if (
|
|
4906
|
-
await
|
|
4942
|
+
if (config.enableAnalytics && transactionsToUpdate.length > 0 && updateAnalyticsFn) {
|
|
4943
|
+
await updateAnalyticsFn(transactionsToUpdate);
|
|
4907
4944
|
}
|
|
4908
|
-
|
|
4909
|
-
|
|
4910
|
-
|
|
4911
|
-
|
|
4912
|
-
|
|
4913
|
-
|
|
4945
|
+
if (targetResource && targetResource.cache && typeof targetResource.cache.delete === "function") {
|
|
4946
|
+
try {
|
|
4947
|
+
const cacheKey = await targetResource.cacheKeyFor({ id: originalId });
|
|
4948
|
+
await targetResource.cache.delete(cacheKey);
|
|
4949
|
+
if (config.verbose) {
|
|
4950
|
+
console.log(
|
|
4951
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Cache invalidated for ${originalId}`
|
|
4952
|
+
);
|
|
4953
|
+
}
|
|
4954
|
+
} catch (cacheErr) {
|
|
4955
|
+
if (config.verbose) {
|
|
4956
|
+
console.warn(
|
|
4957
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Failed to invalidate cache for ${originalId}: ${cacheErr?.message}`
|
|
4958
|
+
);
|
|
4959
|
+
}
|
|
4960
|
+
}
|
|
4914
4961
|
}
|
|
4915
4962
|
}
|
|
4916
|
-
return
|
|
4963
|
+
return consolidatedValue;
|
|
4964
|
+
} finally {
|
|
4965
|
+
const [lockReleased, lockReleaseErr] = await tryFn(() => lockResource.delete(lockId));
|
|
4966
|
+
if (!lockReleased && config.verbose) {
|
|
4967
|
+
console.warn(`[EventualConsistency] Failed to release lock ${lockId}:`, lockReleaseErr?.message);
|
|
4968
|
+
}
|
|
4917
4969
|
}
|
|
4918
|
-
|
|
4919
|
-
|
|
4920
|
-
|
|
4921
|
-
|
|
4922
|
-
|
|
4923
|
-
|
|
4924
|
-
|
|
4925
|
-
|
|
4926
|
-
|
|
4927
|
-
|
|
4928
|
-
|
|
4929
|
-
|
|
4930
|
-
|
|
4970
|
+
}
|
|
4971
|
+
async function getConsolidatedValue(originalId, options, transactionResource, targetResource, config) {
|
|
4972
|
+
const includeApplied = options.includeApplied || false;
|
|
4973
|
+
const startDate = options.startDate;
|
|
4974
|
+
const endDate = options.endDate;
|
|
4975
|
+
const query = { originalId };
|
|
4976
|
+
if (!includeApplied) {
|
|
4977
|
+
query.applied = false;
|
|
4978
|
+
}
|
|
4979
|
+
const [ok, err, transactions] = await tryFn(
|
|
4980
|
+
() => transactionResource.query(query)
|
|
4981
|
+
);
|
|
4982
|
+
if (!ok || !transactions || transactions.length === 0) {
|
|
4983
|
+
const [recordOk2, recordErr2, record2] = await tryFn(
|
|
4984
|
+
() => targetResource.get(originalId)
|
|
4985
|
+
);
|
|
4986
|
+
if (recordOk2 && record2) {
|
|
4987
|
+
return record2[config.field] || 0;
|
|
4931
4988
|
}
|
|
4989
|
+
return 0;
|
|
4932
4990
|
}
|
|
4933
|
-
|
|
4934
|
-
|
|
4935
|
-
|
|
4936
|
-
|
|
4937
|
-
|
|
4938
|
-
|
|
4939
|
-
|
|
4940
|
-
|
|
4941
|
-
return {
|
|
4942
|
-
date: `${year}-${month}-${day}`,
|
|
4943
|
-
hour: `${year}-${month}-${day}T${hour}`,
|
|
4944
|
-
// ISO-like format for hour partition
|
|
4945
|
-
month: `${year}-${month}`
|
|
4946
|
-
};
|
|
4991
|
+
let filtered = transactions;
|
|
4992
|
+
if (startDate || endDate) {
|
|
4993
|
+
filtered = transactions.filter((t) => {
|
|
4994
|
+
const timestamp = new Date(t.timestamp);
|
|
4995
|
+
if (startDate && timestamp < new Date(startDate)) return false;
|
|
4996
|
+
if (endDate && timestamp > new Date(endDate)) return false;
|
|
4997
|
+
return true;
|
|
4998
|
+
});
|
|
4947
4999
|
}
|
|
4948
|
-
|
|
4949
|
-
|
|
4950
|
-
|
|
4951
|
-
|
|
4952
|
-
|
|
4953
|
-
|
|
4954
|
-
|
|
4955
|
-
|
|
4956
|
-
|
|
4957
|
-
|
|
4958
|
-
|
|
4959
|
-
|
|
4960
|
-
|
|
4961
|
-
|
|
4962
|
-
|
|
4963
|
-
|
|
4964
|
-
|
|
4965
|
-
|
|
4966
|
-
|
|
4967
|
-
|
|
5000
|
+
const [recordOk, recordErr, record] = await tryFn(
|
|
5001
|
+
() => targetResource.get(originalId)
|
|
5002
|
+
);
|
|
5003
|
+
const currentValue = recordOk && record ? record[config.field] || 0 : 0;
|
|
5004
|
+
const hasSetOperation = filtered.some((t) => t.operation === "set");
|
|
5005
|
+
if (currentValue !== 0 && !hasSetOperation) {
|
|
5006
|
+
filtered.unshift(createSyntheticSetTransaction(currentValue));
|
|
5007
|
+
}
|
|
5008
|
+
filtered.sort(
|
|
5009
|
+
(a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()
|
|
5010
|
+
);
|
|
5011
|
+
return config.reducer(filtered);
|
|
5012
|
+
}
|
|
5013
|
+
async function getCohortStats(cohortDate, transactionResource) {
|
|
5014
|
+
const [ok, err, transactions] = await tryFn(
|
|
5015
|
+
() => transactionResource.query({
|
|
5016
|
+
cohortDate
|
|
5017
|
+
})
|
|
5018
|
+
);
|
|
5019
|
+
if (!ok) return null;
|
|
5020
|
+
const stats = {
|
|
5021
|
+
date: cohortDate,
|
|
5022
|
+
transactionCount: transactions.length,
|
|
5023
|
+
totalValue: 0,
|
|
5024
|
+
byOperation: { set: 0, add: 0, sub: 0 },
|
|
5025
|
+
byOriginalId: {}
|
|
5026
|
+
};
|
|
5027
|
+
for (const txn of transactions) {
|
|
5028
|
+
stats.totalValue += txn.value || 0;
|
|
5029
|
+
stats.byOperation[txn.operation] = (stats.byOperation[txn.operation] || 0) + 1;
|
|
5030
|
+
if (!stats.byOriginalId[txn.originalId]) {
|
|
5031
|
+
stats.byOriginalId[txn.originalId] = {
|
|
5032
|
+
count: 0,
|
|
5033
|
+
value: 0
|
|
4968
5034
|
};
|
|
4969
|
-
|
|
4970
|
-
|
|
4971
|
-
|
|
5035
|
+
}
|
|
5036
|
+
stats.byOriginalId[txn.originalId].count++;
|
|
5037
|
+
stats.byOriginalId[txn.originalId].value += txn.value || 0;
|
|
5038
|
+
}
|
|
5039
|
+
return stats;
|
|
5040
|
+
}
|
|
5041
|
+
async function recalculateRecord(originalId, transactionResource, targetResource, lockResource, consolidateRecordFn, config) {
|
|
5042
|
+
await cleanupStaleLocks(lockResource, config);
|
|
5043
|
+
const lockId = `lock-recalculate-${originalId}`;
|
|
5044
|
+
const [lockAcquired, lockErr, lock] = await tryFn(
|
|
5045
|
+
() => lockResource.insert({
|
|
5046
|
+
id: lockId,
|
|
5047
|
+
lockedAt: Date.now(),
|
|
5048
|
+
workerId: process.pid ? String(process.pid) : "unknown"
|
|
5049
|
+
})
|
|
5050
|
+
);
|
|
5051
|
+
if (!lockAcquired) {
|
|
5052
|
+
if (config.verbose) {
|
|
5053
|
+
console.log(`[EventualConsistency] Recalculate lock for ${originalId} already held, skipping`);
|
|
5054
|
+
}
|
|
5055
|
+
throw new Error(`Cannot recalculate ${originalId}: lock already held by another worker`);
|
|
5056
|
+
}
|
|
5057
|
+
try {
|
|
5058
|
+
if (config.verbose) {
|
|
5059
|
+
console.log(
|
|
5060
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Starting recalculation for ${originalId} (resetting all transactions to pending)`
|
|
5061
|
+
);
|
|
5062
|
+
}
|
|
5063
|
+
const [allOk, allErr, allTransactions] = await tryFn(
|
|
5064
|
+
() => transactionResource.query({
|
|
5065
|
+
originalId
|
|
5066
|
+
})
|
|
5067
|
+
);
|
|
5068
|
+
if (!allOk || !allTransactions || allTransactions.length === 0) {
|
|
5069
|
+
if (config.verbose) {
|
|
5070
|
+
console.log(
|
|
5071
|
+
`[EventualConsistency] ${config.resource}.${config.field} - No transactions found for ${originalId}, nothing to recalculate`
|
|
4972
5072
|
);
|
|
4973
5073
|
}
|
|
4974
|
-
return
|
|
5074
|
+
return 0;
|
|
4975
5075
|
}
|
|
4976
|
-
|
|
4977
|
-
startConsolidationTimer() {
|
|
4978
|
-
const intervalMs = this.config.consolidationInterval * 1e3;
|
|
4979
|
-
if (this.config.verbose) {
|
|
4980
|
-
const nextRun = new Date(Date.now() + intervalMs);
|
|
5076
|
+
if (config.verbose) {
|
|
4981
5077
|
console.log(
|
|
4982
|
-
`[EventualConsistency] ${
|
|
5078
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Found ${allTransactions.length} total transactions for ${originalId}, marking all as pending...`
|
|
4983
5079
|
);
|
|
4984
5080
|
}
|
|
4985
|
-
|
|
4986
|
-
|
|
4987
|
-
|
|
4988
|
-
|
|
4989
|
-
|
|
4990
|
-
|
|
4991
|
-
|
|
4992
|
-
|
|
5081
|
+
const transactionsToReset = allTransactions.filter((txn) => txn.source !== "anchor");
|
|
5082
|
+
const { results, errors } = await promisePool.PromisePool.for(transactionsToReset).withConcurrency(10).process(async (txn) => {
|
|
5083
|
+
const [ok, err] = await tryFn(
|
|
5084
|
+
() => transactionResource.update(txn.id, { applied: false })
|
|
5085
|
+
);
|
|
5086
|
+
if (!ok && config.verbose) {
|
|
5087
|
+
console.warn(`[EventualConsistency] Failed to reset transaction ${txn.id}:`, err?.message);
|
|
5088
|
+
}
|
|
5089
|
+
return ok;
|
|
5090
|
+
});
|
|
5091
|
+
if (errors && errors.length > 0) {
|
|
5092
|
+
console.warn(
|
|
5093
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Failed to reset ${errors.length} transactions during recalculation`
|
|
5094
|
+
);
|
|
5095
|
+
}
|
|
5096
|
+
if (config.verbose) {
|
|
4993
5097
|
console.log(
|
|
4994
|
-
`[EventualConsistency] ${
|
|
5098
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Reset ${results.length} transactions to pending, now resetting record value and running consolidation...`
|
|
4995
5099
|
);
|
|
4996
5100
|
}
|
|
4997
|
-
|
|
4998
|
-
|
|
4999
|
-
|
|
5000
|
-
|
|
5001
|
-
|
|
5002
|
-
|
|
5003
|
-
|
|
5004
|
-
|
|
5005
|
-
|
|
5006
|
-
const oldLockResource = this.lockResource;
|
|
5007
|
-
const oldAnalyticsResource = this.analyticsResource;
|
|
5008
|
-
this.config.resource = resourceName;
|
|
5009
|
-
this.config.field = fieldName;
|
|
5010
|
-
this.transactionResource = handler.transactionResource;
|
|
5011
|
-
this.targetResource = handler.targetResource;
|
|
5012
|
-
this.lockResource = handler.lockResource;
|
|
5013
|
-
this.analyticsResource = handler.analyticsResource;
|
|
5014
|
-
try {
|
|
5015
|
-
await this.runConsolidation();
|
|
5016
|
-
} finally {
|
|
5017
|
-
this.config.resource = oldResource;
|
|
5018
|
-
this.config.field = oldField;
|
|
5019
|
-
this.transactionResource = oldTransactionResource;
|
|
5020
|
-
this.targetResource = oldTargetResource;
|
|
5021
|
-
this.lockResource = oldLockResource;
|
|
5022
|
-
this.analyticsResource = oldAnalyticsResource;
|
|
5101
|
+
const [resetOk, resetErr] = await tryFn(
|
|
5102
|
+
() => targetResource.update(originalId, {
|
|
5103
|
+
[config.field]: 0
|
|
5104
|
+
})
|
|
5105
|
+
);
|
|
5106
|
+
if (!resetOk && config.verbose) {
|
|
5107
|
+
console.warn(
|
|
5108
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Failed to reset record value for ${originalId}: ${resetErr?.message}`
|
|
5109
|
+
);
|
|
5023
5110
|
}
|
|
5024
|
-
|
|
5025
|
-
|
|
5026
|
-
const startTime = Date.now();
|
|
5027
|
-
if (this.config.verbose) {
|
|
5111
|
+
const consolidatedValue = await consolidateRecordFn(originalId);
|
|
5112
|
+
if (config.verbose) {
|
|
5028
5113
|
console.log(
|
|
5029
|
-
`[EventualConsistency] ${
|
|
5114
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Recalculation complete for ${originalId}: final value = ${consolidatedValue}`
|
|
5030
5115
|
);
|
|
5031
5116
|
}
|
|
5032
|
-
|
|
5033
|
-
|
|
5034
|
-
|
|
5035
|
-
|
|
5036
|
-
|
|
5037
|
-
|
|
5038
|
-
|
|
5039
|
-
|
|
5117
|
+
return consolidatedValue;
|
|
5118
|
+
} finally {
|
|
5119
|
+
const [lockReleased, lockReleaseErr] = await tryFn(() => lockResource.delete(lockId));
|
|
5120
|
+
if (!lockReleased && config.verbose) {
|
|
5121
|
+
console.warn(`[EventualConsistency] Failed to release recalculate lock ${lockId}:`, lockReleaseErr?.message);
|
|
5122
|
+
}
|
|
5123
|
+
}
|
|
5124
|
+
}
|
|
5125
|
+
|
|
5126
|
+
function startGarbageCollectionTimer(handler, resourceName, fieldName, runGCCallback, config) {
|
|
5127
|
+
const gcIntervalMs = config.gcInterval * 1e3;
|
|
5128
|
+
handler.gcTimer = setInterval(async () => {
|
|
5129
|
+
await runGCCallback(handler, resourceName, fieldName);
|
|
5130
|
+
}, gcIntervalMs);
|
|
5131
|
+
return handler.gcTimer;
|
|
5132
|
+
}
|
|
5133
|
+
async function runGarbageCollection(transactionResource, lockResource, config, emitFn) {
|
|
5134
|
+
const gcLockId = `lock-gc-${config.resource}-${config.field}`;
|
|
5135
|
+
const [lockAcquired] = await tryFn(
|
|
5136
|
+
() => lockResource.insert({
|
|
5137
|
+
id: gcLockId,
|
|
5138
|
+
lockedAt: Date.now(),
|
|
5139
|
+
workerId: process.pid ? String(process.pid) : "unknown"
|
|
5140
|
+
})
|
|
5141
|
+
);
|
|
5142
|
+
if (!lockAcquired) {
|
|
5143
|
+
if (config.verbose) {
|
|
5144
|
+
console.log(`[EventualConsistency] GC already running in another container`);
|
|
5145
|
+
}
|
|
5146
|
+
return;
|
|
5147
|
+
}
|
|
5148
|
+
try {
|
|
5149
|
+
const now = Date.now();
|
|
5150
|
+
const retentionMs = config.transactionRetention * 24 * 60 * 60 * 1e3;
|
|
5151
|
+
const cutoffDate = new Date(now - retentionMs);
|
|
5152
|
+
const cutoffIso = cutoffDate.toISOString();
|
|
5153
|
+
if (config.verbose) {
|
|
5154
|
+
console.log(`[EventualConsistency] Running GC for transactions older than ${cutoffIso} (${config.transactionRetention} days)`);
|
|
5155
|
+
}
|
|
5156
|
+
const [ok, err, oldTransactions] = await tryFn(
|
|
5157
|
+
() => transactionResource.query({
|
|
5158
|
+
applied: true,
|
|
5159
|
+
timestamp: { "<": cutoffIso }
|
|
5160
|
+
})
|
|
5161
|
+
);
|
|
5162
|
+
if (!ok) {
|
|
5163
|
+
if (config.verbose) {
|
|
5164
|
+
console.warn(`[EventualConsistency] GC failed to query transactions:`, err?.message);
|
|
5040
5165
|
}
|
|
5041
|
-
|
|
5042
|
-
|
|
5043
|
-
|
|
5044
|
-
|
|
5166
|
+
return;
|
|
5167
|
+
}
|
|
5168
|
+
if (!oldTransactions || oldTransactions.length === 0) {
|
|
5169
|
+
if (config.verbose) {
|
|
5170
|
+
console.log(`[EventualConsistency] No old transactions to clean up`);
|
|
5045
5171
|
}
|
|
5046
|
-
|
|
5047
|
-
|
|
5048
|
-
|
|
5049
|
-
|
|
5050
|
-
|
|
5051
|
-
|
|
5052
|
-
|
|
5053
|
-
|
|
5054
|
-
|
|
5055
|
-
|
|
5172
|
+
return;
|
|
5173
|
+
}
|
|
5174
|
+
if (config.verbose) {
|
|
5175
|
+
console.log(`[EventualConsistency] Deleting ${oldTransactions.length} old transactions`);
|
|
5176
|
+
}
|
|
5177
|
+
const { results, errors } = await promisePool.PromisePool.for(oldTransactions).withConcurrency(10).process(async (txn) => {
|
|
5178
|
+
const [deleted] = await tryFn(() => transactionResource.delete(txn.id));
|
|
5179
|
+
return deleted;
|
|
5180
|
+
});
|
|
5181
|
+
if (config.verbose) {
|
|
5182
|
+
console.log(`[EventualConsistency] GC completed: ${results.length} deleted, ${errors.length} errors`);
|
|
5183
|
+
}
|
|
5184
|
+
if (emitFn) {
|
|
5185
|
+
emitFn("eventual-consistency.gc-completed", {
|
|
5186
|
+
resource: config.resource,
|
|
5187
|
+
field: config.field,
|
|
5188
|
+
deletedCount: results.length,
|
|
5189
|
+
errorCount: errors.length
|
|
5190
|
+
});
|
|
5191
|
+
}
|
|
5192
|
+
} catch (error) {
|
|
5193
|
+
if (config.verbose) {
|
|
5194
|
+
console.warn(`[EventualConsistency] GC error:`, error.message);
|
|
5195
|
+
}
|
|
5196
|
+
if (emitFn) {
|
|
5197
|
+
emitFn("eventual-consistency.gc-error", error);
|
|
5198
|
+
}
|
|
5199
|
+
} finally {
|
|
5200
|
+
await tryFn(() => lockResource.delete(gcLockId));
|
|
5201
|
+
}
|
|
5202
|
+
}
|
|
5203
|
+
|
|
5204
|
+
async function updateAnalytics(transactions, analyticsResource, config) {
|
|
5205
|
+
if (!analyticsResource || transactions.length === 0) return;
|
|
5206
|
+
if (config.verbose) {
|
|
5207
|
+
console.log(
|
|
5208
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Updating analytics for ${transactions.length} transactions...`
|
|
5209
|
+
);
|
|
5210
|
+
}
|
|
5211
|
+
try {
|
|
5212
|
+
const byHour = groupByCohort(transactions, "cohortHour");
|
|
5213
|
+
const cohortCount = Object.keys(byHour).length;
|
|
5214
|
+
if (config.verbose) {
|
|
5215
|
+
console.log(
|
|
5216
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Updating ${cohortCount} hourly analytics cohorts...`
|
|
5056
5217
|
);
|
|
5057
|
-
|
|
5058
|
-
|
|
5059
|
-
|
|
5060
|
-
|
|
5061
|
-
|
|
5062
|
-
|
|
5063
|
-
|
|
5064
|
-
return;
|
|
5065
|
-
}
|
|
5066
|
-
const uniqueIds = [...new Set(transactions.map((t) => t.originalId))];
|
|
5067
|
-
if (this.config.verbose) {
|
|
5218
|
+
}
|
|
5219
|
+
for (const [cohort, txns] of Object.entries(byHour)) {
|
|
5220
|
+
await upsertAnalytics("hour", cohort, txns, analyticsResource, config);
|
|
5221
|
+
}
|
|
5222
|
+
if (config.analyticsConfig.rollupStrategy === "incremental") {
|
|
5223
|
+
const uniqueHours = Object.keys(byHour);
|
|
5224
|
+
if (config.verbose) {
|
|
5068
5225
|
console.log(
|
|
5069
|
-
`[EventualConsistency] ${
|
|
5070
|
-
);
|
|
5071
|
-
}
|
|
5072
|
-
const { results, errors } = await promisePool.PromisePool.for(uniqueIds).withConcurrency(this.config.consolidationConcurrency).process(async (id) => {
|
|
5073
|
-
return await this.consolidateRecord(id);
|
|
5074
|
-
});
|
|
5075
|
-
const duration = Date.now() - startTime;
|
|
5076
|
-
if (errors && errors.length > 0) {
|
|
5077
|
-
console.error(
|
|
5078
|
-
`[EventualConsistency] ${this.config.resource}.${this.config.field} - Consolidation completed with ${errors.length} errors in ${duration}ms:`,
|
|
5079
|
-
errors
|
|
5226
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Rolling up ${uniqueHours.length} hours to daily/monthly analytics...`
|
|
5080
5227
|
);
|
|
5081
5228
|
}
|
|
5082
|
-
|
|
5083
|
-
|
|
5084
|
-
`[EventualConsistency] ${this.config.resource}.${this.config.field} - Consolidation complete: ${results.length} records consolidated in ${duration}ms (${errors.length} errors). Next run in ${this.config.consolidationInterval}s`
|
|
5085
|
-
);
|
|
5229
|
+
for (const cohortHour of uniqueHours) {
|
|
5230
|
+
await rollupAnalytics(cohortHour, analyticsResource, config);
|
|
5086
5231
|
}
|
|
5087
|
-
|
|
5088
|
-
|
|
5089
|
-
|
|
5090
|
-
|
|
5091
|
-
successCount: results.length,
|
|
5092
|
-
errorCount: errors.length,
|
|
5093
|
-
duration
|
|
5094
|
-
});
|
|
5095
|
-
} catch (error) {
|
|
5096
|
-
const duration = Date.now() - startTime;
|
|
5097
|
-
console.error(
|
|
5098
|
-
`[EventualConsistency] ${this.config.resource}.${this.config.field} - Consolidation error after ${duration}ms:`,
|
|
5099
|
-
error
|
|
5232
|
+
}
|
|
5233
|
+
if (config.verbose) {
|
|
5234
|
+
console.log(
|
|
5235
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Analytics update complete for ${cohortCount} cohorts`
|
|
5100
5236
|
);
|
|
5101
|
-
this.emit("eventual-consistency.consolidation-error", error);
|
|
5102
5237
|
}
|
|
5238
|
+
} catch (error) {
|
|
5239
|
+
console.warn(
|
|
5240
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Analytics update error:`,
|
|
5241
|
+
error.message
|
|
5242
|
+
);
|
|
5103
5243
|
}
|
|
5104
|
-
|
|
5105
|
-
|
|
5106
|
-
|
|
5107
|
-
|
|
5108
|
-
|
|
5109
|
-
|
|
5110
|
-
|
|
5111
|
-
|
|
5244
|
+
}
|
|
5245
|
+
async function upsertAnalytics(period, cohort, transactions, analyticsResource, config) {
|
|
5246
|
+
const id = `${period}-${cohort}`;
|
|
5247
|
+
const transactionCount = transactions.length;
|
|
5248
|
+
const signedValues = transactions.map((t) => {
|
|
5249
|
+
if (t.operation === "sub") return -t.value;
|
|
5250
|
+
return t.value;
|
|
5251
|
+
});
|
|
5252
|
+
const totalValue = signedValues.reduce((sum, v) => sum + v, 0);
|
|
5253
|
+
const avgValue = totalValue / transactionCount;
|
|
5254
|
+
const minValue = Math.min(...signedValues);
|
|
5255
|
+
const maxValue = Math.max(...signedValues);
|
|
5256
|
+
const operations = calculateOperationBreakdown(transactions);
|
|
5257
|
+
const recordCount = new Set(transactions.map((t) => t.originalId)).size;
|
|
5258
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
5259
|
+
const [existingOk, existingErr, existing] = await tryFn(
|
|
5260
|
+
() => analyticsResource.get(id)
|
|
5261
|
+
);
|
|
5262
|
+
if (existingOk && existing) {
|
|
5263
|
+
const newTransactionCount = existing.transactionCount + transactionCount;
|
|
5264
|
+
const newTotalValue = existing.totalValue + totalValue;
|
|
5265
|
+
const newAvgValue = newTotalValue / newTransactionCount;
|
|
5266
|
+
const newMinValue = Math.min(existing.minValue, minValue);
|
|
5267
|
+
const newMaxValue = Math.max(existing.maxValue, maxValue);
|
|
5268
|
+
const newOperations = { ...existing.operations };
|
|
5269
|
+
for (const [op, stats] of Object.entries(operations)) {
|
|
5270
|
+
if (!newOperations[op]) {
|
|
5271
|
+
newOperations[op] = { count: 0, sum: 0 };
|
|
5272
|
+
}
|
|
5273
|
+
newOperations[op].count += stats.count;
|
|
5274
|
+
newOperations[op].sum += stats.sum;
|
|
5275
|
+
}
|
|
5276
|
+
const newRecordCount = Math.max(existing.recordCount, recordCount);
|
|
5277
|
+
await tryFn(
|
|
5278
|
+
() => analyticsResource.update(id, {
|
|
5279
|
+
transactionCount: newTransactionCount,
|
|
5280
|
+
totalValue: newTotalValue,
|
|
5281
|
+
avgValue: newAvgValue,
|
|
5282
|
+
minValue: newMinValue,
|
|
5283
|
+
maxValue: newMaxValue,
|
|
5284
|
+
operations: newOperations,
|
|
5285
|
+
recordCount: newRecordCount,
|
|
5286
|
+
updatedAt: now
|
|
5112
5287
|
})
|
|
5113
5288
|
);
|
|
5114
|
-
|
|
5115
|
-
|
|
5116
|
-
|
|
5117
|
-
|
|
5118
|
-
|
|
5119
|
-
|
|
5120
|
-
|
|
5121
|
-
|
|
5122
|
-
|
|
5123
|
-
|
|
5124
|
-
|
|
5125
|
-
|
|
5126
|
-
|
|
5127
|
-
|
|
5128
|
-
|
|
5129
|
-
)
|
|
5130
|
-
|
|
5131
|
-
|
|
5132
|
-
|
|
5133
|
-
|
|
5134
|
-
|
|
5135
|
-
|
|
5136
|
-
|
|
5137
|
-
|
|
5138
|
-
|
|
5139
|
-
|
|
5140
|
-
|
|
5141
|
-
|
|
5142
|
-
|
|
5143
|
-
|
|
5144
|
-
|
|
5145
|
-
|
|
5146
|
-
|
|
5147
|
-
|
|
5148
|
-
|
|
5149
|
-
|
|
5150
|
-
|
|
5151
|
-
|
|
5152
|
-
|
|
5153
|
-
|
|
5154
|
-
|
|
5155
|
-
|
|
5156
|
-
|
|
5157
|
-
|
|
5158
|
-
|
|
5159
|
-
|
|
5160
|
-
|
|
5161
|
-
|
|
5162
|
-
|
|
5163
|
-
|
|
5164
|
-
|
|
5165
|
-
|
|
5166
|
-
|
|
5167
|
-
|
|
5168
|
-
|
|
5169
|
-
|
|
5170
|
-
|
|
5171
|
-
|
|
5172
|
-
|
|
5173
|
-
|
|
5174
|
-
|
|
5175
|
-
|
|
5176
|
-
|
|
5177
|
-
|
|
5178
|
-
|
|
5179
|
-
|
|
5180
|
-
|
|
5181
|
-
|
|
5182
|
-
|
|
5183
|
-
|
|
5184
|
-
|
|
5185
|
-
|
|
5186
|
-
|
|
5187
|
-
|
|
5188
|
-
|
|
5189
|
-
|
|
5190
|
-
|
|
5191
|
-
|
|
5192
|
-
|
|
5193
|
-
|
|
5194
|
-
|
|
5195
|
-
|
|
5196
|
-
|
|
5197
|
-
|
|
5198
|
-
|
|
5199
|
-
|
|
5200
|
-
|
|
5201
|
-
|
|
5202
|
-
|
|
5203
|
-
|
|
5204
|
-
|
|
5205
|
-
|
|
5206
|
-
|
|
5207
|
-
|
|
5208
|
-
|
|
5209
|
-
|
|
5210
|
-
|
|
5211
|
-
|
|
5212
|
-
|
|
5213
|
-
|
|
5214
|
-
|
|
5215
|
-
|
|
5216
|
-
|
|
5217
|
-
|
|
5218
|
-
|
|
5219
|
-
|
|
5220
|
-
|
|
5221
|
-
|
|
5222
|
-
|
|
5223
|
-
|
|
5224
|
-
|
|
5225
|
-
|
|
5226
|
-
|
|
5227
|
-
|
|
5228
|
-
|
|
5229
|
-
|
|
5230
|
-
|
|
5231
|
-
|
|
5232
|
-
|
|
5233
|
-
|
|
5234
|
-
|
|
5235
|
-
|
|
5236
|
-
|
|
5237
|
-
|
|
5238
|
-
|
|
5239
|
-
|
|
5240
|
-
|
|
5241
|
-
|
|
5242
|
-
|
|
5243
|
-
|
|
5244
|
-
|
|
5289
|
+
} else {
|
|
5290
|
+
await tryFn(
|
|
5291
|
+
() => analyticsResource.insert({
|
|
5292
|
+
id,
|
|
5293
|
+
period,
|
|
5294
|
+
cohort,
|
|
5295
|
+
transactionCount,
|
|
5296
|
+
totalValue,
|
|
5297
|
+
avgValue,
|
|
5298
|
+
minValue,
|
|
5299
|
+
maxValue,
|
|
5300
|
+
operations,
|
|
5301
|
+
recordCount,
|
|
5302
|
+
consolidatedAt: now,
|
|
5303
|
+
updatedAt: now
|
|
5304
|
+
})
|
|
5305
|
+
);
|
|
5306
|
+
}
|
|
5307
|
+
}
|
|
5308
|
+
function calculateOperationBreakdown(transactions) {
|
|
5309
|
+
const breakdown = {};
|
|
5310
|
+
for (const txn of transactions) {
|
|
5311
|
+
const op = txn.operation;
|
|
5312
|
+
if (!breakdown[op]) {
|
|
5313
|
+
breakdown[op] = { count: 0, sum: 0 };
|
|
5314
|
+
}
|
|
5315
|
+
breakdown[op].count++;
|
|
5316
|
+
const signedValue = op === "sub" ? -txn.value : txn.value;
|
|
5317
|
+
breakdown[op].sum += signedValue;
|
|
5318
|
+
}
|
|
5319
|
+
return breakdown;
|
|
5320
|
+
}
|
|
5321
|
+
async function rollupAnalytics(cohortHour, analyticsResource, config) {
|
|
5322
|
+
const cohortDate = cohortHour.substring(0, 10);
|
|
5323
|
+
const cohortMonth = cohortHour.substring(0, 7);
|
|
5324
|
+
await rollupPeriod("day", cohortDate, cohortDate, analyticsResource);
|
|
5325
|
+
await rollupPeriod("month", cohortMonth, cohortMonth, analyticsResource);
|
|
5326
|
+
}
|
|
5327
|
+
async function rollupPeriod(period, cohort, sourcePrefix, analyticsResource, config) {
|
|
5328
|
+
const sourcePeriod = period === "day" ? "hour" : "day";
|
|
5329
|
+
const [ok, err, allAnalytics] = await tryFn(
|
|
5330
|
+
() => analyticsResource.list()
|
|
5331
|
+
);
|
|
5332
|
+
if (!ok || !allAnalytics) return;
|
|
5333
|
+
const sourceAnalytics = allAnalytics.filter(
|
|
5334
|
+
(a) => a.period === sourcePeriod && a.cohort.startsWith(sourcePrefix)
|
|
5335
|
+
);
|
|
5336
|
+
if (sourceAnalytics.length === 0) return;
|
|
5337
|
+
const transactionCount = sourceAnalytics.reduce((sum, a) => sum + a.transactionCount, 0);
|
|
5338
|
+
const totalValue = sourceAnalytics.reduce((sum, a) => sum + a.totalValue, 0);
|
|
5339
|
+
const avgValue = totalValue / transactionCount;
|
|
5340
|
+
const minValue = Math.min(...sourceAnalytics.map((a) => a.minValue));
|
|
5341
|
+
const maxValue = Math.max(...sourceAnalytics.map((a) => a.maxValue));
|
|
5342
|
+
const operations = {};
|
|
5343
|
+
for (const analytics of sourceAnalytics) {
|
|
5344
|
+
for (const [op, stats] of Object.entries(analytics.operations || {})) {
|
|
5345
|
+
if (!operations[op]) {
|
|
5346
|
+
operations[op] = { count: 0, sum: 0 };
|
|
5347
|
+
}
|
|
5348
|
+
operations[op].count += stats.count;
|
|
5349
|
+
operations[op].sum += stats.sum;
|
|
5350
|
+
}
|
|
5351
|
+
}
|
|
5352
|
+
const recordCount = Math.max(...sourceAnalytics.map((a) => a.recordCount));
|
|
5353
|
+
const id = `${period}-${cohort}`;
|
|
5354
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
5355
|
+
const [existingOk, existingErr, existing] = await tryFn(
|
|
5356
|
+
() => analyticsResource.get(id)
|
|
5357
|
+
);
|
|
5358
|
+
if (existingOk && existing) {
|
|
5359
|
+
await tryFn(
|
|
5360
|
+
() => analyticsResource.update(id, {
|
|
5361
|
+
transactionCount,
|
|
5362
|
+
totalValue,
|
|
5363
|
+
avgValue,
|
|
5364
|
+
minValue,
|
|
5365
|
+
maxValue,
|
|
5366
|
+
operations,
|
|
5367
|
+
recordCount,
|
|
5368
|
+
updatedAt: now
|
|
5369
|
+
})
|
|
5370
|
+
);
|
|
5371
|
+
} else {
|
|
5372
|
+
await tryFn(
|
|
5373
|
+
() => analyticsResource.insert({
|
|
5374
|
+
id,
|
|
5375
|
+
period,
|
|
5376
|
+
cohort,
|
|
5377
|
+
transactionCount,
|
|
5378
|
+
totalValue,
|
|
5379
|
+
avgValue,
|
|
5380
|
+
minValue,
|
|
5381
|
+
maxValue,
|
|
5382
|
+
operations,
|
|
5383
|
+
recordCount,
|
|
5384
|
+
consolidatedAt: now,
|
|
5385
|
+
updatedAt: now
|
|
5386
|
+
})
|
|
5387
|
+
);
|
|
5388
|
+
}
|
|
5389
|
+
}
|
|
5390
|
+
function fillGaps(data, period, startDate, endDate) {
|
|
5391
|
+
if (!data || data.length === 0) {
|
|
5392
|
+
data = [];
|
|
5393
|
+
}
|
|
5394
|
+
const dataMap = /* @__PURE__ */ new Map();
|
|
5395
|
+
data.forEach((item) => {
|
|
5396
|
+
dataMap.set(item.cohort, item);
|
|
5397
|
+
});
|
|
5398
|
+
const result = [];
|
|
5399
|
+
const emptyRecord = {
|
|
5400
|
+
count: 0,
|
|
5401
|
+
sum: 0,
|
|
5402
|
+
avg: 0,
|
|
5403
|
+
min: 0,
|
|
5404
|
+
max: 0,
|
|
5405
|
+
recordCount: 0
|
|
5406
|
+
};
|
|
5407
|
+
if (period === "hour") {
|
|
5408
|
+
const start = /* @__PURE__ */ new Date(startDate + "T00:00:00Z");
|
|
5409
|
+
const end = /* @__PURE__ */ new Date(endDate + "T23:59:59Z");
|
|
5410
|
+
for (let dt = new Date(start); dt <= end; dt.setHours(dt.getHours() + 1)) {
|
|
5411
|
+
const cohort = dt.toISOString().substring(0, 13);
|
|
5412
|
+
result.push(dataMap.get(cohort) || { cohort, ...emptyRecord });
|
|
5413
|
+
}
|
|
5414
|
+
} else if (period === "day") {
|
|
5415
|
+
const start = new Date(startDate);
|
|
5416
|
+
const end = new Date(endDate);
|
|
5417
|
+
for (let dt = new Date(start); dt <= end; dt.setDate(dt.getDate() + 1)) {
|
|
5418
|
+
const cohort = dt.toISOString().substring(0, 10);
|
|
5419
|
+
result.push(dataMap.get(cohort) || { cohort, ...emptyRecord });
|
|
5420
|
+
}
|
|
5421
|
+
} else if (period === "month") {
|
|
5422
|
+
const startYear = parseInt(startDate.substring(0, 4));
|
|
5423
|
+
const startMonth = parseInt(startDate.substring(5, 7));
|
|
5424
|
+
const endYear = parseInt(endDate.substring(0, 4));
|
|
5425
|
+
const endMonth = parseInt(endDate.substring(5, 7));
|
|
5426
|
+
for (let year = startYear; year <= endYear; year++) {
|
|
5427
|
+
const firstMonth = year === startYear ? startMonth : 1;
|
|
5428
|
+
const lastMonth = year === endYear ? endMonth : 12;
|
|
5429
|
+
for (let month = firstMonth; month <= lastMonth; month++) {
|
|
5430
|
+
const cohort = `${year}-${month.toString().padStart(2, "0")}`;
|
|
5431
|
+
result.push(dataMap.get(cohort) || { cohort, ...emptyRecord });
|
|
5245
5432
|
}
|
|
5246
|
-
|
|
5247
|
-
|
|
5433
|
+
}
|
|
5434
|
+
}
|
|
5435
|
+
return result;
|
|
5436
|
+
}
|
|
5437
|
+
async function getAnalytics(resourceName, field, options, fieldHandlers) {
|
|
5438
|
+
const resourceHandlers = fieldHandlers.get(resourceName);
|
|
5439
|
+
if (!resourceHandlers) {
|
|
5440
|
+
throw new Error(`No eventual consistency configured for resource: ${resourceName}`);
|
|
5441
|
+
}
|
|
5442
|
+
const handler = resourceHandlers.get(field);
|
|
5443
|
+
if (!handler) {
|
|
5444
|
+
throw new Error(`No eventual consistency configured for field: ${resourceName}.${field}`);
|
|
5445
|
+
}
|
|
5446
|
+
if (!handler.analyticsResource) {
|
|
5447
|
+
throw new Error("Analytics not enabled for this plugin");
|
|
5448
|
+
}
|
|
5449
|
+
const { period = "day", date, startDate, endDate, month, year, breakdown = false } = options;
|
|
5450
|
+
const [ok, err, allAnalytics] = await tryFn(
|
|
5451
|
+
() => handler.analyticsResource.list()
|
|
5452
|
+
);
|
|
5453
|
+
if (!ok || !allAnalytics) {
|
|
5454
|
+
return [];
|
|
5455
|
+
}
|
|
5456
|
+
let filtered = allAnalytics.filter((a) => a.period === period);
|
|
5457
|
+
if (date) {
|
|
5458
|
+
if (period === "hour") {
|
|
5459
|
+
filtered = filtered.filter((a) => a.cohort.startsWith(date));
|
|
5460
|
+
} else {
|
|
5461
|
+
filtered = filtered.filter((a) => a.cohort === date);
|
|
5462
|
+
}
|
|
5463
|
+
} else if (startDate && endDate) {
|
|
5464
|
+
filtered = filtered.filter((a) => a.cohort >= startDate && a.cohort <= endDate);
|
|
5465
|
+
} else if (month) {
|
|
5466
|
+
filtered = filtered.filter((a) => a.cohort.startsWith(month));
|
|
5467
|
+
} else if (year) {
|
|
5468
|
+
filtered = filtered.filter((a) => a.cohort.startsWith(String(year)));
|
|
5469
|
+
}
|
|
5470
|
+
filtered.sort((a, b) => a.cohort.localeCompare(b.cohort));
|
|
5471
|
+
if (breakdown === "operations") {
|
|
5472
|
+
return filtered.map((a) => ({
|
|
5473
|
+
cohort: a.cohort,
|
|
5474
|
+
...a.operations
|
|
5475
|
+
}));
|
|
5476
|
+
}
|
|
5477
|
+
return filtered.map((a) => ({
|
|
5478
|
+
cohort: a.cohort,
|
|
5479
|
+
count: a.transactionCount,
|
|
5480
|
+
sum: a.totalValue,
|
|
5481
|
+
avg: a.avgValue,
|
|
5482
|
+
min: a.minValue,
|
|
5483
|
+
max: a.maxValue,
|
|
5484
|
+
operations: a.operations,
|
|
5485
|
+
recordCount: a.recordCount
|
|
5486
|
+
}));
|
|
5487
|
+
}
|
|
5488
|
+
async function getMonthByDay(resourceName, field, month, options, fieldHandlers) {
|
|
5489
|
+
const year = parseInt(month.substring(0, 4));
|
|
5490
|
+
const monthNum = parseInt(month.substring(5, 7));
|
|
5491
|
+
const firstDay = new Date(year, monthNum - 1, 1);
|
|
5492
|
+
const lastDay = new Date(year, monthNum, 0);
|
|
5493
|
+
const startDate = firstDay.toISOString().substring(0, 10);
|
|
5494
|
+
const endDate = lastDay.toISOString().substring(0, 10);
|
|
5495
|
+
const data = await getAnalytics(resourceName, field, {
|
|
5496
|
+
period: "day",
|
|
5497
|
+
startDate,
|
|
5498
|
+
endDate
|
|
5499
|
+
}, fieldHandlers);
|
|
5500
|
+
if (options.fillGaps) {
|
|
5501
|
+
return fillGaps(data, "day", startDate, endDate);
|
|
5502
|
+
}
|
|
5503
|
+
return data;
|
|
5504
|
+
}
|
|
5505
|
+
async function getDayByHour(resourceName, field, date, options, fieldHandlers) {
|
|
5506
|
+
const data = await getAnalytics(resourceName, field, {
|
|
5507
|
+
period: "hour",
|
|
5508
|
+
date
|
|
5509
|
+
}, fieldHandlers);
|
|
5510
|
+
if (options.fillGaps) {
|
|
5511
|
+
return fillGaps(data, "hour", date, date);
|
|
5512
|
+
}
|
|
5513
|
+
return data;
|
|
5514
|
+
}
|
|
5515
|
+
async function getLastNDays(resourceName, field, days, options, fieldHandlers) {
|
|
5516
|
+
const dates = Array.from({ length: days }, (_, i) => {
|
|
5517
|
+
const date = /* @__PURE__ */ new Date();
|
|
5518
|
+
date.setDate(date.getDate() - i);
|
|
5519
|
+
return date.toISOString().substring(0, 10);
|
|
5520
|
+
}).reverse();
|
|
5521
|
+
const data = await getAnalytics(resourceName, field, {
|
|
5522
|
+
period: "day",
|
|
5523
|
+
startDate: dates[0],
|
|
5524
|
+
endDate: dates[dates.length - 1]
|
|
5525
|
+
}, fieldHandlers);
|
|
5526
|
+
if (options.fillGaps) {
|
|
5527
|
+
return fillGaps(data, "day", dates[0], dates[dates.length - 1]);
|
|
5528
|
+
}
|
|
5529
|
+
return data;
|
|
5530
|
+
}
|
|
5531
|
+
async function getYearByMonth(resourceName, field, year, options, fieldHandlers) {
|
|
5532
|
+
const data = await getAnalytics(resourceName, field, {
|
|
5533
|
+
period: "month",
|
|
5534
|
+
year
|
|
5535
|
+
}, fieldHandlers);
|
|
5536
|
+
if (options.fillGaps) {
|
|
5537
|
+
const startDate = `${year}-01`;
|
|
5538
|
+
const endDate = `${year}-12`;
|
|
5539
|
+
return fillGaps(data, "month", startDate, endDate);
|
|
5540
|
+
}
|
|
5541
|
+
return data;
|
|
5542
|
+
}
|
|
5543
|
+
async function getMonthByHour(resourceName, field, month, options, fieldHandlers) {
|
|
5544
|
+
let year, monthNum;
|
|
5545
|
+
if (month === "last") {
|
|
5546
|
+
const now = /* @__PURE__ */ new Date();
|
|
5547
|
+
now.setMonth(now.getMonth() - 1);
|
|
5548
|
+
year = now.getFullYear();
|
|
5549
|
+
monthNum = now.getMonth() + 1;
|
|
5550
|
+
} else {
|
|
5551
|
+
year = parseInt(month.substring(0, 4));
|
|
5552
|
+
monthNum = parseInt(month.substring(5, 7));
|
|
5553
|
+
}
|
|
5554
|
+
const firstDay = new Date(year, monthNum - 1, 1);
|
|
5555
|
+
const lastDay = new Date(year, monthNum, 0);
|
|
5556
|
+
const startDate = firstDay.toISOString().substring(0, 10);
|
|
5557
|
+
const endDate = lastDay.toISOString().substring(0, 10);
|
|
5558
|
+
const data = await getAnalytics(resourceName, field, {
|
|
5559
|
+
period: "hour",
|
|
5560
|
+
startDate,
|
|
5561
|
+
endDate
|
|
5562
|
+
}, fieldHandlers);
|
|
5563
|
+
if (options.fillGaps) {
|
|
5564
|
+
return fillGaps(data, "hour", startDate, endDate);
|
|
5565
|
+
}
|
|
5566
|
+
return data;
|
|
5567
|
+
}
|
|
5568
|
+
async function getTopRecords(resourceName, field, options, fieldHandlers) {
|
|
5569
|
+
const resourceHandlers = fieldHandlers.get(resourceName);
|
|
5570
|
+
if (!resourceHandlers) {
|
|
5571
|
+
throw new Error(`No eventual consistency configured for resource: ${resourceName}`);
|
|
5572
|
+
}
|
|
5573
|
+
const handler = resourceHandlers.get(field);
|
|
5574
|
+
if (!handler) {
|
|
5575
|
+
throw new Error(`No eventual consistency configured for field: ${resourceName}.${field}`);
|
|
5576
|
+
}
|
|
5577
|
+
if (!handler.transactionResource) {
|
|
5578
|
+
throw new Error("Transaction resource not initialized");
|
|
5579
|
+
}
|
|
5580
|
+
const { period = "day", date, metric = "transactionCount", limit = 10 } = options;
|
|
5581
|
+
const [ok, err, transactions] = await tryFn(
|
|
5582
|
+
() => handler.transactionResource.list()
|
|
5583
|
+
);
|
|
5584
|
+
if (!ok || !transactions) {
|
|
5585
|
+
return [];
|
|
5586
|
+
}
|
|
5587
|
+
let filtered = transactions;
|
|
5588
|
+
if (date) {
|
|
5589
|
+
if (period === "hour") {
|
|
5590
|
+
filtered = transactions.filter((t) => t.cohortHour && t.cohortHour.startsWith(date));
|
|
5591
|
+
} else if (period === "day") {
|
|
5592
|
+
filtered = transactions.filter((t) => t.cohortDate === date);
|
|
5593
|
+
} else if (period === "month") {
|
|
5594
|
+
filtered = transactions.filter((t) => t.cohortMonth && t.cohortMonth.startsWith(date));
|
|
5595
|
+
}
|
|
5596
|
+
}
|
|
5597
|
+
const byRecord = {};
|
|
5598
|
+
for (const txn of filtered) {
|
|
5599
|
+
const recordId = txn.originalId;
|
|
5600
|
+
if (!byRecord[recordId]) {
|
|
5601
|
+
byRecord[recordId] = { count: 0, sum: 0 };
|
|
5602
|
+
}
|
|
5603
|
+
byRecord[recordId].count++;
|
|
5604
|
+
byRecord[recordId].sum += txn.value;
|
|
5605
|
+
}
|
|
5606
|
+
const records = Object.entries(byRecord).map(([recordId, stats]) => ({
|
|
5607
|
+
recordId,
|
|
5608
|
+
count: stats.count,
|
|
5609
|
+
sum: stats.sum
|
|
5610
|
+
}));
|
|
5611
|
+
records.sort((a, b) => {
|
|
5612
|
+
if (metric === "transactionCount") {
|
|
5613
|
+
return b.count - a.count;
|
|
5614
|
+
} else if (metric === "totalValue") {
|
|
5615
|
+
return b.sum - a.sum;
|
|
5616
|
+
}
|
|
5617
|
+
return 0;
|
|
5618
|
+
});
|
|
5619
|
+
return records.slice(0, limit);
|
|
5620
|
+
}
|
|
5621
|
+
|
|
5622
|
+
function addHelperMethods(resource, plugin, config) {
|
|
5623
|
+
resource.set = async (id, field, value) => {
|
|
5624
|
+
const { plugin: handler } = resolveFieldAndPlugin(resource, field, value);
|
|
5625
|
+
const now = /* @__PURE__ */ new Date();
|
|
5626
|
+
const cohortInfo = getCohortInfo(now, config.cohort.timezone, config.verbose);
|
|
5627
|
+
const transaction = {
|
|
5628
|
+
id: idGenerator(),
|
|
5629
|
+
originalId: id,
|
|
5630
|
+
field: handler.field,
|
|
5631
|
+
value,
|
|
5632
|
+
operation: "set",
|
|
5633
|
+
timestamp: now.toISOString(),
|
|
5634
|
+
cohortDate: cohortInfo.date,
|
|
5635
|
+
cohortHour: cohortInfo.hour,
|
|
5636
|
+
cohortMonth: cohortInfo.month,
|
|
5637
|
+
source: "set",
|
|
5638
|
+
applied: false
|
|
5639
|
+
};
|
|
5640
|
+
await handler.transactionResource.insert(transaction);
|
|
5641
|
+
if (config.mode === "sync") {
|
|
5642
|
+
return await plugin._syncModeConsolidate(handler, id, field);
|
|
5643
|
+
}
|
|
5644
|
+
return value;
|
|
5645
|
+
};
|
|
5646
|
+
resource.add = async (id, field, amount) => {
|
|
5647
|
+
const { plugin: handler } = resolveFieldAndPlugin(resource, field, amount);
|
|
5648
|
+
const now = /* @__PURE__ */ new Date();
|
|
5649
|
+
const cohortInfo = getCohortInfo(now, config.cohort.timezone, config.verbose);
|
|
5650
|
+
const transaction = {
|
|
5651
|
+
id: idGenerator(),
|
|
5652
|
+
originalId: id,
|
|
5653
|
+
field: handler.field,
|
|
5654
|
+
value: amount,
|
|
5655
|
+
operation: "add",
|
|
5656
|
+
timestamp: now.toISOString(),
|
|
5657
|
+
cohortDate: cohortInfo.date,
|
|
5658
|
+
cohortHour: cohortInfo.hour,
|
|
5659
|
+
cohortMonth: cohortInfo.month,
|
|
5660
|
+
source: "add",
|
|
5661
|
+
applied: false
|
|
5662
|
+
};
|
|
5663
|
+
await handler.transactionResource.insert(transaction);
|
|
5664
|
+
if (config.mode === "sync") {
|
|
5665
|
+
return await plugin._syncModeConsolidate(handler, id, field);
|
|
5666
|
+
}
|
|
5667
|
+
const [ok, err, record] = await tryFn(() => handler.targetResource.get(id));
|
|
5668
|
+
const currentValue = ok && record ? record[field] || 0 : 0;
|
|
5669
|
+
return currentValue + amount;
|
|
5670
|
+
};
|
|
5671
|
+
resource.sub = async (id, field, amount) => {
|
|
5672
|
+
const { plugin: handler } = resolveFieldAndPlugin(resource, field, amount);
|
|
5673
|
+
const now = /* @__PURE__ */ new Date();
|
|
5674
|
+
const cohortInfo = getCohortInfo(now, config.cohort.timezone, config.verbose);
|
|
5675
|
+
const transaction = {
|
|
5676
|
+
id: idGenerator(),
|
|
5677
|
+
originalId: id,
|
|
5678
|
+
field: handler.field,
|
|
5679
|
+
value: amount,
|
|
5680
|
+
operation: "sub",
|
|
5681
|
+
timestamp: now.toISOString(),
|
|
5682
|
+
cohortDate: cohortInfo.date,
|
|
5683
|
+
cohortHour: cohortInfo.hour,
|
|
5684
|
+
cohortMonth: cohortInfo.month,
|
|
5685
|
+
source: "sub",
|
|
5686
|
+
applied: false
|
|
5687
|
+
};
|
|
5688
|
+
await handler.transactionResource.insert(transaction);
|
|
5689
|
+
if (config.mode === "sync") {
|
|
5690
|
+
return await plugin._syncModeConsolidate(handler, id, field);
|
|
5691
|
+
}
|
|
5692
|
+
const [ok, err, record] = await tryFn(() => handler.targetResource.get(id));
|
|
5693
|
+
const currentValue = ok && record ? record[field] || 0 : 0;
|
|
5694
|
+
return currentValue - amount;
|
|
5695
|
+
};
|
|
5696
|
+
resource.consolidate = async (id, field) => {
|
|
5697
|
+
if (!field) {
|
|
5698
|
+
throw new Error(`Field parameter is required: consolidate(id, field)`);
|
|
5699
|
+
}
|
|
5700
|
+
const handler = resource._eventualConsistencyPlugins[field];
|
|
5701
|
+
if (!handler) {
|
|
5702
|
+
const availableFields = Object.keys(resource._eventualConsistencyPlugins).join(", ");
|
|
5703
|
+
throw new Error(
|
|
5704
|
+
`No eventual consistency plugin found for field "${field}". Available fields: ${availableFields}`
|
|
5248
5705
|
);
|
|
5249
|
-
|
|
5250
|
-
|
|
5251
|
-
|
|
5252
|
-
|
|
5253
|
-
|
|
5254
|
-
|
|
5255
|
-
|
|
5256
|
-
|
|
5257
|
-
|
|
5258
|
-
}
|
|
5259
|
-
const [updateOk, updateErr] = await tryFn(
|
|
5260
|
-
() => this.targetResource.update(originalId, {
|
|
5261
|
-
[this.config.field]: consolidatedValue
|
|
5262
|
-
})
|
|
5706
|
+
}
|
|
5707
|
+
return await plugin._consolidateWithHandler(handler, id);
|
|
5708
|
+
};
|
|
5709
|
+
resource.getConsolidatedValue = async (id, field, options = {}) => {
|
|
5710
|
+
const handler = resource._eventualConsistencyPlugins[field];
|
|
5711
|
+
if (!handler) {
|
|
5712
|
+
const availableFields = Object.keys(resource._eventualConsistencyPlugins).join(", ");
|
|
5713
|
+
throw new Error(
|
|
5714
|
+
`No eventual consistency plugin found for field "${field}". Available fields: ${availableFields}`
|
|
5263
5715
|
);
|
|
5264
|
-
if (!updateOk) {
|
|
5265
|
-
if (updateErr?.message?.includes("does not exist")) {
|
|
5266
|
-
if (this.config.verbose) {
|
|
5267
|
-
console.warn(
|
|
5268
|
-
`[EventualConsistency] ${this.config.resource}.${this.config.field} - Record ${originalId} doesn't exist. Skipping consolidation. ${transactions.length} transactions will remain pending until record is created.`
|
|
5269
|
-
);
|
|
5270
|
-
}
|
|
5271
|
-
return consolidatedValue;
|
|
5272
|
-
}
|
|
5273
|
-
console.error(
|
|
5274
|
-
`[EventualConsistency] ${this.config.resource}.${this.config.field} - FAILED to update ${originalId}: ${updateErr?.message || updateErr}`,
|
|
5275
|
-
{ error: updateErr, consolidatedValue, currentValue }
|
|
5276
|
-
);
|
|
5277
|
-
throw updateErr;
|
|
5278
|
-
}
|
|
5279
|
-
if (updateOk) {
|
|
5280
|
-
const transactionsToUpdate = transactions.filter((txn) => txn.id !== "__synthetic__");
|
|
5281
|
-
const { results, errors } = await promisePool.PromisePool.for(transactionsToUpdate).withConcurrency(10).process(async (txn) => {
|
|
5282
|
-
const [ok2, err2] = await tryFn(
|
|
5283
|
-
() => this.transactionResource.update(txn.id, { applied: true })
|
|
5284
|
-
);
|
|
5285
|
-
if (!ok2 && this.config.verbose) {
|
|
5286
|
-
console.warn(`[EventualConsistency] Failed to mark transaction ${txn.id} as applied:`, err2?.message);
|
|
5287
|
-
}
|
|
5288
|
-
return ok2;
|
|
5289
|
-
});
|
|
5290
|
-
if (errors && errors.length > 0 && this.config.verbose) {
|
|
5291
|
-
console.warn(`[EventualConsistency] ${errors.length} transactions failed to mark as applied`);
|
|
5292
|
-
}
|
|
5293
|
-
if (this.config.enableAnalytics && transactionsToUpdate.length > 0) {
|
|
5294
|
-
await this.updateAnalytics(transactionsToUpdate);
|
|
5295
|
-
}
|
|
5296
|
-
if (this.targetResource && this.targetResource.cache && typeof this.targetResource.cache.delete === "function") {
|
|
5297
|
-
try {
|
|
5298
|
-
const cacheKey = await this.targetResource.cacheKeyFor({ id: originalId });
|
|
5299
|
-
await this.targetResource.cache.delete(cacheKey);
|
|
5300
|
-
if (this.config.verbose) {
|
|
5301
|
-
console.log(
|
|
5302
|
-
`[EventualConsistency] ${this.config.resource}.${this.config.field} - Cache invalidated for ${originalId}`
|
|
5303
|
-
);
|
|
5304
|
-
}
|
|
5305
|
-
} catch (cacheErr) {
|
|
5306
|
-
if (this.config.verbose) {
|
|
5307
|
-
console.warn(
|
|
5308
|
-
`[EventualConsistency] ${this.config.resource}.${this.config.field} - Failed to invalidate cache for ${originalId}: ${cacheErr?.message}`
|
|
5309
|
-
);
|
|
5310
|
-
}
|
|
5311
|
-
}
|
|
5312
|
-
}
|
|
5313
|
-
}
|
|
5314
|
-
return consolidatedValue;
|
|
5315
|
-
} finally {
|
|
5316
|
-
const [lockReleased, lockReleaseErr] = await tryFn(() => this.lockResource.delete(lockId));
|
|
5317
|
-
if (!lockReleased && this.config.verbose) {
|
|
5318
|
-
console.warn(`[EventualConsistency] Failed to release lock ${lockId}:`, lockReleaseErr?.message);
|
|
5319
|
-
}
|
|
5320
5716
|
}
|
|
5321
|
-
|
|
5322
|
-
|
|
5323
|
-
|
|
5324
|
-
|
|
5325
|
-
|
|
5326
|
-
const query = { originalId };
|
|
5327
|
-
if (!includeApplied) {
|
|
5328
|
-
query.applied = false;
|
|
5717
|
+
return await plugin._getConsolidatedValueWithHandler(handler, id, options);
|
|
5718
|
+
};
|
|
5719
|
+
resource.recalculate = async (id, field) => {
|
|
5720
|
+
if (!field) {
|
|
5721
|
+
throw new Error(`Field parameter is required: recalculate(id, field)`);
|
|
5329
5722
|
}
|
|
5330
|
-
const
|
|
5331
|
-
|
|
5332
|
-
|
|
5333
|
-
|
|
5334
|
-
|
|
5335
|
-
() => this.targetResource.get(originalId)
|
|
5723
|
+
const handler = resource._eventualConsistencyPlugins[field];
|
|
5724
|
+
if (!handler) {
|
|
5725
|
+
const availableFields = Object.keys(resource._eventualConsistencyPlugins).join(", ");
|
|
5726
|
+
throw new Error(
|
|
5727
|
+
`No eventual consistency plugin found for field "${field}". Available fields: ${availableFields}`
|
|
5336
5728
|
);
|
|
5337
|
-
|
|
5338
|
-
|
|
5729
|
+
}
|
|
5730
|
+
return await plugin._recalculateWithHandler(handler, id);
|
|
5731
|
+
};
|
|
5732
|
+
}
|
|
5733
|
+
|
|
5734
|
+
async function onSetup(database, fieldHandlers, completeFieldSetupFn, watchForResourceFn) {
|
|
5735
|
+
for (const [resourceName, resourceHandlers] of fieldHandlers) {
|
|
5736
|
+
const targetResource = database.resources[resourceName];
|
|
5737
|
+
if (!targetResource) {
|
|
5738
|
+
for (const handler of resourceHandlers.values()) {
|
|
5739
|
+
handler.deferredSetup = true;
|
|
5339
5740
|
}
|
|
5340
|
-
|
|
5741
|
+
watchForResourceFn(resourceName);
|
|
5742
|
+
continue;
|
|
5341
5743
|
}
|
|
5342
|
-
|
|
5343
|
-
|
|
5344
|
-
|
|
5345
|
-
const timestamp = new Date(t.timestamp);
|
|
5346
|
-
if (startDate && timestamp < new Date(startDate)) return false;
|
|
5347
|
-
if (endDate && timestamp > new Date(endDate)) return false;
|
|
5348
|
-
return true;
|
|
5349
|
-
});
|
|
5744
|
+
for (const [fieldName, handler] of resourceHandlers) {
|
|
5745
|
+
handler.targetResource = targetResource;
|
|
5746
|
+
await completeFieldSetupFn(handler);
|
|
5350
5747
|
}
|
|
5351
|
-
|
|
5352
|
-
|
|
5353
|
-
|
|
5354
|
-
|
|
5355
|
-
|
|
5356
|
-
|
|
5357
|
-
|
|
5748
|
+
}
|
|
5749
|
+
}
|
|
5750
|
+
function watchForResource(resourceName, database, fieldHandlers, completeFieldSetupFn) {
|
|
5751
|
+
const hookCallback = async ({ resource, config }) => {
|
|
5752
|
+
if (config.name === resourceName) {
|
|
5753
|
+
const resourceHandlers = fieldHandlers.get(resourceName);
|
|
5754
|
+
if (!resourceHandlers) return;
|
|
5755
|
+
for (const [fieldName, handler] of resourceHandlers) {
|
|
5756
|
+
if (handler.deferredSetup) {
|
|
5757
|
+
handler.targetResource = resource;
|
|
5758
|
+
handler.deferredSetup = false;
|
|
5759
|
+
await completeFieldSetupFn(handler);
|
|
5760
|
+
}
|
|
5761
|
+
}
|
|
5358
5762
|
}
|
|
5359
|
-
|
|
5360
|
-
|
|
5763
|
+
};
|
|
5764
|
+
database.addHook("afterCreateResource", hookCallback);
|
|
5765
|
+
}
|
|
5766
|
+
async function completeFieldSetup(handler, database, config, plugin) {
|
|
5767
|
+
if (!handler.targetResource) return;
|
|
5768
|
+
const resourceName = handler.resource;
|
|
5769
|
+
const fieldName = handler.field;
|
|
5770
|
+
const transactionResourceName = `${resourceName}_transactions_${fieldName}`;
|
|
5771
|
+
const partitionConfig = createPartitionConfig();
|
|
5772
|
+
const [ok, err, transactionResource] = await tryFn(
|
|
5773
|
+
() => database.createResource({
|
|
5774
|
+
name: transactionResourceName,
|
|
5775
|
+
attributes: {
|
|
5776
|
+
id: "string|required",
|
|
5777
|
+
originalId: "string|required",
|
|
5778
|
+
field: "string|required",
|
|
5779
|
+
value: "number|required",
|
|
5780
|
+
operation: "string|required",
|
|
5781
|
+
timestamp: "string|required",
|
|
5782
|
+
cohortDate: "string|required",
|
|
5783
|
+
cohortHour: "string|required",
|
|
5784
|
+
cohortMonth: "string|optional",
|
|
5785
|
+
source: "string|optional",
|
|
5786
|
+
applied: "boolean|optional"
|
|
5787
|
+
},
|
|
5788
|
+
behavior: "body-overflow",
|
|
5789
|
+
timestamps: true,
|
|
5790
|
+
partitions: partitionConfig,
|
|
5791
|
+
asyncPartitions: true,
|
|
5792
|
+
createdBy: "EventualConsistencyPlugin"
|
|
5793
|
+
})
|
|
5794
|
+
);
|
|
5795
|
+
if (!ok && !database.resources[transactionResourceName]) {
|
|
5796
|
+
throw new Error(`Failed to create transaction resource for ${resourceName}.${fieldName}: ${err?.message}`);
|
|
5797
|
+
}
|
|
5798
|
+
handler.transactionResource = ok ? transactionResource : database.resources[transactionResourceName];
|
|
5799
|
+
const lockResourceName = `${resourceName}_consolidation_locks_${fieldName}`;
|
|
5800
|
+
const [lockOk, lockErr, lockResource] = await tryFn(
|
|
5801
|
+
() => database.createResource({
|
|
5802
|
+
name: lockResourceName,
|
|
5803
|
+
attributes: {
|
|
5804
|
+
id: "string|required",
|
|
5805
|
+
lockedAt: "number|required",
|
|
5806
|
+
workerId: "string|optional"
|
|
5807
|
+
},
|
|
5808
|
+
behavior: "body-only",
|
|
5809
|
+
timestamps: false,
|
|
5810
|
+
createdBy: "EventualConsistencyPlugin"
|
|
5811
|
+
})
|
|
5812
|
+
);
|
|
5813
|
+
if (!lockOk && !database.resources[lockResourceName]) {
|
|
5814
|
+
throw new Error(`Failed to create lock resource for ${resourceName}.${fieldName}: ${lockErr?.message}`);
|
|
5815
|
+
}
|
|
5816
|
+
handler.lockResource = lockOk ? lockResource : database.resources[lockResourceName];
|
|
5817
|
+
if (config.enableAnalytics) {
|
|
5818
|
+
await createAnalyticsResource(handler, database, resourceName, fieldName);
|
|
5819
|
+
}
|
|
5820
|
+
addHelperMethodsForHandler(handler, plugin, config);
|
|
5821
|
+
if (config.verbose) {
|
|
5822
|
+
console.log(
|
|
5823
|
+
`[EventualConsistency] ${resourceName}.${fieldName} - Setup complete. Resources: ${transactionResourceName}, ${lockResourceName}${config.enableAnalytics ? `, ${resourceName}_analytics_${fieldName}` : ""}`
|
|
5361
5824
|
);
|
|
5362
|
-
return this.config.reducer(filtered);
|
|
5363
5825
|
}
|
|
5364
|
-
|
|
5365
|
-
|
|
5366
|
-
|
|
5367
|
-
|
|
5368
|
-
|
|
5369
|
-
|
|
5370
|
-
|
|
5371
|
-
|
|
5372
|
-
|
|
5373
|
-
|
|
5374
|
-
|
|
5375
|
-
|
|
5376
|
-
|
|
5377
|
-
|
|
5378
|
-
|
|
5379
|
-
|
|
5380
|
-
|
|
5381
|
-
|
|
5382
|
-
|
|
5383
|
-
|
|
5384
|
-
|
|
5385
|
-
|
|
5386
|
-
|
|
5826
|
+
}
|
|
5827
|
+
async function createAnalyticsResource(handler, database, resourceName, fieldName) {
|
|
5828
|
+
const analyticsResourceName = `${resourceName}_analytics_${fieldName}`;
|
|
5829
|
+
const [ok, err, analyticsResource] = await tryFn(
|
|
5830
|
+
() => database.createResource({
|
|
5831
|
+
name: analyticsResourceName,
|
|
5832
|
+
attributes: {
|
|
5833
|
+
id: "string|required",
|
|
5834
|
+
period: "string|required",
|
|
5835
|
+
cohort: "string|required",
|
|
5836
|
+
transactionCount: "number|required",
|
|
5837
|
+
totalValue: "number|required",
|
|
5838
|
+
avgValue: "number|required",
|
|
5839
|
+
minValue: "number|required",
|
|
5840
|
+
maxValue: "number|required",
|
|
5841
|
+
operations: "object|optional",
|
|
5842
|
+
recordCount: "number|required",
|
|
5843
|
+
consolidatedAt: "string|required",
|
|
5844
|
+
updatedAt: "string|required"
|
|
5845
|
+
},
|
|
5846
|
+
behavior: "body-overflow",
|
|
5847
|
+
timestamps: false,
|
|
5848
|
+
createdBy: "EventualConsistencyPlugin"
|
|
5849
|
+
})
|
|
5850
|
+
);
|
|
5851
|
+
if (!ok && !database.resources[analyticsResourceName]) {
|
|
5852
|
+
throw new Error(`Failed to create analytics resource for ${resourceName}.${fieldName}: ${err?.message}`);
|
|
5853
|
+
}
|
|
5854
|
+
handler.analyticsResource = ok ? analyticsResource : database.resources[analyticsResourceName];
|
|
5855
|
+
}
|
|
5856
|
+
function addHelperMethodsForHandler(handler, plugin, config) {
|
|
5857
|
+
const resource = handler.targetResource;
|
|
5858
|
+
const fieldName = handler.field;
|
|
5859
|
+
if (!resource._eventualConsistencyPlugins) {
|
|
5860
|
+
resource._eventualConsistencyPlugins = {};
|
|
5861
|
+
}
|
|
5862
|
+
resource._eventualConsistencyPlugins[fieldName] = handler;
|
|
5863
|
+
if (!resource.add) {
|
|
5864
|
+
addHelperMethods(resource, plugin, config);
|
|
5865
|
+
}
|
|
5866
|
+
}
|
|
5867
|
+
async function onStart(fieldHandlers, config, runConsolidationFn, runGCFn, emitFn) {
|
|
5868
|
+
for (const [resourceName, resourceHandlers] of fieldHandlers) {
|
|
5869
|
+
for (const [fieldName, handler] of resourceHandlers) {
|
|
5870
|
+
if (!handler.deferredSetup) {
|
|
5871
|
+
if (config.autoConsolidate && config.mode === "async") {
|
|
5872
|
+
startConsolidationTimer(handler, resourceName, fieldName, runConsolidationFn, config);
|
|
5873
|
+
}
|
|
5874
|
+
if (config.transactionRetention && config.transactionRetention > 0) {
|
|
5875
|
+
startGarbageCollectionTimer(handler, resourceName, fieldName, runGCFn, config);
|
|
5876
|
+
}
|
|
5877
|
+
if (emitFn) {
|
|
5878
|
+
emitFn("eventual-consistency.started", {
|
|
5879
|
+
resource: resourceName,
|
|
5880
|
+
field: fieldName,
|
|
5881
|
+
cohort: config.cohort
|
|
5882
|
+
});
|
|
5883
|
+
}
|
|
5387
5884
|
}
|
|
5388
|
-
stats.byOriginalId[txn.originalId].count++;
|
|
5389
|
-
stats.byOriginalId[txn.originalId].value += txn.value || 0;
|
|
5390
5885
|
}
|
|
5391
|
-
return stats;
|
|
5392
5886
|
}
|
|
5393
|
-
|
|
5394
|
-
|
|
5395
|
-
|
|
5396
|
-
|
|
5397
|
-
|
|
5398
|
-
|
|
5399
|
-
|
|
5400
|
-
|
|
5401
|
-
|
|
5402
|
-
|
|
5403
|
-
|
|
5404
|
-
|
|
5405
|
-
|
|
5406
|
-
|
|
5407
|
-
}
|
|
5408
|
-
|
|
5409
|
-
|
|
5410
|
-
|
|
5411
|
-
|
|
5887
|
+
}
|
|
5888
|
+
async function onStop(fieldHandlers, emitFn) {
|
|
5889
|
+
for (const [resourceName, resourceHandlers] of fieldHandlers) {
|
|
5890
|
+
for (const [fieldName, handler] of resourceHandlers) {
|
|
5891
|
+
if (handler.consolidationTimer) {
|
|
5892
|
+
clearInterval(handler.consolidationTimer);
|
|
5893
|
+
handler.consolidationTimer = null;
|
|
5894
|
+
}
|
|
5895
|
+
if (handler.gcTimer) {
|
|
5896
|
+
clearInterval(handler.gcTimer);
|
|
5897
|
+
handler.gcTimer = null;
|
|
5898
|
+
}
|
|
5899
|
+
if (handler.pendingTransactions && handler.pendingTransactions.size > 0) {
|
|
5900
|
+
await flushPendingTransactions(handler);
|
|
5901
|
+
}
|
|
5902
|
+
if (emitFn) {
|
|
5903
|
+
emitFn("eventual-consistency.stopped", {
|
|
5904
|
+
resource: resourceName,
|
|
5905
|
+
field: fieldName
|
|
5906
|
+
});
|
|
5412
5907
|
}
|
|
5413
|
-
return;
|
|
5414
5908
|
}
|
|
5415
|
-
|
|
5416
|
-
|
|
5417
|
-
|
|
5418
|
-
|
|
5419
|
-
|
|
5420
|
-
|
|
5421
|
-
|
|
5422
|
-
|
|
5423
|
-
|
|
5424
|
-
|
|
5425
|
-
|
|
5426
|
-
|
|
5427
|
-
|
|
5428
|
-
|
|
5429
|
-
|
|
5430
|
-
console.warn(`[EventualConsistency] ${errors.length} stale locks failed to delete`);
|
|
5431
|
-
}
|
|
5432
|
-
} catch (error) {
|
|
5433
|
-
if (this.config.verbose) {
|
|
5434
|
-
console.warn(`[EventualConsistency] Error cleaning up stale locks:`, error.message);
|
|
5909
|
+
}
|
|
5910
|
+
}
|
|
5911
|
+
|
|
5912
|
+
class EventualConsistencyPlugin extends Plugin {
|
|
5913
|
+
constructor(options = {}) {
|
|
5914
|
+
super(options);
|
|
5915
|
+
validateResourcesConfig(options.resources);
|
|
5916
|
+
const detectedTimezone = detectTimezone();
|
|
5917
|
+
const timezoneAutoDetected = !options.cohort?.timezone;
|
|
5918
|
+
this.config = createConfig(options, detectedTimezone);
|
|
5919
|
+
this.fieldHandlers = /* @__PURE__ */ new Map();
|
|
5920
|
+
for (const [resourceName, fields] of Object.entries(options.resources)) {
|
|
5921
|
+
const resourceHandlers = /* @__PURE__ */ new Map();
|
|
5922
|
+
for (const fieldName of fields) {
|
|
5923
|
+
resourceHandlers.set(fieldName, createFieldHandler(resourceName, fieldName));
|
|
5435
5924
|
}
|
|
5436
|
-
|
|
5437
|
-
await tryFn(() => this.lockResource.delete(cleanupLockId));
|
|
5925
|
+
this.fieldHandlers.set(resourceName, resourceHandlers);
|
|
5438
5926
|
}
|
|
5927
|
+
logConfigWarnings(this.config);
|
|
5928
|
+
logInitialization(this.config, this.fieldHandlers, timezoneAutoDetected);
|
|
5929
|
+
}
|
|
5930
|
+
/**
|
|
5931
|
+
* Setup hook - create resources and register helpers
|
|
5932
|
+
*/
|
|
5933
|
+
async onSetup() {
|
|
5934
|
+
await onSetup(
|
|
5935
|
+
this.database,
|
|
5936
|
+
this.fieldHandlers,
|
|
5937
|
+
(handler) => completeFieldSetup(handler, this.database, this.config, this),
|
|
5938
|
+
(resourceName) => watchForResource(
|
|
5939
|
+
resourceName,
|
|
5940
|
+
this.database,
|
|
5941
|
+
this.fieldHandlers,
|
|
5942
|
+
(handler) => completeFieldSetup(handler, this.database, this.config, this)
|
|
5943
|
+
)
|
|
5944
|
+
);
|
|
5945
|
+
}
|
|
5946
|
+
/**
|
|
5947
|
+
* Start hook - begin timers and emit events
|
|
5948
|
+
*/
|
|
5949
|
+
async onStart() {
|
|
5950
|
+
await onStart(
|
|
5951
|
+
this.fieldHandlers,
|
|
5952
|
+
this.config,
|
|
5953
|
+
(handler, resourceName, fieldName) => this._runConsolidationForHandler(handler, resourceName, fieldName),
|
|
5954
|
+
(handler, resourceName, fieldName) => this._runGarbageCollectionForHandler(handler, resourceName, fieldName),
|
|
5955
|
+
(event, data) => this.emit(event, data)
|
|
5956
|
+
);
|
|
5957
|
+
}
|
|
5958
|
+
/**
|
|
5959
|
+
* Stop hook - stop timers and flush pending
|
|
5960
|
+
*/
|
|
5961
|
+
async onStop() {
|
|
5962
|
+
await onStop(
|
|
5963
|
+
this.fieldHandlers,
|
|
5964
|
+
(event, data) => this.emit(event, data)
|
|
5965
|
+
);
|
|
5966
|
+
}
|
|
5967
|
+
/**
|
|
5968
|
+
* Create partition configuration
|
|
5969
|
+
* @returns {Object} Partition configuration
|
|
5970
|
+
*/
|
|
5971
|
+
createPartitionConfig() {
|
|
5972
|
+
return createPartitionConfig();
|
|
5973
|
+
}
|
|
5974
|
+
/**
|
|
5975
|
+
* Get cohort information for a date
|
|
5976
|
+
* @param {Date} date - Date to get cohort info for
|
|
5977
|
+
* @returns {Object} Cohort information
|
|
5978
|
+
*/
|
|
5979
|
+
getCohortInfo(date) {
|
|
5980
|
+
return getCohortInfo(date, this.config.cohort.timezone, this.config.verbose);
|
|
5981
|
+
}
|
|
5982
|
+
/**
|
|
5983
|
+
* Create a transaction for a field handler
|
|
5984
|
+
* @param {Object} handler - Field handler
|
|
5985
|
+
* @param {Object} data - Transaction data
|
|
5986
|
+
* @returns {Promise<Object|null>} Created transaction
|
|
5987
|
+
*/
|
|
5988
|
+
async createTransaction(handler, data) {
|
|
5989
|
+
return await createTransaction(handler, data, this.config);
|
|
5439
5990
|
}
|
|
5440
5991
|
/**
|
|
5441
|
-
*
|
|
5992
|
+
* Consolidate a single record (internal method)
|
|
5993
|
+
* This is used internally by consolidation timers and helper methods
|
|
5994
|
+
* @private
|
|
5442
5995
|
*/
|
|
5443
|
-
|
|
5444
|
-
|
|
5445
|
-
|
|
5446
|
-
|
|
5447
|
-
|
|
5996
|
+
async consolidateRecord(originalId) {
|
|
5997
|
+
return await consolidateRecord(
|
|
5998
|
+
originalId,
|
|
5999
|
+
this.transactionResource,
|
|
6000
|
+
this.targetResource,
|
|
6001
|
+
this.lockResource,
|
|
6002
|
+
this.analyticsResource,
|
|
6003
|
+
(transactions) => this.updateAnalytics(transactions),
|
|
6004
|
+
this.config
|
|
6005
|
+
);
|
|
5448
6006
|
}
|
|
5449
|
-
|
|
5450
|
-
|
|
5451
|
-
|
|
5452
|
-
|
|
5453
|
-
|
|
6007
|
+
/**
|
|
6008
|
+
* Get consolidated value without applying (internal method)
|
|
6009
|
+
* @private
|
|
6010
|
+
*/
|
|
6011
|
+
async getConsolidatedValue(originalId, options = {}) {
|
|
6012
|
+
return await getConsolidatedValue(
|
|
6013
|
+
originalId,
|
|
6014
|
+
options,
|
|
6015
|
+
this.transactionResource,
|
|
6016
|
+
this.targetResource,
|
|
6017
|
+
this.config
|
|
6018
|
+
);
|
|
5454
6019
|
}
|
|
5455
|
-
|
|
5456
|
-
|
|
5457
|
-
|
|
5458
|
-
|
|
5459
|
-
|
|
5460
|
-
|
|
5461
|
-
this.
|
|
5462
|
-
this.config.field = fieldName;
|
|
5463
|
-
this.transactionResource = handler.transactionResource;
|
|
5464
|
-
this.targetResource = handler.targetResource;
|
|
5465
|
-
this.lockResource = handler.lockResource;
|
|
5466
|
-
try {
|
|
5467
|
-
await this.runGarbageCollection();
|
|
5468
|
-
} finally {
|
|
5469
|
-
this.config.resource = oldResource;
|
|
5470
|
-
this.config.field = oldField;
|
|
5471
|
-
this.transactionResource = oldTransactionResource;
|
|
5472
|
-
this.targetResource = oldTargetResource;
|
|
5473
|
-
this.lockResource = oldLockResource;
|
|
5474
|
-
}
|
|
6020
|
+
/**
|
|
6021
|
+
* Get cohort statistics
|
|
6022
|
+
* @param {string} cohortDate - Cohort date
|
|
6023
|
+
* @returns {Promise<Object|null>} Cohort statistics
|
|
6024
|
+
*/
|
|
6025
|
+
async getCohortStats(cohortDate) {
|
|
6026
|
+
return await getCohortStats(cohortDate, this.transactionResource);
|
|
5475
6027
|
}
|
|
5476
6028
|
/**
|
|
5477
|
-
*
|
|
5478
|
-
*
|
|
6029
|
+
* Recalculate from scratch (internal method)
|
|
6030
|
+
* @private
|
|
5479
6031
|
*/
|
|
5480
|
-
async
|
|
5481
|
-
|
|
5482
|
-
|
|
5483
|
-
|
|
5484
|
-
|
|
5485
|
-
|
|
5486
|
-
|
|
5487
|
-
|
|
6032
|
+
async recalculateRecord(originalId) {
|
|
6033
|
+
return await recalculateRecord(
|
|
6034
|
+
originalId,
|
|
6035
|
+
this.transactionResource,
|
|
6036
|
+
this.targetResource,
|
|
6037
|
+
this.lockResource,
|
|
6038
|
+
(id) => this.consolidateRecord(id),
|
|
6039
|
+
this.config
|
|
5488
6040
|
);
|
|
5489
|
-
if (!lockAcquired) {
|
|
5490
|
-
if (this.config.verbose) {
|
|
5491
|
-
console.log(`[EventualConsistency] GC already running in another container`);
|
|
5492
|
-
}
|
|
5493
|
-
return;
|
|
5494
|
-
}
|
|
5495
|
-
try {
|
|
5496
|
-
const now = Date.now();
|
|
5497
|
-
const retentionMs = this.config.transactionRetention * 24 * 60 * 60 * 1e3;
|
|
5498
|
-
const cutoffDate = new Date(now - retentionMs);
|
|
5499
|
-
const cutoffIso = cutoffDate.toISOString();
|
|
5500
|
-
if (this.config.verbose) {
|
|
5501
|
-
console.log(`[EventualConsistency] Running GC for transactions older than ${cutoffIso} (${this.config.transactionRetention} days)`);
|
|
5502
|
-
}
|
|
5503
|
-
const cutoffMonth = cutoffDate.toISOString().substring(0, 7);
|
|
5504
|
-
const [ok, err, oldTransactions] = await tryFn(
|
|
5505
|
-
() => this.transactionResource.query({
|
|
5506
|
-
applied: true,
|
|
5507
|
-
timestamp: { "<": cutoffIso }
|
|
5508
|
-
})
|
|
5509
|
-
);
|
|
5510
|
-
if (!ok) {
|
|
5511
|
-
if (this.config.verbose) {
|
|
5512
|
-
console.warn(`[EventualConsistency] GC failed to query transactions:`, err?.message);
|
|
5513
|
-
}
|
|
5514
|
-
return;
|
|
5515
|
-
}
|
|
5516
|
-
if (!oldTransactions || oldTransactions.length === 0) {
|
|
5517
|
-
if (this.config.verbose) {
|
|
5518
|
-
console.log(`[EventualConsistency] No old transactions to clean up`);
|
|
5519
|
-
}
|
|
5520
|
-
return;
|
|
5521
|
-
}
|
|
5522
|
-
if (this.config.verbose) {
|
|
5523
|
-
console.log(`[EventualConsistency] Deleting ${oldTransactions.length} old transactions`);
|
|
5524
|
-
}
|
|
5525
|
-
const { results, errors } = await promisePool.PromisePool.for(oldTransactions).withConcurrency(10).process(async (txn) => {
|
|
5526
|
-
const [deleted] = await tryFn(() => this.transactionResource.delete(txn.id));
|
|
5527
|
-
return deleted;
|
|
5528
|
-
});
|
|
5529
|
-
if (this.config.verbose) {
|
|
5530
|
-
console.log(`[EventualConsistency] GC completed: ${results.length} deleted, ${errors.length} errors`);
|
|
5531
|
-
}
|
|
5532
|
-
this.emit("eventual-consistency.gc-completed", {
|
|
5533
|
-
resource: this.config.resource,
|
|
5534
|
-
field: this.config.field,
|
|
5535
|
-
deletedCount: results.length,
|
|
5536
|
-
errorCount: errors.length
|
|
5537
|
-
});
|
|
5538
|
-
} catch (error) {
|
|
5539
|
-
if (this.config.verbose) {
|
|
5540
|
-
console.warn(`[EventualConsistency] GC error:`, error.message);
|
|
5541
|
-
}
|
|
5542
|
-
this.emit("eventual-consistency.gc-error", error);
|
|
5543
|
-
} finally {
|
|
5544
|
-
await tryFn(() => this.lockResource.delete(gcLockId));
|
|
5545
|
-
}
|
|
5546
6041
|
}
|
|
5547
6042
|
/**
|
|
5548
|
-
* Update analytics
|
|
5549
|
-
* @param {Array} transactions - Array of transactions that were just consolidated
|
|
6043
|
+
* Update analytics
|
|
5550
6044
|
* @private
|
|
5551
6045
|
*/
|
|
5552
6046
|
async updateAnalytics(transactions) {
|
|
5553
|
-
|
|
5554
|
-
if (this.config.verbose) {
|
|
5555
|
-
console.log(
|
|
5556
|
-
`[EventualConsistency] ${this.config.resource}.${this.config.field} - Updating analytics for ${transactions.length} transactions...`
|
|
5557
|
-
);
|
|
5558
|
-
}
|
|
5559
|
-
try {
|
|
5560
|
-
const byHour = this._groupByCohort(transactions, "cohortHour");
|
|
5561
|
-
const cohortCount = Object.keys(byHour).length;
|
|
5562
|
-
if (this.config.verbose) {
|
|
5563
|
-
console.log(
|
|
5564
|
-
`[EventualConsistency] ${this.config.resource}.${this.config.field} - Updating ${cohortCount} hourly analytics cohorts...`
|
|
5565
|
-
);
|
|
5566
|
-
}
|
|
5567
|
-
for (const [cohort, txns] of Object.entries(byHour)) {
|
|
5568
|
-
await this._upsertAnalytics("hour", cohort, txns);
|
|
5569
|
-
}
|
|
5570
|
-
if (this.config.analyticsConfig.rollupStrategy === "incremental") {
|
|
5571
|
-
const uniqueHours = Object.keys(byHour);
|
|
5572
|
-
if (this.config.verbose) {
|
|
5573
|
-
console.log(
|
|
5574
|
-
`[EventualConsistency] ${this.config.resource}.${this.config.field} - Rolling up ${uniqueHours.length} hours to daily/monthly analytics...`
|
|
5575
|
-
);
|
|
5576
|
-
}
|
|
5577
|
-
for (const cohortHour of uniqueHours) {
|
|
5578
|
-
await this._rollupAnalytics(cohortHour);
|
|
5579
|
-
}
|
|
5580
|
-
}
|
|
5581
|
-
if (this.config.verbose) {
|
|
5582
|
-
console.log(
|
|
5583
|
-
`[EventualConsistency] ${this.config.resource}.${this.config.field} - Analytics update complete for ${cohortCount} cohorts`
|
|
5584
|
-
);
|
|
5585
|
-
}
|
|
5586
|
-
} catch (error) {
|
|
5587
|
-
console.warn(
|
|
5588
|
-
`[EventualConsistency] ${this.config.resource}.${this.config.field} - Analytics update error:`,
|
|
5589
|
-
error.message
|
|
5590
|
-
);
|
|
5591
|
-
}
|
|
6047
|
+
return await updateAnalytics(transactions, this.analyticsResource, this.config);
|
|
5592
6048
|
}
|
|
5593
6049
|
/**
|
|
5594
|
-
*
|
|
6050
|
+
* Helper method for sync mode consolidation
|
|
5595
6051
|
* @private
|
|
5596
6052
|
*/
|
|
5597
|
-
|
|
5598
|
-
const
|
|
5599
|
-
|
|
5600
|
-
|
|
5601
|
-
|
|
5602
|
-
|
|
5603
|
-
|
|
5604
|
-
|
|
5605
|
-
|
|
5606
|
-
|
|
5607
|
-
|
|
6053
|
+
async _syncModeConsolidate(handler, id, field) {
|
|
6054
|
+
const oldResource = this.config.resource;
|
|
6055
|
+
const oldField = this.config.field;
|
|
6056
|
+
const oldTransactionResource = this.transactionResource;
|
|
6057
|
+
const oldTargetResource = this.targetResource;
|
|
6058
|
+
const oldLockResource = this.lockResource;
|
|
6059
|
+
const oldAnalyticsResource = this.analyticsResource;
|
|
6060
|
+
this.config.resource = handler.resource;
|
|
6061
|
+
this.config.field = handler.field;
|
|
6062
|
+
this.transactionResource = handler.transactionResource;
|
|
6063
|
+
this.targetResource = handler.targetResource;
|
|
6064
|
+
this.lockResource = handler.lockResource;
|
|
6065
|
+
this.analyticsResource = handler.analyticsResource;
|
|
6066
|
+
const result = await this.consolidateRecord(id);
|
|
6067
|
+
this.config.resource = oldResource;
|
|
6068
|
+
this.config.field = oldField;
|
|
6069
|
+
this.transactionResource = oldTransactionResource;
|
|
6070
|
+
this.targetResource = oldTargetResource;
|
|
6071
|
+
this.lockResource = oldLockResource;
|
|
6072
|
+
this.analyticsResource = oldAnalyticsResource;
|
|
6073
|
+
return result;
|
|
5608
6074
|
}
|
|
5609
6075
|
/**
|
|
5610
|
-
*
|
|
6076
|
+
* Helper method for consolidate with handler
|
|
5611
6077
|
* @private
|
|
5612
6078
|
*/
|
|
5613
|
-
async
|
|
5614
|
-
const
|
|
5615
|
-
const
|
|
5616
|
-
const
|
|
5617
|
-
|
|
5618
|
-
|
|
5619
|
-
|
|
5620
|
-
|
|
5621
|
-
|
|
5622
|
-
|
|
5623
|
-
|
|
5624
|
-
|
|
5625
|
-
|
|
5626
|
-
const
|
|
5627
|
-
|
|
5628
|
-
|
|
5629
|
-
|
|
5630
|
-
|
|
5631
|
-
|
|
5632
|
-
|
|
5633
|
-
|
|
5634
|
-
const newMinValue = Math.min(existing.minValue, minValue);
|
|
5635
|
-
const newMaxValue = Math.max(existing.maxValue, maxValue);
|
|
5636
|
-
const newOperations = { ...existing.operations };
|
|
5637
|
-
for (const [op, stats] of Object.entries(operations)) {
|
|
5638
|
-
if (!newOperations[op]) {
|
|
5639
|
-
newOperations[op] = { count: 0, sum: 0 };
|
|
5640
|
-
}
|
|
5641
|
-
newOperations[op].count += stats.count;
|
|
5642
|
-
newOperations[op].sum += stats.sum;
|
|
5643
|
-
}
|
|
5644
|
-
const newRecordCount = Math.max(existing.recordCount, recordCount);
|
|
5645
|
-
await tryFn(
|
|
5646
|
-
() => this.analyticsResource.update(id, {
|
|
5647
|
-
transactionCount: newTransactionCount,
|
|
5648
|
-
totalValue: newTotalValue,
|
|
5649
|
-
avgValue: newAvgValue,
|
|
5650
|
-
minValue: newMinValue,
|
|
5651
|
-
maxValue: newMaxValue,
|
|
5652
|
-
operations: newOperations,
|
|
5653
|
-
recordCount: newRecordCount,
|
|
5654
|
-
updatedAt: now
|
|
5655
|
-
})
|
|
5656
|
-
);
|
|
5657
|
-
} else {
|
|
5658
|
-
await tryFn(
|
|
5659
|
-
() => this.analyticsResource.insert({
|
|
5660
|
-
id,
|
|
5661
|
-
period,
|
|
5662
|
-
cohort,
|
|
5663
|
-
transactionCount,
|
|
5664
|
-
totalValue,
|
|
5665
|
-
avgValue,
|
|
5666
|
-
minValue,
|
|
5667
|
-
maxValue,
|
|
5668
|
-
operations,
|
|
5669
|
-
recordCount,
|
|
5670
|
-
consolidatedAt: now,
|
|
5671
|
-
updatedAt: now
|
|
5672
|
-
})
|
|
5673
|
-
);
|
|
5674
|
-
}
|
|
6079
|
+
async _consolidateWithHandler(handler, id) {
|
|
6080
|
+
const oldResource = this.config.resource;
|
|
6081
|
+
const oldField = this.config.field;
|
|
6082
|
+
const oldTransactionResource = this.transactionResource;
|
|
6083
|
+
const oldTargetResource = this.targetResource;
|
|
6084
|
+
const oldLockResource = this.lockResource;
|
|
6085
|
+
const oldAnalyticsResource = this.analyticsResource;
|
|
6086
|
+
this.config.resource = handler.resource;
|
|
6087
|
+
this.config.field = handler.field;
|
|
6088
|
+
this.transactionResource = handler.transactionResource;
|
|
6089
|
+
this.targetResource = handler.targetResource;
|
|
6090
|
+
this.lockResource = handler.lockResource;
|
|
6091
|
+
this.analyticsResource = handler.analyticsResource;
|
|
6092
|
+
const result = await this.consolidateRecord(id);
|
|
6093
|
+
this.config.resource = oldResource;
|
|
6094
|
+
this.config.field = oldField;
|
|
6095
|
+
this.transactionResource = oldTransactionResource;
|
|
6096
|
+
this.targetResource = oldTargetResource;
|
|
6097
|
+
this.lockResource = oldLockResource;
|
|
6098
|
+
this.analyticsResource = oldAnalyticsResource;
|
|
6099
|
+
return result;
|
|
5675
6100
|
}
|
|
5676
6101
|
/**
|
|
5677
|
-
*
|
|
6102
|
+
* Helper method for getConsolidatedValue with handler
|
|
5678
6103
|
* @private
|
|
5679
6104
|
*/
|
|
5680
|
-
|
|
5681
|
-
const
|
|
5682
|
-
|
|
5683
|
-
|
|
5684
|
-
|
|
5685
|
-
|
|
5686
|
-
|
|
5687
|
-
|
|
5688
|
-
|
|
5689
|
-
|
|
5690
|
-
|
|
5691
|
-
|
|
6105
|
+
async _getConsolidatedValueWithHandler(handler, id, options) {
|
|
6106
|
+
const oldResource = this.config.resource;
|
|
6107
|
+
const oldField = this.config.field;
|
|
6108
|
+
const oldTransactionResource = this.transactionResource;
|
|
6109
|
+
const oldTargetResource = this.targetResource;
|
|
6110
|
+
this.config.resource = handler.resource;
|
|
6111
|
+
this.config.field = handler.field;
|
|
6112
|
+
this.transactionResource = handler.transactionResource;
|
|
6113
|
+
this.targetResource = handler.targetResource;
|
|
6114
|
+
const result = await this.getConsolidatedValue(id, options);
|
|
6115
|
+
this.config.resource = oldResource;
|
|
6116
|
+
this.config.field = oldField;
|
|
6117
|
+
this.transactionResource = oldTransactionResource;
|
|
6118
|
+
this.targetResource = oldTargetResource;
|
|
6119
|
+
return result;
|
|
5692
6120
|
}
|
|
5693
6121
|
/**
|
|
5694
|
-
*
|
|
6122
|
+
* Helper method for recalculate with handler
|
|
5695
6123
|
* @private
|
|
5696
6124
|
*/
|
|
5697
|
-
async
|
|
5698
|
-
const
|
|
5699
|
-
const
|
|
5700
|
-
|
|
5701
|
-
|
|
6125
|
+
async _recalculateWithHandler(handler, id) {
|
|
6126
|
+
const oldResource = this.config.resource;
|
|
6127
|
+
const oldField = this.config.field;
|
|
6128
|
+
const oldTransactionResource = this.transactionResource;
|
|
6129
|
+
const oldTargetResource = this.targetResource;
|
|
6130
|
+
const oldLockResource = this.lockResource;
|
|
6131
|
+
const oldAnalyticsResource = this.analyticsResource;
|
|
6132
|
+
this.config.resource = handler.resource;
|
|
6133
|
+
this.config.field = handler.field;
|
|
6134
|
+
this.transactionResource = handler.transactionResource;
|
|
6135
|
+
this.targetResource = handler.targetResource;
|
|
6136
|
+
this.lockResource = handler.lockResource;
|
|
6137
|
+
this.analyticsResource = handler.analyticsResource;
|
|
6138
|
+
const result = await this.recalculateRecord(id);
|
|
6139
|
+
this.config.resource = oldResource;
|
|
6140
|
+
this.config.field = oldField;
|
|
6141
|
+
this.transactionResource = oldTransactionResource;
|
|
6142
|
+
this.targetResource = oldTargetResource;
|
|
6143
|
+
this.lockResource = oldLockResource;
|
|
6144
|
+
this.analyticsResource = oldAnalyticsResource;
|
|
6145
|
+
return result;
|
|
5702
6146
|
}
|
|
5703
6147
|
/**
|
|
5704
|
-
*
|
|
6148
|
+
* Run consolidation for a handler
|
|
5705
6149
|
* @private
|
|
5706
6150
|
*/
|
|
5707
|
-
async
|
|
5708
|
-
const
|
|
5709
|
-
const
|
|
5710
|
-
|
|
5711
|
-
|
|
5712
|
-
|
|
5713
|
-
const
|
|
5714
|
-
|
|
5715
|
-
|
|
5716
|
-
|
|
5717
|
-
|
|
5718
|
-
|
|
5719
|
-
|
|
5720
|
-
|
|
5721
|
-
|
|
5722
|
-
|
|
5723
|
-
|
|
5724
|
-
|
|
5725
|
-
|
|
5726
|
-
operations[op] = { count: 0, sum: 0 };
|
|
5727
|
-
}
|
|
5728
|
-
operations[op].count += stats.count;
|
|
5729
|
-
operations[op].sum += stats.sum;
|
|
5730
|
-
}
|
|
5731
|
-
}
|
|
5732
|
-
const recordCount = Math.max(...sourceAnalytics.map((a) => a.recordCount));
|
|
5733
|
-
const id = `${period}-${cohort}`;
|
|
5734
|
-
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
5735
|
-
const [existingOk, existingErr, existing] = await tryFn(
|
|
5736
|
-
() => this.analyticsResource.get(id)
|
|
5737
|
-
);
|
|
5738
|
-
if (existingOk && existing) {
|
|
5739
|
-
await tryFn(
|
|
5740
|
-
() => this.analyticsResource.update(id, {
|
|
5741
|
-
transactionCount,
|
|
5742
|
-
totalValue,
|
|
5743
|
-
avgValue,
|
|
5744
|
-
minValue,
|
|
5745
|
-
maxValue,
|
|
5746
|
-
operations,
|
|
5747
|
-
recordCount,
|
|
5748
|
-
updatedAt: now
|
|
5749
|
-
})
|
|
6151
|
+
async _runConsolidationForHandler(handler, resourceName, fieldName) {
|
|
6152
|
+
const oldResource = this.config.resource;
|
|
6153
|
+
const oldField = this.config.field;
|
|
6154
|
+
const oldTransactionResource = this.transactionResource;
|
|
6155
|
+
const oldTargetResource = this.targetResource;
|
|
6156
|
+
const oldLockResource = this.lockResource;
|
|
6157
|
+
const oldAnalyticsResource = this.analyticsResource;
|
|
6158
|
+
this.config.resource = resourceName;
|
|
6159
|
+
this.config.field = fieldName;
|
|
6160
|
+
this.transactionResource = handler.transactionResource;
|
|
6161
|
+
this.targetResource = handler.targetResource;
|
|
6162
|
+
this.lockResource = handler.lockResource;
|
|
6163
|
+
this.analyticsResource = handler.analyticsResource;
|
|
6164
|
+
try {
|
|
6165
|
+
await runConsolidation(
|
|
6166
|
+
this.transactionResource,
|
|
6167
|
+
(id) => this.consolidateRecord(id),
|
|
6168
|
+
(event, data) => this.emit(event, data),
|
|
6169
|
+
this.config
|
|
5750
6170
|
);
|
|
5751
|
-
}
|
|
5752
|
-
|
|
5753
|
-
|
|
5754
|
-
|
|
5755
|
-
|
|
5756
|
-
|
|
5757
|
-
|
|
5758
|
-
|
|
5759
|
-
|
|
5760
|
-
|
|
5761
|
-
|
|
5762
|
-
|
|
5763
|
-
|
|
5764
|
-
|
|
5765
|
-
|
|
5766
|
-
|
|
6171
|
+
} finally {
|
|
6172
|
+
this.config.resource = oldResource;
|
|
6173
|
+
this.config.field = oldField;
|
|
6174
|
+
this.transactionResource = oldTransactionResource;
|
|
6175
|
+
this.targetResource = oldTargetResource;
|
|
6176
|
+
this.lockResource = oldLockResource;
|
|
6177
|
+
this.analyticsResource = oldAnalyticsResource;
|
|
6178
|
+
}
|
|
6179
|
+
}
|
|
6180
|
+
/**
|
|
6181
|
+
* Run garbage collection for a handler
|
|
6182
|
+
* @private
|
|
6183
|
+
*/
|
|
6184
|
+
async _runGarbageCollectionForHandler(handler, resourceName, fieldName) {
|
|
6185
|
+
const oldResource = this.config.resource;
|
|
6186
|
+
const oldField = this.config.field;
|
|
6187
|
+
const oldTransactionResource = this.transactionResource;
|
|
6188
|
+
const oldTargetResource = this.targetResource;
|
|
6189
|
+
const oldLockResource = this.lockResource;
|
|
6190
|
+
this.config.resource = resourceName;
|
|
6191
|
+
this.config.field = fieldName;
|
|
6192
|
+
this.transactionResource = handler.transactionResource;
|
|
6193
|
+
this.targetResource = handler.targetResource;
|
|
6194
|
+
this.lockResource = handler.lockResource;
|
|
6195
|
+
try {
|
|
6196
|
+
await runGarbageCollection(
|
|
6197
|
+
this.transactionResource,
|
|
6198
|
+
this.lockResource,
|
|
6199
|
+
this.config,
|
|
6200
|
+
(event, data) => this.emit(event, data)
|
|
5767
6201
|
);
|
|
6202
|
+
} finally {
|
|
6203
|
+
this.config.resource = oldResource;
|
|
6204
|
+
this.config.field = oldField;
|
|
6205
|
+
this.transactionResource = oldTransactionResource;
|
|
6206
|
+
this.targetResource = oldTargetResource;
|
|
6207
|
+
this.lockResource = oldLockResource;
|
|
5768
6208
|
}
|
|
5769
6209
|
}
|
|
6210
|
+
// Public Analytics API
|
|
5770
6211
|
/**
|
|
5771
6212
|
* Get analytics for a specific period
|
|
5772
6213
|
* @param {string} resourceName - Resource name
|
|
@@ -5775,111 +6216,7 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
5775
6216
|
* @returns {Promise<Array>} Analytics data
|
|
5776
6217
|
*/
|
|
5777
6218
|
async getAnalytics(resourceName, field, options = {}) {
|
|
5778
|
-
|
|
5779
|
-
if (!fieldHandlers) {
|
|
5780
|
-
throw new Error(`No eventual consistency configured for resource: ${resourceName}`);
|
|
5781
|
-
}
|
|
5782
|
-
const handler = fieldHandlers.get(field);
|
|
5783
|
-
if (!handler) {
|
|
5784
|
-
throw new Error(`No eventual consistency configured for field: ${resourceName}.${field}`);
|
|
5785
|
-
}
|
|
5786
|
-
if (!handler.analyticsResource) {
|
|
5787
|
-
throw new Error("Analytics not enabled for this plugin");
|
|
5788
|
-
}
|
|
5789
|
-
const { period = "day", date, startDate, endDate, month, year, breakdown = false } = options;
|
|
5790
|
-
const [ok, err, allAnalytics] = await tryFn(
|
|
5791
|
-
() => handler.analyticsResource.list()
|
|
5792
|
-
);
|
|
5793
|
-
if (!ok || !allAnalytics) {
|
|
5794
|
-
return [];
|
|
5795
|
-
}
|
|
5796
|
-
let filtered = allAnalytics.filter((a) => a.period === period);
|
|
5797
|
-
if (date) {
|
|
5798
|
-
if (period === "hour") {
|
|
5799
|
-
filtered = filtered.filter((a) => a.cohort.startsWith(date));
|
|
5800
|
-
} else {
|
|
5801
|
-
filtered = filtered.filter((a) => a.cohort === date);
|
|
5802
|
-
}
|
|
5803
|
-
} else if (startDate && endDate) {
|
|
5804
|
-
filtered = filtered.filter((a) => a.cohort >= startDate && a.cohort <= endDate);
|
|
5805
|
-
} else if (month) {
|
|
5806
|
-
filtered = filtered.filter((a) => a.cohort.startsWith(month));
|
|
5807
|
-
} else if (year) {
|
|
5808
|
-
filtered = filtered.filter((a) => a.cohort.startsWith(String(year)));
|
|
5809
|
-
}
|
|
5810
|
-
filtered.sort((a, b) => a.cohort.localeCompare(b.cohort));
|
|
5811
|
-
if (breakdown === "operations") {
|
|
5812
|
-
return filtered.map((a) => ({
|
|
5813
|
-
cohort: a.cohort,
|
|
5814
|
-
...a.operations
|
|
5815
|
-
}));
|
|
5816
|
-
}
|
|
5817
|
-
return filtered.map((a) => ({
|
|
5818
|
-
cohort: a.cohort,
|
|
5819
|
-
count: a.transactionCount,
|
|
5820
|
-
sum: a.totalValue,
|
|
5821
|
-
avg: a.avgValue,
|
|
5822
|
-
min: a.minValue,
|
|
5823
|
-
max: a.maxValue,
|
|
5824
|
-
operations: a.operations,
|
|
5825
|
-
recordCount: a.recordCount
|
|
5826
|
-
}));
|
|
5827
|
-
}
|
|
5828
|
-
/**
|
|
5829
|
-
* Fill gaps in analytics data with zeros for continuous time series
|
|
5830
|
-
* @private
|
|
5831
|
-
* @param {Array} data - Sparse analytics data
|
|
5832
|
-
* @param {string} period - Period type ('hour', 'day', 'month')
|
|
5833
|
-
* @param {string} startDate - Start date (ISO format)
|
|
5834
|
-
* @param {string} endDate - End date (ISO format)
|
|
5835
|
-
* @returns {Array} Complete time series with gaps filled
|
|
5836
|
-
*/
|
|
5837
|
-
_fillGaps(data, period, startDate, endDate) {
|
|
5838
|
-
if (!data || data.length === 0) {
|
|
5839
|
-
data = [];
|
|
5840
|
-
}
|
|
5841
|
-
const dataMap = /* @__PURE__ */ new Map();
|
|
5842
|
-
data.forEach((item) => {
|
|
5843
|
-
dataMap.set(item.cohort, item);
|
|
5844
|
-
});
|
|
5845
|
-
const result = [];
|
|
5846
|
-
const emptyRecord = {
|
|
5847
|
-
count: 0,
|
|
5848
|
-
sum: 0,
|
|
5849
|
-
avg: 0,
|
|
5850
|
-
min: 0,
|
|
5851
|
-
max: 0,
|
|
5852
|
-
recordCount: 0
|
|
5853
|
-
};
|
|
5854
|
-
if (period === "hour") {
|
|
5855
|
-
const start = /* @__PURE__ */ new Date(startDate + "T00:00:00Z");
|
|
5856
|
-
const end = /* @__PURE__ */ new Date(endDate + "T23:59:59Z");
|
|
5857
|
-
for (let dt = new Date(start); dt <= end; dt.setHours(dt.getHours() + 1)) {
|
|
5858
|
-
const cohort = dt.toISOString().substring(0, 13);
|
|
5859
|
-
result.push(dataMap.get(cohort) || { cohort, ...emptyRecord });
|
|
5860
|
-
}
|
|
5861
|
-
} else if (period === "day") {
|
|
5862
|
-
const start = new Date(startDate);
|
|
5863
|
-
const end = new Date(endDate);
|
|
5864
|
-
for (let dt = new Date(start); dt <= end; dt.setDate(dt.getDate() + 1)) {
|
|
5865
|
-
const cohort = dt.toISOString().substring(0, 10);
|
|
5866
|
-
result.push(dataMap.get(cohort) || { cohort, ...emptyRecord });
|
|
5867
|
-
}
|
|
5868
|
-
} else if (period === "month") {
|
|
5869
|
-
const startYear = parseInt(startDate.substring(0, 4));
|
|
5870
|
-
const startMonth = parseInt(startDate.substring(5, 7));
|
|
5871
|
-
const endYear = parseInt(endDate.substring(0, 4));
|
|
5872
|
-
const endMonth = parseInt(endDate.substring(5, 7));
|
|
5873
|
-
for (let year = startYear; year <= endYear; year++) {
|
|
5874
|
-
const firstMonth = year === startYear ? startMonth : 1;
|
|
5875
|
-
const lastMonth = year === endYear ? endMonth : 12;
|
|
5876
|
-
for (let month = firstMonth; month <= lastMonth; month++) {
|
|
5877
|
-
const cohort = `${year}-${month.toString().padStart(2, "0")}`;
|
|
5878
|
-
result.push(dataMap.get(cohort) || { cohort, ...emptyRecord });
|
|
5879
|
-
}
|
|
5880
|
-
}
|
|
5881
|
-
}
|
|
5882
|
-
return result;
|
|
6219
|
+
return await getAnalytics(resourceName, field, options, this.fieldHandlers);
|
|
5883
6220
|
}
|
|
5884
6221
|
/**
|
|
5885
6222
|
* Get analytics for entire month, broken down by days
|
|
@@ -5887,25 +6224,10 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
5887
6224
|
* @param {string} field - Field name
|
|
5888
6225
|
* @param {string} month - Month in YYYY-MM format
|
|
5889
6226
|
* @param {Object} options - Options
|
|
5890
|
-
* @param {boolean} options.fillGaps - Fill missing days with zeros (default: false)
|
|
5891
6227
|
* @returns {Promise<Array>} Daily analytics for the month
|
|
5892
6228
|
*/
|
|
5893
6229
|
async getMonthByDay(resourceName, field, month, options = {}) {
|
|
5894
|
-
|
|
5895
|
-
const monthNum = parseInt(month.substring(5, 7));
|
|
5896
|
-
const firstDay = new Date(year, monthNum - 1, 1);
|
|
5897
|
-
const lastDay = new Date(year, monthNum, 0);
|
|
5898
|
-
const startDate = firstDay.toISOString().substring(0, 10);
|
|
5899
|
-
const endDate = lastDay.toISOString().substring(0, 10);
|
|
5900
|
-
const data = await this.getAnalytics(resourceName, field, {
|
|
5901
|
-
period: "day",
|
|
5902
|
-
startDate,
|
|
5903
|
-
endDate
|
|
5904
|
-
});
|
|
5905
|
-
if (options.fillGaps) {
|
|
5906
|
-
return this._fillGaps(data, "day", startDate, endDate);
|
|
5907
|
-
}
|
|
5908
|
-
return data;
|
|
6230
|
+
return await getMonthByDay(resourceName, field, month, options, this.fieldHandlers);
|
|
5909
6231
|
}
|
|
5910
6232
|
/**
|
|
5911
6233
|
* Get analytics for entire day, broken down by hours
|
|
@@ -5913,18 +6235,10 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
5913
6235
|
* @param {string} field - Field name
|
|
5914
6236
|
* @param {string} date - Date in YYYY-MM-DD format
|
|
5915
6237
|
* @param {Object} options - Options
|
|
5916
|
-
* @param {boolean} options.fillGaps - Fill missing hours with zeros (default: false)
|
|
5917
6238
|
* @returns {Promise<Array>} Hourly analytics for the day
|
|
5918
6239
|
*/
|
|
5919
6240
|
async getDayByHour(resourceName, field, date, options = {}) {
|
|
5920
|
-
|
|
5921
|
-
period: "hour",
|
|
5922
|
-
date
|
|
5923
|
-
});
|
|
5924
|
-
if (options.fillGaps) {
|
|
5925
|
-
return this._fillGaps(data, "hour", date, date);
|
|
5926
|
-
}
|
|
5927
|
-
return data;
|
|
6241
|
+
return await getDayByHour(resourceName, field, date, options, this.fieldHandlers);
|
|
5928
6242
|
}
|
|
5929
6243
|
/**
|
|
5930
6244
|
* Get analytics for last N days, broken down by days
|
|
@@ -5932,24 +6246,10 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
5932
6246
|
* @param {string} field - Field name
|
|
5933
6247
|
* @param {number} days - Number of days to look back (default: 7)
|
|
5934
6248
|
* @param {Object} options - Options
|
|
5935
|
-
* @param {boolean} options.fillGaps - Fill missing days with zeros (default: false)
|
|
5936
6249
|
* @returns {Promise<Array>} Daily analytics
|
|
5937
6250
|
*/
|
|
5938
6251
|
async getLastNDays(resourceName, field, days = 7, options = {}) {
|
|
5939
|
-
|
|
5940
|
-
const date = /* @__PURE__ */ new Date();
|
|
5941
|
-
date.setDate(date.getDate() - i);
|
|
5942
|
-
return date.toISOString().substring(0, 10);
|
|
5943
|
-
}).reverse();
|
|
5944
|
-
const data = await this.getAnalytics(resourceName, field, {
|
|
5945
|
-
period: "day",
|
|
5946
|
-
startDate: dates[0],
|
|
5947
|
-
endDate: dates[dates.length - 1]
|
|
5948
|
-
});
|
|
5949
|
-
if (options.fillGaps) {
|
|
5950
|
-
return this._fillGaps(data, "day", dates[0], dates[dates.length - 1]);
|
|
5951
|
-
}
|
|
5952
|
-
return data;
|
|
6252
|
+
return await getLastNDays(resourceName, field, days, options, this.fieldHandlers);
|
|
5953
6253
|
}
|
|
5954
6254
|
/**
|
|
5955
6255
|
* Get analytics for entire year, broken down by months
|
|
@@ -5957,20 +6257,10 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
5957
6257
|
* @param {string} field - Field name
|
|
5958
6258
|
* @param {number} year - Year (e.g., 2025)
|
|
5959
6259
|
* @param {Object} options - Options
|
|
5960
|
-
* @param {boolean} options.fillGaps - Fill missing months with zeros (default: false)
|
|
5961
6260
|
* @returns {Promise<Array>} Monthly analytics for the year
|
|
5962
6261
|
*/
|
|
5963
6262
|
async getYearByMonth(resourceName, field, year, options = {}) {
|
|
5964
|
-
|
|
5965
|
-
period: "month",
|
|
5966
|
-
year
|
|
5967
|
-
});
|
|
5968
|
-
if (options.fillGaps) {
|
|
5969
|
-
const startDate = `${year}-01`;
|
|
5970
|
-
const endDate = `${year}-12`;
|
|
5971
|
-
return this._fillGaps(data, "month", startDate, endDate);
|
|
5972
|
-
}
|
|
5973
|
-
return data;
|
|
6263
|
+
return await getYearByMonth(resourceName, field, year, options, this.fieldHandlers);
|
|
5974
6264
|
}
|
|
5975
6265
|
/**
|
|
5976
6266
|
* Get analytics for entire month, broken down by hours
|
|
@@ -5978,33 +6268,10 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
5978
6268
|
* @param {string} field - Field name
|
|
5979
6269
|
* @param {string} month - Month in YYYY-MM format (or 'last' for previous month)
|
|
5980
6270
|
* @param {Object} options - Options
|
|
5981
|
-
* @
|
|
5982
|
-
* @returns {Promise<Array>} Hourly analytics for the month (up to 24*31=744 records)
|
|
6271
|
+
* @returns {Promise<Array>} Hourly analytics for the month
|
|
5983
6272
|
*/
|
|
5984
6273
|
async getMonthByHour(resourceName, field, month, options = {}) {
|
|
5985
|
-
|
|
5986
|
-
if (month === "last") {
|
|
5987
|
-
const now = /* @__PURE__ */ new Date();
|
|
5988
|
-
now.setMonth(now.getMonth() - 1);
|
|
5989
|
-
year = now.getFullYear();
|
|
5990
|
-
monthNum = now.getMonth() + 1;
|
|
5991
|
-
} else {
|
|
5992
|
-
year = parseInt(month.substring(0, 4));
|
|
5993
|
-
monthNum = parseInt(month.substring(5, 7));
|
|
5994
|
-
}
|
|
5995
|
-
const firstDay = new Date(year, monthNum - 1, 1);
|
|
5996
|
-
const lastDay = new Date(year, monthNum, 0);
|
|
5997
|
-
const startDate = firstDay.toISOString().substring(0, 10);
|
|
5998
|
-
const endDate = lastDay.toISOString().substring(0, 10);
|
|
5999
|
-
const data = await this.getAnalytics(resourceName, field, {
|
|
6000
|
-
period: "hour",
|
|
6001
|
-
startDate,
|
|
6002
|
-
endDate
|
|
6003
|
-
});
|
|
6004
|
-
if (options.fillGaps) {
|
|
6005
|
-
return this._fillGaps(data, "hour", startDate, endDate);
|
|
6006
|
-
}
|
|
6007
|
-
return data;
|
|
6274
|
+
return await getMonthByHour(resourceName, field, month, options, this.fieldHandlers);
|
|
6008
6275
|
}
|
|
6009
6276
|
/**
|
|
6010
6277
|
* Get top records by volume
|
|
@@ -6014,57 +6281,7 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
6014
6281
|
* @returns {Promise<Array>} Top records
|
|
6015
6282
|
*/
|
|
6016
6283
|
async getTopRecords(resourceName, field, options = {}) {
|
|
6017
|
-
|
|
6018
|
-
if (!fieldHandlers) {
|
|
6019
|
-
throw new Error(`No eventual consistency configured for resource: ${resourceName}`);
|
|
6020
|
-
}
|
|
6021
|
-
const handler = fieldHandlers.get(field);
|
|
6022
|
-
if (!handler) {
|
|
6023
|
-
throw new Error(`No eventual consistency configured for field: ${resourceName}.${field}`);
|
|
6024
|
-
}
|
|
6025
|
-
if (!handler.transactionResource) {
|
|
6026
|
-
throw new Error("Transaction resource not initialized");
|
|
6027
|
-
}
|
|
6028
|
-
const { period = "day", date, metric = "transactionCount", limit = 10 } = options;
|
|
6029
|
-
const [ok, err, transactions] = await tryFn(
|
|
6030
|
-
() => handler.transactionResource.list()
|
|
6031
|
-
);
|
|
6032
|
-
if (!ok || !transactions) {
|
|
6033
|
-
return [];
|
|
6034
|
-
}
|
|
6035
|
-
let filtered = transactions;
|
|
6036
|
-
if (date) {
|
|
6037
|
-
if (period === "hour") {
|
|
6038
|
-
filtered = transactions.filter((t) => t.cohortHour && t.cohortHour.startsWith(date));
|
|
6039
|
-
} else if (period === "day") {
|
|
6040
|
-
filtered = transactions.filter((t) => t.cohortDate === date);
|
|
6041
|
-
} else if (period === "month") {
|
|
6042
|
-
filtered = transactions.filter((t) => t.cohortMonth && t.cohortMonth.startsWith(date));
|
|
6043
|
-
}
|
|
6044
|
-
}
|
|
6045
|
-
const byRecord = {};
|
|
6046
|
-
for (const txn of filtered) {
|
|
6047
|
-
const recordId = txn.originalId;
|
|
6048
|
-
if (!byRecord[recordId]) {
|
|
6049
|
-
byRecord[recordId] = { count: 0, sum: 0 };
|
|
6050
|
-
}
|
|
6051
|
-
byRecord[recordId].count++;
|
|
6052
|
-
byRecord[recordId].sum += txn.value;
|
|
6053
|
-
}
|
|
6054
|
-
const records = Object.entries(byRecord).map(([recordId, stats]) => ({
|
|
6055
|
-
recordId,
|
|
6056
|
-
count: stats.count,
|
|
6057
|
-
sum: stats.sum
|
|
6058
|
-
}));
|
|
6059
|
-
records.sort((a, b) => {
|
|
6060
|
-
if (metric === "transactionCount") {
|
|
6061
|
-
return b.count - a.count;
|
|
6062
|
-
} else if (metric === "totalValue") {
|
|
6063
|
-
return b.sum - a.sum;
|
|
6064
|
-
}
|
|
6065
|
-
return 0;
|
|
6066
|
-
});
|
|
6067
|
-
return records.slice(0, limit);
|
|
6284
|
+
return await getTopRecords(resourceName, field, options, this.fieldHandlers);
|
|
6068
6285
|
}
|
|
6069
6286
|
}
|
|
6070
6287
|
|
|
@@ -12135,7 +12352,7 @@ class Database extends EventEmitter {
|
|
|
12135
12352
|
this.id = idGenerator(7);
|
|
12136
12353
|
this.version = "1";
|
|
12137
12354
|
this.s3dbVersion = (() => {
|
|
12138
|
-
const [ok, err, version] = tryFn(() => true ? "10.0.
|
|
12355
|
+
const [ok, err, version] = tryFn(() => true ? "10.0.18" : "latest");
|
|
12139
12356
|
return ok ? version : "latest";
|
|
12140
12357
|
})();
|
|
12141
12358
|
this.resources = {};
|