s3db.js 10.0.16 → 10.0.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/s3db.cjs.js +1766 -1549
- package/dist/s3db.cjs.js.map +1 -1
- package/dist/s3db.es.js +1766 -1549
- package/dist/s3db.es.js.map +1 -1
- package/package.json +1 -1
- package/src/plugins/eventual-consistency/analytics.js +668 -0
- package/src/plugins/eventual-consistency/config.js +156 -0
- package/src/plugins/eventual-consistency/consolidation.js +770 -0
- package/src/plugins/eventual-consistency/garbage-collection.js +126 -0
- package/src/plugins/eventual-consistency/helpers.js +179 -0
- package/src/plugins/eventual-consistency/index.js +455 -0
- package/src/plugins/eventual-consistency/locks.js +77 -0
- package/src/plugins/eventual-consistency/partitions.js +45 -0
- package/src/plugins/eventual-consistency/setup.js +298 -0
- package/src/plugins/eventual-consistency/transactions.js +119 -0
- package/src/plugins/eventual-consistency/utils.js +172 -0
- package/src/plugins/eventual-consistency.plugin.js +195 -2
- package/src/plugins/index.js +1 -1
package/dist/s3db.es.js
CHANGED
|
@@ -4303,1466 +4303,1907 @@ const CostsPlugin = {
|
|
|
4303
4303
|
}
|
|
4304
4304
|
};
|
|
4305
4305
|
|
|
4306
|
-
|
|
4307
|
-
|
|
4308
|
-
|
|
4309
|
-
|
|
4306
|
+
function createConfig(options, detectedTimezone) {
|
|
4307
|
+
const consolidation = options.consolidation || {};
|
|
4308
|
+
const locks = options.locks || {};
|
|
4309
|
+
const gc = options.garbageCollection || {};
|
|
4310
|
+
const analytics = options.analytics || {};
|
|
4311
|
+
const batch = options.batch || {};
|
|
4312
|
+
const lateArrivals = options.lateArrivals || {};
|
|
4313
|
+
const checkpoints = options.checkpoints || {};
|
|
4314
|
+
return {
|
|
4315
|
+
// Cohort (timezone)
|
|
4316
|
+
cohort: {
|
|
4317
|
+
timezone: options.cohort?.timezone || detectedTimezone
|
|
4318
|
+
},
|
|
4319
|
+
// Reducer function
|
|
4320
|
+
reducer: options.reducer || ((transactions) => {
|
|
4321
|
+
let baseValue = 0;
|
|
4322
|
+
for (const t of transactions) {
|
|
4323
|
+
if (t.operation === "set") {
|
|
4324
|
+
baseValue = t.value;
|
|
4325
|
+
} else if (t.operation === "add") {
|
|
4326
|
+
baseValue += t.value;
|
|
4327
|
+
} else if (t.operation === "sub") {
|
|
4328
|
+
baseValue -= t.value;
|
|
4329
|
+
}
|
|
4330
|
+
}
|
|
4331
|
+
return baseValue;
|
|
4332
|
+
}),
|
|
4333
|
+
// Consolidation settings
|
|
4334
|
+
consolidationInterval: consolidation.interval ?? 300,
|
|
4335
|
+
consolidationConcurrency: consolidation.concurrency ?? 5,
|
|
4336
|
+
consolidationWindow: consolidation.window ?? 24,
|
|
4337
|
+
autoConsolidate: consolidation.auto !== false,
|
|
4338
|
+
mode: consolidation.mode || "async",
|
|
4339
|
+
// Late arrivals
|
|
4340
|
+
lateArrivalStrategy: lateArrivals.strategy || "warn",
|
|
4341
|
+
// Batch transactions
|
|
4342
|
+
batchTransactions: batch.enabled || false,
|
|
4343
|
+
batchSize: batch.size || 100,
|
|
4344
|
+
// Locks
|
|
4345
|
+
lockTimeout: locks.timeout || 300,
|
|
4346
|
+
// Garbage collection
|
|
4347
|
+
transactionRetention: gc.retention ?? 30,
|
|
4348
|
+
gcInterval: gc.interval ?? 86400,
|
|
4349
|
+
// Analytics
|
|
4350
|
+
enableAnalytics: analytics.enabled || false,
|
|
4351
|
+
analyticsConfig: {
|
|
4352
|
+
periods: analytics.periods || ["hour", "day", "month"],
|
|
4353
|
+
metrics: analytics.metrics || ["count", "sum", "avg", "min", "max"],
|
|
4354
|
+
rollupStrategy: analytics.rollupStrategy || "incremental",
|
|
4355
|
+
retentionDays: analytics.retentionDays ?? 365
|
|
4356
|
+
},
|
|
4357
|
+
// Checkpoints
|
|
4358
|
+
enableCheckpoints: checkpoints.enabled !== false,
|
|
4359
|
+
checkpointStrategy: checkpoints.strategy || "hourly",
|
|
4360
|
+
checkpointRetention: checkpoints.retention ?? 90,
|
|
4361
|
+
checkpointThreshold: checkpoints.threshold ?? 1e3,
|
|
4362
|
+
deleteConsolidatedTransactions: checkpoints.deleteConsolidated !== false,
|
|
4363
|
+
autoCheckpoint: checkpoints.auto !== false,
|
|
4364
|
+
// Debug
|
|
4365
|
+
verbose: options.verbose || false
|
|
4366
|
+
};
|
|
4367
|
+
}
|
|
4368
|
+
function validateResourcesConfig(resources) {
|
|
4369
|
+
if (!resources || typeof resources !== "object") {
|
|
4370
|
+
throw new Error(
|
|
4371
|
+
"EventualConsistencyPlugin requires 'resources' option.\nExample: { resources: { urls: ['clicks', 'views'], posts: ['likes'] } }"
|
|
4372
|
+
);
|
|
4373
|
+
}
|
|
4374
|
+
for (const [resourceName, fields] of Object.entries(resources)) {
|
|
4375
|
+
if (!Array.isArray(fields)) {
|
|
4310
4376
|
throw new Error(
|
|
4311
|
-
|
|
4377
|
+
`EventualConsistencyPlugin resources.${resourceName} must be an array of field names`
|
|
4312
4378
|
);
|
|
4313
4379
|
}
|
|
4314
|
-
|
|
4315
|
-
|
|
4316
|
-
|
|
4317
|
-
|
|
4318
|
-
|
|
4319
|
-
|
|
4320
|
-
|
|
4321
|
-
|
|
4322
|
-
|
|
4323
|
-
|
|
4324
|
-
|
|
4325
|
-
|
|
4326
|
-
|
|
4327
|
-
|
|
4328
|
-
|
|
4329
|
-
|
|
4330
|
-
|
|
4331
|
-
|
|
4332
|
-
|
|
4333
|
-
|
|
4334
|
-
|
|
4335
|
-
|
|
4336
|
-
|
|
4337
|
-
|
|
4338
|
-
|
|
4339
|
-
|
|
4340
|
-
|
|
4341
|
-
|
|
4342
|
-
|
|
4343
|
-
|
|
4344
|
-
|
|
4345
|
-
|
|
4346
|
-
|
|
4347
|
-
|
|
4348
|
-
|
|
4349
|
-
|
|
4350
|
-
|
|
4380
|
+
}
|
|
4381
|
+
}
|
|
4382
|
+
function logConfigWarnings(config) {
|
|
4383
|
+
if (config.batchTransactions && !config.verbose) {
|
|
4384
|
+
console.warn(
|
|
4385
|
+
`[EventualConsistency] WARNING: batch.enabled is true. This stores transactions in memory and will lose data if container crashes. Not recommended for distributed/production environments.`
|
|
4386
|
+
);
|
|
4387
|
+
}
|
|
4388
|
+
if (!config.enableCheckpoints && !config.verbose) {
|
|
4389
|
+
console.warn(
|
|
4390
|
+
`[EventualConsistency] INFO: checkpoints.enabled is false. Checkpoints improve performance in high-volume scenarios by creating snapshots. Consider enabling for production use.`
|
|
4391
|
+
);
|
|
4392
|
+
}
|
|
4393
|
+
}
|
|
4394
|
+
function logInitialization(config, fieldHandlers, timezoneAutoDetected) {
|
|
4395
|
+
if (!config.verbose) return;
|
|
4396
|
+
const totalFields = Array.from(fieldHandlers.values()).reduce((sum, handlers) => sum + handlers.size, 0);
|
|
4397
|
+
console.log(
|
|
4398
|
+
`[EventualConsistency] Initialized with ${fieldHandlers.size} resource(s), ${totalFields} field(s) total`
|
|
4399
|
+
);
|
|
4400
|
+
if (timezoneAutoDetected) {
|
|
4401
|
+
console.log(
|
|
4402
|
+
`[EventualConsistency] Using timezone: ${config.cohort.timezone} (${process.env.TZ ? "from TZ env var" : "default UTC"})`
|
|
4403
|
+
);
|
|
4404
|
+
}
|
|
4405
|
+
}
|
|
4406
|
+
|
|
4407
|
+
function detectTimezone() {
|
|
4408
|
+
if (process.env.TZ) {
|
|
4409
|
+
return process.env.TZ;
|
|
4410
|
+
}
|
|
4411
|
+
return "UTC";
|
|
4412
|
+
}
|
|
4413
|
+
function getTimezoneOffset(timezone, verbose = false) {
|
|
4414
|
+
try {
|
|
4415
|
+
const now = /* @__PURE__ */ new Date();
|
|
4416
|
+
const utcDate = new Date(now.toLocaleString("en-US", { timeZone: "UTC" }));
|
|
4417
|
+
const tzDate = new Date(now.toLocaleString("en-US", { timeZone: timezone }));
|
|
4418
|
+
return tzDate.getTime() - utcDate.getTime();
|
|
4419
|
+
} catch (err) {
|
|
4420
|
+
const offsets = {
|
|
4421
|
+
"UTC": 0,
|
|
4422
|
+
"America/New_York": -5 * 36e5,
|
|
4423
|
+
"America/Chicago": -6 * 36e5,
|
|
4424
|
+
"America/Denver": -7 * 36e5,
|
|
4425
|
+
"America/Los_Angeles": -8 * 36e5,
|
|
4426
|
+
"America/Sao_Paulo": -3 * 36e5,
|
|
4427
|
+
"Europe/London": 0,
|
|
4428
|
+
"Europe/Paris": 1 * 36e5,
|
|
4429
|
+
"Europe/Berlin": 1 * 36e5,
|
|
4430
|
+
"Asia/Tokyo": 9 * 36e5,
|
|
4431
|
+
"Asia/Shanghai": 8 * 36e5,
|
|
4432
|
+
"Australia/Sydney": 10 * 36e5
|
|
4351
4433
|
};
|
|
4352
|
-
|
|
4353
|
-
|
|
4354
|
-
|
|
4355
|
-
|
|
4356
|
-
|
|
4357
|
-
|
|
4434
|
+
if (verbose && !offsets[timezone]) {
|
|
4435
|
+
console.warn(
|
|
4436
|
+
`[EventualConsistency] Unknown timezone '${timezone}', using UTC. Consider using a valid IANA timezone (e.g., 'America/New_York')`
|
|
4437
|
+
);
|
|
4438
|
+
}
|
|
4439
|
+
return offsets[timezone] || 0;
|
|
4440
|
+
}
|
|
4441
|
+
}
|
|
4442
|
+
function getCohortInfo(date, timezone, verbose = false) {
|
|
4443
|
+
const offset = getTimezoneOffset(timezone, verbose);
|
|
4444
|
+
const localDate = new Date(date.getTime() + offset);
|
|
4445
|
+
const year = localDate.getFullYear();
|
|
4446
|
+
const month = String(localDate.getMonth() + 1).padStart(2, "0");
|
|
4447
|
+
const day = String(localDate.getDate()).padStart(2, "0");
|
|
4448
|
+
const hour = String(localDate.getHours()).padStart(2, "0");
|
|
4449
|
+
return {
|
|
4450
|
+
date: `${year}-${month}-${day}`,
|
|
4451
|
+
hour: `${year}-${month}-${day}T${hour}`,
|
|
4452
|
+
// ISO-like format for hour partition
|
|
4453
|
+
month: `${year}-${month}`
|
|
4454
|
+
};
|
|
4455
|
+
}
|
|
4456
|
+
function createSyntheticSetTransaction(currentValue) {
|
|
4457
|
+
return {
|
|
4458
|
+
id: "__synthetic__",
|
|
4459
|
+
operation: "set",
|
|
4460
|
+
value: currentValue,
|
|
4461
|
+
timestamp: (/* @__PURE__ */ new Date(0)).toISOString(),
|
|
4462
|
+
synthetic: true
|
|
4463
|
+
};
|
|
4464
|
+
}
|
|
4465
|
+
function createFieldHandler(resourceName, fieldName) {
|
|
4466
|
+
return {
|
|
4467
|
+
resource: resourceName,
|
|
4468
|
+
field: fieldName,
|
|
4469
|
+
transactionResource: null,
|
|
4470
|
+
targetResource: null,
|
|
4471
|
+
analyticsResource: null,
|
|
4472
|
+
lockResource: null,
|
|
4473
|
+
checkpointResource: null,
|
|
4474
|
+
consolidationTimer: null,
|
|
4475
|
+
gcTimer: null,
|
|
4476
|
+
pendingTransactions: /* @__PURE__ */ new Map(),
|
|
4477
|
+
deferredSetup: false
|
|
4478
|
+
};
|
|
4479
|
+
}
|
|
4480
|
+
function resolveFieldAndPlugin(resource, field, value) {
|
|
4481
|
+
if (!resource._eventualConsistencyPlugins) {
|
|
4482
|
+
throw new Error(`No eventual consistency plugins configured for this resource`);
|
|
4483
|
+
}
|
|
4484
|
+
const fieldPlugin = resource._eventualConsistencyPlugins[field];
|
|
4485
|
+
if (!fieldPlugin) {
|
|
4486
|
+
const availableFields = Object.keys(resource._eventualConsistencyPlugins).join(", ");
|
|
4487
|
+
throw new Error(
|
|
4488
|
+
`No eventual consistency plugin found for field "${field}". Available fields: ${availableFields}`
|
|
4489
|
+
);
|
|
4490
|
+
}
|
|
4491
|
+
return { field, value, plugin: fieldPlugin };
|
|
4492
|
+
}
|
|
4493
|
+
function groupByCohort(transactions, cohortField) {
|
|
4494
|
+
const groups = {};
|
|
4495
|
+
for (const txn of transactions) {
|
|
4496
|
+
const cohort = txn[cohortField];
|
|
4497
|
+
if (!cohort) continue;
|
|
4498
|
+
if (!groups[cohort]) {
|
|
4499
|
+
groups[cohort] = [];
|
|
4500
|
+
}
|
|
4501
|
+
groups[cohort].push(txn);
|
|
4502
|
+
}
|
|
4503
|
+
return groups;
|
|
4504
|
+
}
|
|
4505
|
+
|
|
4506
|
+
function createPartitionConfig() {
|
|
4507
|
+
const partitions = {
|
|
4508
|
+
// Composite partition by originalId + applied status
|
|
4509
|
+
// This is THE MOST CRITICAL optimization for consolidation!
|
|
4510
|
+
// Why: Consolidation always queries { originalId, applied: false }
|
|
4511
|
+
// Without this: Reads ALL transactions (applied + pending) and filters manually
|
|
4512
|
+
// With this: Reads ONLY pending transactions - can be 1000x faster!
|
|
4513
|
+
byOriginalIdAndApplied: {
|
|
4514
|
+
fields: {
|
|
4515
|
+
originalId: "string",
|
|
4516
|
+
applied: "boolean"
|
|
4358
4517
|
}
|
|
4359
|
-
|
|
4360
|
-
|
|
4361
|
-
|
|
4518
|
+
},
|
|
4519
|
+
// Partition by time cohorts for batch consolidation across many records
|
|
4520
|
+
byHour: {
|
|
4521
|
+
fields: {
|
|
4522
|
+
cohortHour: "string"
|
|
4523
|
+
}
|
|
4524
|
+
},
|
|
4525
|
+
byDay: {
|
|
4526
|
+
fields: {
|
|
4527
|
+
cohortDate: "string"
|
|
4528
|
+
}
|
|
4529
|
+
},
|
|
4530
|
+
byMonth: {
|
|
4531
|
+
fields: {
|
|
4532
|
+
cohortMonth: "string"
|
|
4362
4533
|
}
|
|
4363
|
-
this.fieldHandlers.set(resourceName, resourceHandlers);
|
|
4364
4534
|
}
|
|
4365
|
-
|
|
4535
|
+
};
|
|
4536
|
+
return partitions;
|
|
4537
|
+
}
|
|
4538
|
+
|
|
4539
|
+
async function createTransaction(handler, data, config) {
|
|
4540
|
+
const now = /* @__PURE__ */ new Date();
|
|
4541
|
+
const cohortInfo = getCohortInfo(now, config.cohort.timezone, config.verbose);
|
|
4542
|
+
const watermarkMs = config.consolidationWindow * 60 * 60 * 1e3;
|
|
4543
|
+
const watermarkTime = now.getTime() - watermarkMs;
|
|
4544
|
+
const cohortHourDate = /* @__PURE__ */ new Date(cohortInfo.hour + ":00:00Z");
|
|
4545
|
+
if (cohortHourDate.getTime() < watermarkTime) {
|
|
4546
|
+
const hoursLate = Math.floor((now.getTime() - cohortHourDate.getTime()) / (60 * 60 * 1e3));
|
|
4547
|
+
if (config.lateArrivalStrategy === "ignore") {
|
|
4548
|
+
if (config.verbose) {
|
|
4549
|
+
console.warn(
|
|
4550
|
+
`[EventualConsistency] Late arrival ignored: transaction for ${cohortInfo.hour} is ${hoursLate}h late (watermark: ${config.consolidationWindow}h)`
|
|
4551
|
+
);
|
|
4552
|
+
}
|
|
4553
|
+
return null;
|
|
4554
|
+
} else if (config.lateArrivalStrategy === "warn") {
|
|
4366
4555
|
console.warn(
|
|
4367
|
-
`[EventualConsistency]
|
|
4556
|
+
`[EventualConsistency] Late arrival detected: transaction for ${cohortInfo.hour} is ${hoursLate}h late (watermark: ${config.consolidationWindow}h). Processing anyway, but consolidation may not pick it up.`
|
|
4368
4557
|
);
|
|
4369
4558
|
}
|
|
4370
|
-
|
|
4371
|
-
|
|
4559
|
+
}
|
|
4560
|
+
const transaction = {
|
|
4561
|
+
id: idGenerator(),
|
|
4562
|
+
originalId: data.originalId,
|
|
4563
|
+
field: handler.field,
|
|
4564
|
+
value: data.value || 0,
|
|
4565
|
+
operation: data.operation || "set",
|
|
4566
|
+
timestamp: now.toISOString(),
|
|
4567
|
+
cohortDate: cohortInfo.date,
|
|
4568
|
+
cohortHour: cohortInfo.hour,
|
|
4569
|
+
cohortMonth: cohortInfo.month,
|
|
4570
|
+
source: data.source || "unknown",
|
|
4571
|
+
applied: false
|
|
4572
|
+
};
|
|
4573
|
+
if (config.batchTransactions) {
|
|
4574
|
+
handler.pendingTransactions.set(transaction.id, transaction);
|
|
4575
|
+
if (config.verbose) {
|
|
4372
4576
|
console.log(
|
|
4373
|
-
`[EventualConsistency]
|
|
4577
|
+
`[EventualConsistency] ${handler.resource}.${handler.field} - Transaction batched: ${data.operation} ${data.value} for ${data.originalId} (batch: ${handler.pendingTransactions.size}/${config.batchSize})`
|
|
4578
|
+
);
|
|
4579
|
+
}
|
|
4580
|
+
if (handler.pendingTransactions.size >= config.batchSize) {
|
|
4581
|
+
await flushPendingTransactions(handler);
|
|
4582
|
+
}
|
|
4583
|
+
} else {
|
|
4584
|
+
await handler.transactionResource.insert(transaction);
|
|
4585
|
+
if (config.verbose) {
|
|
4586
|
+
console.log(
|
|
4587
|
+
`[EventualConsistency] ${handler.resource}.${handler.field} - Transaction created: ${data.operation} ${data.value} for ${data.originalId} (cohort: ${cohortInfo.hour}, applied: false)`
|
|
4374
4588
|
);
|
|
4375
|
-
if (!options.cohort?.timezone) {
|
|
4376
|
-
console.log(
|
|
4377
|
-
`[EventualConsistency] Auto-detected timezone: ${this.config.cohort.timezone} (from ${process.env.TZ ? "TZ env var" : "system Intl API"})`
|
|
4378
|
-
);
|
|
4379
|
-
}
|
|
4380
4589
|
}
|
|
4381
4590
|
}
|
|
4382
|
-
|
|
4383
|
-
|
|
4384
|
-
|
|
4385
|
-
|
|
4386
|
-
|
|
4387
|
-
|
|
4388
|
-
|
|
4389
|
-
|
|
4390
|
-
|
|
4391
|
-
|
|
4392
|
-
|
|
4393
|
-
|
|
4394
|
-
|
|
4395
|
-
|
|
4396
|
-
|
|
4397
|
-
deferredSetup: false
|
|
4398
|
-
};
|
|
4591
|
+
return transaction;
|
|
4592
|
+
}
|
|
4593
|
+
async function flushPendingTransactions(handler) {
|
|
4594
|
+
if (handler.pendingTransactions.size === 0) return;
|
|
4595
|
+
const transactions = Array.from(handler.pendingTransactions.values());
|
|
4596
|
+
try {
|
|
4597
|
+
await Promise.all(
|
|
4598
|
+
transactions.map(
|
|
4599
|
+
(transaction) => handler.transactionResource.insert(transaction)
|
|
4600
|
+
)
|
|
4601
|
+
);
|
|
4602
|
+
handler.pendingTransactions.clear();
|
|
4603
|
+
} catch (error) {
|
|
4604
|
+
console.error("Failed to flush pending transactions:", error);
|
|
4605
|
+
throw error;
|
|
4399
4606
|
}
|
|
4400
|
-
|
|
4401
|
-
|
|
4402
|
-
|
|
4403
|
-
|
|
4404
|
-
|
|
4405
|
-
|
|
4406
|
-
|
|
4407
|
-
|
|
4408
|
-
|
|
4409
|
-
|
|
4410
|
-
|
|
4411
|
-
|
|
4412
|
-
|
|
4413
|
-
|
|
4607
|
+
}
|
|
4608
|
+
|
|
4609
|
+
async function cleanupStaleLocks(lockResource, config) {
|
|
4610
|
+
const now = Date.now();
|
|
4611
|
+
const lockTimeoutMs = config.lockTimeout * 1e3;
|
|
4612
|
+
const cutoffTime = now - lockTimeoutMs;
|
|
4613
|
+
const cleanupLockId = `lock-cleanup-${config.resource}-${config.field}`;
|
|
4614
|
+
const [lockAcquired] = await tryFn(
|
|
4615
|
+
() => lockResource.insert({
|
|
4616
|
+
id: cleanupLockId,
|
|
4617
|
+
lockedAt: Date.now(),
|
|
4618
|
+
workerId: process.pid ? String(process.pid) : "unknown"
|
|
4619
|
+
})
|
|
4620
|
+
);
|
|
4621
|
+
if (!lockAcquired) {
|
|
4622
|
+
if (config.verbose) {
|
|
4623
|
+
console.log(`[EventualConsistency] Lock cleanup already running in another container`);
|
|
4414
4624
|
}
|
|
4625
|
+
return;
|
|
4415
4626
|
}
|
|
4416
|
-
|
|
4417
|
-
|
|
4418
|
-
|
|
4419
|
-
|
|
4420
|
-
|
|
4421
|
-
|
|
4422
|
-
|
|
4423
|
-
|
|
4424
|
-
|
|
4425
|
-
|
|
4426
|
-
|
|
4427
|
-
|
|
4428
|
-
|
|
4429
|
-
|
|
4430
|
-
|
|
4431
|
-
|
|
4432
|
-
|
|
4433
|
-
|
|
4434
|
-
|
|
4627
|
+
try {
|
|
4628
|
+
const [ok, err, locks] = await tryFn(() => lockResource.list());
|
|
4629
|
+
if (!ok || !locks || locks.length === 0) return;
|
|
4630
|
+
const staleLocks = locks.filter(
|
|
4631
|
+
(lock) => lock.id !== cleanupLockId && lock.lockedAt < cutoffTime
|
|
4632
|
+
);
|
|
4633
|
+
if (staleLocks.length === 0) return;
|
|
4634
|
+
if (config.verbose) {
|
|
4635
|
+
console.log(`[EventualConsistency] Cleaning up ${staleLocks.length} stale locks`);
|
|
4636
|
+
}
|
|
4637
|
+
const { results, errors } = await PromisePool.for(staleLocks).withConcurrency(5).process(async (lock) => {
|
|
4638
|
+
const [deleted] = await tryFn(() => lockResource.delete(lock.id));
|
|
4639
|
+
return deleted;
|
|
4640
|
+
});
|
|
4641
|
+
if (errors && errors.length > 0 && config.verbose) {
|
|
4642
|
+
console.warn(`[EventualConsistency] ${errors.length} stale locks failed to delete`);
|
|
4643
|
+
}
|
|
4644
|
+
} catch (error) {
|
|
4645
|
+
if (config.verbose) {
|
|
4646
|
+
console.warn(`[EventualConsistency] Error cleaning up stale locks:`, error.message);
|
|
4647
|
+
}
|
|
4648
|
+
} finally {
|
|
4649
|
+
await tryFn(() => lockResource.delete(cleanupLockId));
|
|
4435
4650
|
}
|
|
4436
|
-
|
|
4437
|
-
|
|
4438
|
-
|
|
4439
|
-
|
|
4440
|
-
|
|
4441
|
-
|
|
4442
|
-
|
|
4443
|
-
|
|
4444
|
-
|
|
4445
|
-
|
|
4446
|
-
|
|
4447
|
-
|
|
4448
|
-
|
|
4449
|
-
|
|
4450
|
-
|
|
4451
|
-
|
|
4452
|
-
|
|
4453
|
-
|
|
4454
|
-
|
|
4455
|
-
|
|
4456
|
-
timestamp: "string|required",
|
|
4457
|
-
cohortDate: "string|required",
|
|
4458
|
-
cohortHour: "string|required",
|
|
4459
|
-
cohortMonth: "string|optional",
|
|
4460
|
-
source: "string|optional",
|
|
4461
|
-
applied: "boolean|optional"
|
|
4462
|
-
},
|
|
4463
|
-
behavior: "body-overflow",
|
|
4464
|
-
timestamps: true,
|
|
4465
|
-
partitions: partitionConfig,
|
|
4466
|
-
asyncPartitions: true,
|
|
4467
|
-
createdBy: "EventualConsistencyPlugin"
|
|
4468
|
-
})
|
|
4651
|
+
}
|
|
4652
|
+
|
|
4653
|
+
function startConsolidationTimer(handler, resourceName, fieldName, runConsolidationCallback, config) {
|
|
4654
|
+
const intervalMs = config.consolidationInterval * 1e3;
|
|
4655
|
+
if (config.verbose) {
|
|
4656
|
+
const nextRun = new Date(Date.now() + intervalMs);
|
|
4657
|
+
console.log(
|
|
4658
|
+
`[EventualConsistency] ${resourceName}.${fieldName} - Consolidation timer started. Next run at ${nextRun.toISOString()} (every ${config.consolidationInterval}s)`
|
|
4659
|
+
);
|
|
4660
|
+
}
|
|
4661
|
+
handler.consolidationTimer = setInterval(async () => {
|
|
4662
|
+
await runConsolidationCallback(handler, resourceName, fieldName);
|
|
4663
|
+
}, intervalMs);
|
|
4664
|
+
return handler.consolidationTimer;
|
|
4665
|
+
}
|
|
4666
|
+
async function runConsolidation(transactionResource, consolidateRecordFn, emitFn, config) {
|
|
4667
|
+
const startTime = Date.now();
|
|
4668
|
+
if (config.verbose) {
|
|
4669
|
+
console.log(
|
|
4670
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Starting consolidation run at ${(/* @__PURE__ */ new Date()).toISOString()}`
|
|
4469
4671
|
);
|
|
4470
|
-
|
|
4471
|
-
|
|
4672
|
+
}
|
|
4673
|
+
try {
|
|
4674
|
+
const now = /* @__PURE__ */ new Date();
|
|
4675
|
+
const hoursToCheck = config.consolidationWindow || 24;
|
|
4676
|
+
const cohortHours = [];
|
|
4677
|
+
for (let i = 0; i < hoursToCheck; i++) {
|
|
4678
|
+
const date = new Date(now.getTime() - i * 60 * 60 * 1e3);
|
|
4679
|
+
const cohortInfo = getCohortInfo(date, config.cohort.timezone, config.verbose);
|
|
4680
|
+
cohortHours.push(cohortInfo.hour);
|
|
4472
4681
|
}
|
|
4473
|
-
|
|
4474
|
-
|
|
4475
|
-
|
|
4476
|
-
|
|
4477
|
-
|
|
4478
|
-
|
|
4479
|
-
|
|
4480
|
-
|
|
4481
|
-
|
|
4482
|
-
|
|
4483
|
-
|
|
4484
|
-
|
|
4485
|
-
|
|
4682
|
+
if (config.verbose) {
|
|
4683
|
+
console.log(
|
|
4684
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Querying ${hoursToCheck} hour partitions for pending transactions...`
|
|
4685
|
+
);
|
|
4686
|
+
}
|
|
4687
|
+
const transactionsByHour = await Promise.all(
|
|
4688
|
+
cohortHours.map(async (cohortHour) => {
|
|
4689
|
+
const [ok, err, txns] = await tryFn(
|
|
4690
|
+
() => transactionResource.query({
|
|
4691
|
+
cohortHour,
|
|
4692
|
+
applied: false
|
|
4693
|
+
})
|
|
4694
|
+
);
|
|
4695
|
+
return ok ? txns : [];
|
|
4486
4696
|
})
|
|
4487
4697
|
);
|
|
4488
|
-
|
|
4489
|
-
|
|
4698
|
+
const transactions = transactionsByHour.flat();
|
|
4699
|
+
if (transactions.length === 0) {
|
|
4700
|
+
if (config.verbose) {
|
|
4701
|
+
console.log(
|
|
4702
|
+
`[EventualConsistency] ${config.resource}.${config.field} - No pending transactions found. Next run in ${config.consolidationInterval}s`
|
|
4703
|
+
);
|
|
4704
|
+
}
|
|
4705
|
+
return;
|
|
4706
|
+
}
|
|
4707
|
+
const uniqueIds = [...new Set(transactions.map((t) => t.originalId))];
|
|
4708
|
+
if (config.verbose) {
|
|
4709
|
+
console.log(
|
|
4710
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Found ${transactions.length} pending transactions for ${uniqueIds.length} records. Consolidating with concurrency=${config.consolidationConcurrency}...`
|
|
4711
|
+
);
|
|
4490
4712
|
}
|
|
4491
|
-
|
|
4492
|
-
|
|
4493
|
-
|
|
4713
|
+
const { results, errors } = await PromisePool.for(uniqueIds).withConcurrency(config.consolidationConcurrency).process(async (id) => {
|
|
4714
|
+
return await consolidateRecordFn(id);
|
|
4715
|
+
});
|
|
4716
|
+
const duration = Date.now() - startTime;
|
|
4717
|
+
if (errors && errors.length > 0) {
|
|
4718
|
+
console.error(
|
|
4719
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Consolidation completed with ${errors.length} errors in ${duration}ms:`,
|
|
4720
|
+
errors
|
|
4721
|
+
);
|
|
4494
4722
|
}
|
|
4495
|
-
this._addHelperMethodsForHandler(handler);
|
|
4496
4723
|
if (config.verbose) {
|
|
4497
4724
|
console.log(
|
|
4498
|
-
`[EventualConsistency] ${
|
|
4725
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Consolidation complete: ${results.length} records consolidated in ${duration}ms (${errors.length} errors). Next run in ${config.consolidationInterval}s`
|
|
4499
4726
|
);
|
|
4500
4727
|
}
|
|
4501
|
-
|
|
4502
|
-
|
|
4503
|
-
|
|
4504
|
-
|
|
4505
|
-
|
|
4506
|
-
|
|
4507
|
-
|
|
4508
|
-
|
|
4509
|
-
|
|
4510
|
-
|
|
4511
|
-
|
|
4512
|
-
|
|
4513
|
-
|
|
4514
|
-
|
|
4515
|
-
|
|
4516
|
-
cohort: "string|required",
|
|
4517
|
-
transactionCount: "number|required",
|
|
4518
|
-
totalValue: "number|required",
|
|
4519
|
-
avgValue: "number|required",
|
|
4520
|
-
minValue: "number|required",
|
|
4521
|
-
maxValue: "number|required",
|
|
4522
|
-
operations: "object|optional",
|
|
4523
|
-
recordCount: "number|required",
|
|
4524
|
-
consolidatedAt: "string|required",
|
|
4525
|
-
updatedAt: "string|required"
|
|
4526
|
-
},
|
|
4527
|
-
behavior: "body-overflow",
|
|
4528
|
-
timestamps: false,
|
|
4529
|
-
createdBy: "EventualConsistencyPlugin"
|
|
4530
|
-
})
|
|
4728
|
+
if (emitFn) {
|
|
4729
|
+
emitFn("eventual-consistency.consolidated", {
|
|
4730
|
+
resource: config.resource,
|
|
4731
|
+
field: config.field,
|
|
4732
|
+
recordCount: uniqueIds.length,
|
|
4733
|
+
successCount: results.length,
|
|
4734
|
+
errorCount: errors.length,
|
|
4735
|
+
duration
|
|
4736
|
+
});
|
|
4737
|
+
}
|
|
4738
|
+
} catch (error) {
|
|
4739
|
+
const duration = Date.now() - startTime;
|
|
4740
|
+
console.error(
|
|
4741
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Consolidation error after ${duration}ms:`,
|
|
4742
|
+
error
|
|
4531
4743
|
);
|
|
4532
|
-
if (
|
|
4533
|
-
|
|
4744
|
+
if (emitFn) {
|
|
4745
|
+
emitFn("eventual-consistency.consolidation-error", error);
|
|
4534
4746
|
}
|
|
4535
|
-
handler.analyticsResource = ok ? analyticsResource : this.database.resources[analyticsResourceName];
|
|
4536
4747
|
}
|
|
4537
|
-
|
|
4538
|
-
|
|
4539
|
-
|
|
4540
|
-
|
|
4541
|
-
|
|
4542
|
-
|
|
4543
|
-
|
|
4544
|
-
|
|
4545
|
-
|
|
4546
|
-
}
|
|
4547
|
-
|
|
4548
|
-
|
|
4549
|
-
|
|
4748
|
+
}
|
|
4749
|
+
async function consolidateRecord(originalId, transactionResource, targetResource, lockResource, analyticsResource, updateAnalyticsFn, config) {
|
|
4750
|
+
await cleanupStaleLocks(lockResource, config);
|
|
4751
|
+
const lockId = `lock-${originalId}`;
|
|
4752
|
+
const [lockAcquired, lockErr, lock] = await tryFn(
|
|
4753
|
+
() => lockResource.insert({
|
|
4754
|
+
id: lockId,
|
|
4755
|
+
lockedAt: Date.now(),
|
|
4756
|
+
workerId: process.pid ? String(process.pid) : "unknown"
|
|
4757
|
+
})
|
|
4758
|
+
);
|
|
4759
|
+
if (!lockAcquired) {
|
|
4760
|
+
if (config.verbose) {
|
|
4761
|
+
console.log(`[EventualConsistency] Lock for ${originalId} already held, skipping`);
|
|
4550
4762
|
}
|
|
4763
|
+
const [recordOk, recordErr, record] = await tryFn(
|
|
4764
|
+
() => targetResource.get(originalId)
|
|
4765
|
+
);
|
|
4766
|
+
return recordOk && record ? record[config.field] || 0 : 0;
|
|
4551
4767
|
}
|
|
4552
|
-
|
|
4553
|
-
|
|
4554
|
-
|
|
4555
|
-
|
|
4556
|
-
|
|
4557
|
-
|
|
4558
|
-
|
|
4559
|
-
|
|
4560
|
-
|
|
4561
|
-
|
|
4562
|
-
|
|
4563
|
-
|
|
4564
|
-
|
|
4565
|
-
|
|
4566
|
-
}
|
|
4567
|
-
|
|
4768
|
+
try {
|
|
4769
|
+
const [ok, err, transactions] = await tryFn(
|
|
4770
|
+
() => transactionResource.query({
|
|
4771
|
+
originalId,
|
|
4772
|
+
applied: false
|
|
4773
|
+
})
|
|
4774
|
+
);
|
|
4775
|
+
if (!ok || !transactions || transactions.length === 0) {
|
|
4776
|
+
const [recordOk, recordErr, record] = await tryFn(
|
|
4777
|
+
() => targetResource.get(originalId)
|
|
4778
|
+
);
|
|
4779
|
+
const currentValue2 = recordOk && record ? record[config.field] || 0 : 0;
|
|
4780
|
+
if (config.verbose) {
|
|
4781
|
+
console.log(
|
|
4782
|
+
`[EventualConsistency] ${config.resource}.${config.field} - No pending transactions for ${originalId}, skipping`
|
|
4783
|
+
);
|
|
4568
4784
|
}
|
|
4785
|
+
return currentValue2;
|
|
4569
4786
|
}
|
|
4570
|
-
|
|
4571
|
-
|
|
4572
|
-
|
|
4573
|
-
|
|
4574
|
-
|
|
4575
|
-
|
|
4576
|
-
|
|
4787
|
+
const [appliedOk, appliedErr, appliedTransactions] = await tryFn(
|
|
4788
|
+
() => transactionResource.query({
|
|
4789
|
+
originalId,
|
|
4790
|
+
applied: true
|
|
4791
|
+
})
|
|
4792
|
+
);
|
|
4793
|
+
let currentValue = 0;
|
|
4794
|
+
if (appliedOk && appliedTransactions && appliedTransactions.length > 0) {
|
|
4795
|
+
const [recordExistsOk, recordExistsErr, recordExists] = await tryFn(
|
|
4796
|
+
() => targetResource.get(originalId)
|
|
4797
|
+
);
|
|
4798
|
+
if (!recordExistsOk || !recordExists) {
|
|
4799
|
+
if (config.verbose) {
|
|
4800
|
+
console.log(
|
|
4801
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Record ${originalId} doesn't exist, deleting ${appliedTransactions.length} old applied transactions`
|
|
4802
|
+
);
|
|
4577
4803
|
}
|
|
4578
|
-
|
|
4579
|
-
|
|
4580
|
-
|
|
4804
|
+
const { results, errors } = await PromisePool.for(appliedTransactions).withConcurrency(10).process(async (txn) => {
|
|
4805
|
+
const [deleted] = await tryFn(() => transactionResource.delete(txn.id));
|
|
4806
|
+
return deleted;
|
|
4807
|
+
});
|
|
4808
|
+
if (config.verbose && errors && errors.length > 0) {
|
|
4809
|
+
console.warn(
|
|
4810
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Failed to delete ${errors.length} old applied transactions`
|
|
4811
|
+
);
|
|
4581
4812
|
}
|
|
4582
|
-
|
|
4583
|
-
|
|
4813
|
+
currentValue = 0;
|
|
4814
|
+
} else {
|
|
4815
|
+
appliedTransactions.sort(
|
|
4816
|
+
(a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()
|
|
4817
|
+
);
|
|
4818
|
+
const hasSetInApplied = appliedTransactions.some((t) => t.operation === "set");
|
|
4819
|
+
if (!hasSetInApplied) {
|
|
4820
|
+
const recordValue = recordExists[config.field] || 0;
|
|
4821
|
+
let appliedDelta = 0;
|
|
4822
|
+
for (const t of appliedTransactions) {
|
|
4823
|
+
if (t.operation === "add") appliedDelta += t.value;
|
|
4824
|
+
else if (t.operation === "sub") appliedDelta -= t.value;
|
|
4825
|
+
}
|
|
4826
|
+
const baseValue = recordValue - appliedDelta;
|
|
4827
|
+
const hasExistingAnchor = appliedTransactions.some((t) => t.source === "anchor");
|
|
4828
|
+
if (baseValue !== 0 && !hasExistingAnchor) {
|
|
4829
|
+
const firstTransactionDate = new Date(appliedTransactions[0].timestamp);
|
|
4830
|
+
const cohortInfo = getCohortInfo(firstTransactionDate, config.cohort.timezone, config.verbose);
|
|
4831
|
+
const anchorTransaction = {
|
|
4832
|
+
id: idGenerator(),
|
|
4833
|
+
originalId,
|
|
4834
|
+
field: config.field,
|
|
4835
|
+
value: baseValue,
|
|
4836
|
+
operation: "set",
|
|
4837
|
+
timestamp: new Date(firstTransactionDate.getTime() - 1).toISOString(),
|
|
4838
|
+
// 1ms before first txn to ensure it's first
|
|
4839
|
+
cohortDate: cohortInfo.date,
|
|
4840
|
+
cohortHour: cohortInfo.hour,
|
|
4841
|
+
cohortMonth: cohortInfo.month,
|
|
4842
|
+
source: "anchor",
|
|
4843
|
+
applied: true
|
|
4844
|
+
};
|
|
4845
|
+
await transactionResource.insert(anchorTransaction);
|
|
4846
|
+
appliedTransactions.unshift(anchorTransaction);
|
|
4847
|
+
}
|
|
4584
4848
|
}
|
|
4585
|
-
|
|
4586
|
-
resource: resourceName,
|
|
4587
|
-
field: fieldName
|
|
4588
|
-
});
|
|
4849
|
+
currentValue = config.reducer(appliedTransactions);
|
|
4589
4850
|
}
|
|
4590
|
-
}
|
|
4591
|
-
|
|
4592
|
-
|
|
4593
|
-
|
|
4594
|
-
|
|
4595
|
-
|
|
4596
|
-
|
|
4597
|
-
|
|
4598
|
-
|
|
4599
|
-
|
|
4600
|
-
|
|
4601
|
-
|
|
4851
|
+
} else {
|
|
4852
|
+
const [recordOk, recordErr, record] = await tryFn(
|
|
4853
|
+
() => targetResource.get(originalId)
|
|
4854
|
+
);
|
|
4855
|
+
currentValue = recordOk && record ? record[config.field] || 0 : 0;
|
|
4856
|
+
if (currentValue !== 0) {
|
|
4857
|
+
let anchorTimestamp;
|
|
4858
|
+
if (transactions && transactions.length > 0) {
|
|
4859
|
+
const firstPendingDate = new Date(transactions[0].timestamp);
|
|
4860
|
+
anchorTimestamp = new Date(firstPendingDate.getTime() - 1).toISOString();
|
|
4861
|
+
} else {
|
|
4862
|
+
anchorTimestamp = (/* @__PURE__ */ new Date()).toISOString();
|
|
4602
4863
|
}
|
|
4603
|
-
|
|
4604
|
-
|
|
4605
|
-
|
|
4606
|
-
|
|
4864
|
+
const cohortInfo = getCohortInfo(new Date(anchorTimestamp), config.cohort.timezone, config.verbose);
|
|
4865
|
+
const anchorTransaction = {
|
|
4866
|
+
id: idGenerator(),
|
|
4867
|
+
originalId,
|
|
4868
|
+
field: config.field,
|
|
4869
|
+
value: currentValue,
|
|
4870
|
+
operation: "set",
|
|
4871
|
+
timestamp: anchorTimestamp,
|
|
4872
|
+
cohortDate: cohortInfo.date,
|
|
4873
|
+
cohortHour: cohortInfo.hour,
|
|
4874
|
+
cohortMonth: cohortInfo.month,
|
|
4875
|
+
source: "anchor",
|
|
4876
|
+
applied: true
|
|
4877
|
+
};
|
|
4878
|
+
await transactionResource.insert(anchorTransaction);
|
|
4879
|
+
if (config.verbose) {
|
|
4880
|
+
console.log(
|
|
4881
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Created anchor transaction for ${originalId} with base value ${currentValue}`
|
|
4882
|
+
);
|
|
4607
4883
|
}
|
|
4608
4884
|
}
|
|
4609
|
-
};
|
|
4610
|
-
return partitions;
|
|
4611
|
-
}
|
|
4612
|
-
/**
|
|
4613
|
-
* Auto-detect timezone from environment or system
|
|
4614
|
-
* @private
|
|
4615
|
-
*/
|
|
4616
|
-
_detectTimezone() {
|
|
4617
|
-
if (process.env.TZ) {
|
|
4618
|
-
return process.env.TZ;
|
|
4619
4885
|
}
|
|
4620
|
-
|
|
4621
|
-
|
|
4622
|
-
|
|
4623
|
-
|
|
4624
|
-
}
|
|
4625
|
-
} catch (err) {
|
|
4886
|
+
if (config.verbose) {
|
|
4887
|
+
console.log(
|
|
4888
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Consolidating ${originalId}: ${transactions.length} pending transactions (current: ${currentValue} from ${appliedOk && appliedTransactions?.length > 0 ? "applied transactions" : "record"})`
|
|
4889
|
+
);
|
|
4626
4890
|
}
|
|
4627
|
-
|
|
4628
|
-
|
|
4629
|
-
|
|
4630
|
-
|
|
4631
|
-
|
|
4632
|
-
|
|
4633
|
-
_resolveFieldAndPlugin(resource, field, value) {
|
|
4634
|
-
if (!resource._eventualConsistencyPlugins) {
|
|
4635
|
-
throw new Error(`No eventual consistency plugins configured for this resource`);
|
|
4891
|
+
transactions.sort(
|
|
4892
|
+
(a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()
|
|
4893
|
+
);
|
|
4894
|
+
const hasSetOperation = transactions.some((t) => t.operation === "set");
|
|
4895
|
+
if (currentValue !== 0 && !hasSetOperation) {
|
|
4896
|
+
transactions.unshift(createSyntheticSetTransaction(currentValue));
|
|
4636
4897
|
}
|
|
4637
|
-
const
|
|
4638
|
-
if (
|
|
4639
|
-
|
|
4640
|
-
|
|
4641
|
-
`No eventual consistency plugin found for field "${field}". Available fields: ${availableFields}`
|
|
4898
|
+
const consolidatedValue = config.reducer(transactions);
|
|
4899
|
+
if (config.verbose) {
|
|
4900
|
+
console.log(
|
|
4901
|
+
`[EventualConsistency] ${config.resource}.${config.field} - ${originalId}: ${currentValue} \u2192 ${consolidatedValue} (${consolidatedValue > currentValue ? "+" : ""}${consolidatedValue - currentValue})`
|
|
4642
4902
|
);
|
|
4643
4903
|
}
|
|
4644
|
-
|
|
4645
|
-
|
|
4646
|
-
|
|
4647
|
-
|
|
4648
|
-
|
|
4649
|
-
|
|
4650
|
-
|
|
4651
|
-
|
|
4652
|
-
return consolidatedValue;
|
|
4653
|
-
}
|
|
4654
|
-
/**
|
|
4655
|
-
* Create synthetic 'set' transaction from current value
|
|
4656
|
-
* @private
|
|
4657
|
-
*/
|
|
4658
|
-
_createSyntheticSetTransaction(currentValue) {
|
|
4659
|
-
return {
|
|
4660
|
-
id: "__synthetic__",
|
|
4661
|
-
operation: "set",
|
|
4662
|
-
value: currentValue,
|
|
4663
|
-
timestamp: (/* @__PURE__ */ new Date(0)).toISOString(),
|
|
4664
|
-
synthetic: true
|
|
4665
|
-
};
|
|
4666
|
-
}
|
|
4667
|
-
addHelperMethods() {
|
|
4668
|
-
const firstResource = this.fieldHandlers.values().next().value;
|
|
4669
|
-
if (!firstResource) return;
|
|
4670
|
-
const firstHandler = firstResource.values().next().value;
|
|
4671
|
-
if (!firstHandler || !firstHandler.targetResource) return;
|
|
4672
|
-
const resource = firstHandler.targetResource;
|
|
4673
|
-
const plugin = this;
|
|
4674
|
-
resource.set = async (id, field, value) => {
|
|
4675
|
-
const { plugin: handler } = plugin._resolveFieldAndPlugin(resource, field, value);
|
|
4676
|
-
const now = /* @__PURE__ */ new Date();
|
|
4677
|
-
const cohortInfo = plugin.getCohortInfo(now);
|
|
4678
|
-
const transaction = {
|
|
4679
|
-
id: idGenerator(),
|
|
4680
|
-
originalId: id,
|
|
4681
|
-
field: handler.field,
|
|
4682
|
-
value,
|
|
4683
|
-
operation: "set",
|
|
4684
|
-
timestamp: now.toISOString(),
|
|
4685
|
-
cohortDate: cohortInfo.date,
|
|
4686
|
-
cohortHour: cohortInfo.hour,
|
|
4687
|
-
cohortMonth: cohortInfo.month,
|
|
4688
|
-
source: "set",
|
|
4689
|
-
applied: false
|
|
4690
|
-
};
|
|
4691
|
-
await handler.transactionResource.insert(transaction);
|
|
4692
|
-
if (plugin.config.mode === "sync") {
|
|
4693
|
-
const oldResource = plugin.config.resource;
|
|
4694
|
-
const oldField = plugin.config.field;
|
|
4695
|
-
const oldTransactionResource = plugin.transactionResource;
|
|
4696
|
-
const oldTargetResource = plugin.targetResource;
|
|
4697
|
-
const oldLockResource = plugin.lockResource;
|
|
4698
|
-
const oldAnalyticsResource = plugin.analyticsResource;
|
|
4699
|
-
plugin.config.resource = handler.resource;
|
|
4700
|
-
plugin.config.field = handler.field;
|
|
4701
|
-
plugin.transactionResource = handler.transactionResource;
|
|
4702
|
-
plugin.targetResource = handler.targetResource;
|
|
4703
|
-
plugin.lockResource = handler.lockResource;
|
|
4704
|
-
plugin.analyticsResource = handler.analyticsResource;
|
|
4705
|
-
const result = await plugin._syncModeConsolidate(id, field);
|
|
4706
|
-
plugin.config.resource = oldResource;
|
|
4707
|
-
plugin.config.field = oldField;
|
|
4708
|
-
plugin.transactionResource = oldTransactionResource;
|
|
4709
|
-
plugin.targetResource = oldTargetResource;
|
|
4710
|
-
plugin.lockResource = oldLockResource;
|
|
4711
|
-
plugin.analyticsResource = oldAnalyticsResource;
|
|
4712
|
-
return result;
|
|
4713
|
-
}
|
|
4714
|
-
return value;
|
|
4715
|
-
};
|
|
4716
|
-
resource.add = async (id, field, amount) => {
|
|
4717
|
-
const { plugin: handler } = plugin._resolveFieldAndPlugin(resource, field, amount);
|
|
4718
|
-
const now = /* @__PURE__ */ new Date();
|
|
4719
|
-
const cohortInfo = plugin.getCohortInfo(now);
|
|
4720
|
-
const transaction = {
|
|
4721
|
-
id: idGenerator(),
|
|
4722
|
-
originalId: id,
|
|
4723
|
-
field: handler.field,
|
|
4724
|
-
value: amount,
|
|
4725
|
-
operation: "add",
|
|
4726
|
-
timestamp: now.toISOString(),
|
|
4727
|
-
cohortDate: cohortInfo.date,
|
|
4728
|
-
cohortHour: cohortInfo.hour,
|
|
4729
|
-
cohortMonth: cohortInfo.month,
|
|
4730
|
-
source: "add",
|
|
4731
|
-
applied: false
|
|
4732
|
-
};
|
|
4733
|
-
await handler.transactionResource.insert(transaction);
|
|
4734
|
-
if (plugin.config.mode === "sync") {
|
|
4735
|
-
const oldResource = plugin.config.resource;
|
|
4736
|
-
const oldField = plugin.config.field;
|
|
4737
|
-
const oldTransactionResource = plugin.transactionResource;
|
|
4738
|
-
const oldTargetResource = plugin.targetResource;
|
|
4739
|
-
const oldLockResource = plugin.lockResource;
|
|
4740
|
-
const oldAnalyticsResource = plugin.analyticsResource;
|
|
4741
|
-
plugin.config.resource = handler.resource;
|
|
4742
|
-
plugin.config.field = handler.field;
|
|
4743
|
-
plugin.transactionResource = handler.transactionResource;
|
|
4744
|
-
plugin.targetResource = handler.targetResource;
|
|
4745
|
-
plugin.lockResource = handler.lockResource;
|
|
4746
|
-
plugin.analyticsResource = handler.analyticsResource;
|
|
4747
|
-
const result = await plugin._syncModeConsolidate(id, field);
|
|
4748
|
-
plugin.config.resource = oldResource;
|
|
4749
|
-
plugin.config.field = oldField;
|
|
4750
|
-
plugin.transactionResource = oldTransactionResource;
|
|
4751
|
-
plugin.targetResource = oldTargetResource;
|
|
4752
|
-
plugin.lockResource = oldLockResource;
|
|
4753
|
-
plugin.analyticsResource = oldAnalyticsResource;
|
|
4754
|
-
return result;
|
|
4755
|
-
}
|
|
4756
|
-
const [ok, err, record] = await tryFn(() => handler.targetResource.get(id));
|
|
4757
|
-
const currentValue = ok && record ? record[field] || 0 : 0;
|
|
4758
|
-
return currentValue + amount;
|
|
4759
|
-
};
|
|
4760
|
-
resource.sub = async (id, field, amount) => {
|
|
4761
|
-
const { plugin: handler } = plugin._resolveFieldAndPlugin(resource, field, amount);
|
|
4762
|
-
const now = /* @__PURE__ */ new Date();
|
|
4763
|
-
const cohortInfo = plugin.getCohortInfo(now);
|
|
4764
|
-
const transaction = {
|
|
4765
|
-
id: idGenerator(),
|
|
4766
|
-
originalId: id,
|
|
4767
|
-
field: handler.field,
|
|
4768
|
-
value: amount,
|
|
4769
|
-
operation: "sub",
|
|
4770
|
-
timestamp: now.toISOString(),
|
|
4771
|
-
cohortDate: cohortInfo.date,
|
|
4772
|
-
cohortHour: cohortInfo.hour,
|
|
4773
|
-
cohortMonth: cohortInfo.month,
|
|
4774
|
-
source: "sub",
|
|
4775
|
-
applied: false
|
|
4776
|
-
};
|
|
4777
|
-
await handler.transactionResource.insert(transaction);
|
|
4778
|
-
if (plugin.config.mode === "sync") {
|
|
4779
|
-
const oldResource = plugin.config.resource;
|
|
4780
|
-
const oldField = plugin.config.field;
|
|
4781
|
-
const oldTransactionResource = plugin.transactionResource;
|
|
4782
|
-
const oldTargetResource = plugin.targetResource;
|
|
4783
|
-
const oldLockResource = plugin.lockResource;
|
|
4784
|
-
const oldAnalyticsResource = plugin.analyticsResource;
|
|
4785
|
-
plugin.config.resource = handler.resource;
|
|
4786
|
-
plugin.config.field = handler.field;
|
|
4787
|
-
plugin.transactionResource = handler.transactionResource;
|
|
4788
|
-
plugin.targetResource = handler.targetResource;
|
|
4789
|
-
plugin.lockResource = handler.lockResource;
|
|
4790
|
-
plugin.analyticsResource = handler.analyticsResource;
|
|
4791
|
-
const result = await plugin._syncModeConsolidate(id, field);
|
|
4792
|
-
plugin.config.resource = oldResource;
|
|
4793
|
-
plugin.config.field = oldField;
|
|
4794
|
-
plugin.transactionResource = oldTransactionResource;
|
|
4795
|
-
plugin.targetResource = oldTargetResource;
|
|
4796
|
-
plugin.lockResource = oldLockResource;
|
|
4797
|
-
plugin.analyticsResource = oldAnalyticsResource;
|
|
4798
|
-
return result;
|
|
4799
|
-
}
|
|
4800
|
-
const [ok, err, record] = await tryFn(() => handler.targetResource.get(id));
|
|
4801
|
-
const currentValue = ok && record ? record[field] || 0 : 0;
|
|
4802
|
-
return currentValue - amount;
|
|
4803
|
-
};
|
|
4804
|
-
resource.consolidate = async (id, field) => {
|
|
4805
|
-
if (!field) {
|
|
4806
|
-
throw new Error(`Field parameter is required: consolidate(id, field)`);
|
|
4807
|
-
}
|
|
4808
|
-
const handler = resource._eventualConsistencyPlugins[field];
|
|
4809
|
-
if (!handler) {
|
|
4810
|
-
const availableFields = Object.keys(resource._eventualConsistencyPlugins).join(", ");
|
|
4811
|
-
throw new Error(
|
|
4812
|
-
`No eventual consistency plugin found for field "${field}". Available fields: ${availableFields}`
|
|
4813
|
-
);
|
|
4814
|
-
}
|
|
4815
|
-
const oldResource = plugin.config.resource;
|
|
4816
|
-
const oldField = plugin.config.field;
|
|
4817
|
-
const oldTransactionResource = plugin.transactionResource;
|
|
4818
|
-
const oldTargetResource = plugin.targetResource;
|
|
4819
|
-
const oldLockResource = plugin.lockResource;
|
|
4820
|
-
const oldAnalyticsResource = plugin.analyticsResource;
|
|
4821
|
-
plugin.config.resource = handler.resource;
|
|
4822
|
-
plugin.config.field = handler.field;
|
|
4823
|
-
plugin.transactionResource = handler.transactionResource;
|
|
4824
|
-
plugin.targetResource = handler.targetResource;
|
|
4825
|
-
plugin.lockResource = handler.lockResource;
|
|
4826
|
-
plugin.analyticsResource = handler.analyticsResource;
|
|
4827
|
-
const result = await plugin.consolidateRecord(id);
|
|
4828
|
-
plugin.config.resource = oldResource;
|
|
4829
|
-
plugin.config.field = oldField;
|
|
4830
|
-
plugin.transactionResource = oldTransactionResource;
|
|
4831
|
-
plugin.targetResource = oldTargetResource;
|
|
4832
|
-
plugin.lockResource = oldLockResource;
|
|
4833
|
-
plugin.analyticsResource = oldAnalyticsResource;
|
|
4834
|
-
return result;
|
|
4835
|
-
};
|
|
4836
|
-
resource.getConsolidatedValue = async (id, field, options = {}) => {
|
|
4837
|
-
const handler = resource._eventualConsistencyPlugins[field];
|
|
4838
|
-
if (!handler) {
|
|
4839
|
-
const availableFields = Object.keys(resource._eventualConsistencyPlugins).join(", ");
|
|
4840
|
-
throw new Error(
|
|
4841
|
-
`No eventual consistency plugin found for field "${field}". Available fields: ${availableFields}`
|
|
4842
|
-
);
|
|
4843
|
-
}
|
|
4844
|
-
const oldResource = plugin.config.resource;
|
|
4845
|
-
const oldField = plugin.config.field;
|
|
4846
|
-
const oldTransactionResource = plugin.transactionResource;
|
|
4847
|
-
const oldTargetResource = plugin.targetResource;
|
|
4848
|
-
plugin.config.resource = handler.resource;
|
|
4849
|
-
plugin.config.field = handler.field;
|
|
4850
|
-
plugin.transactionResource = handler.transactionResource;
|
|
4851
|
-
plugin.targetResource = handler.targetResource;
|
|
4852
|
-
const result = await plugin.getConsolidatedValue(id, options);
|
|
4853
|
-
plugin.config.resource = oldResource;
|
|
4854
|
-
plugin.config.field = oldField;
|
|
4855
|
-
plugin.transactionResource = oldTransactionResource;
|
|
4856
|
-
plugin.targetResource = oldTargetResource;
|
|
4857
|
-
return result;
|
|
4858
|
-
};
|
|
4859
|
-
}
|
|
4860
|
-
async createTransaction(handler, data) {
|
|
4861
|
-
const now = /* @__PURE__ */ new Date();
|
|
4862
|
-
const cohortInfo = this.getCohortInfo(now);
|
|
4863
|
-
const watermarkMs = this.config.consolidationWindow * 60 * 60 * 1e3;
|
|
4864
|
-
const watermarkTime = now.getTime() - watermarkMs;
|
|
4865
|
-
const cohortHourDate = /* @__PURE__ */ new Date(cohortInfo.hour + ":00:00Z");
|
|
4866
|
-
if (cohortHourDate.getTime() < watermarkTime) {
|
|
4867
|
-
const hoursLate = Math.floor((now.getTime() - cohortHourDate.getTime()) / (60 * 60 * 1e3));
|
|
4868
|
-
if (this.config.lateArrivalStrategy === "ignore") {
|
|
4869
|
-
if (this.config.verbose) {
|
|
4904
|
+
const [updateOk, updateErr] = await tryFn(
|
|
4905
|
+
() => targetResource.update(originalId, {
|
|
4906
|
+
[config.field]: consolidatedValue
|
|
4907
|
+
})
|
|
4908
|
+
);
|
|
4909
|
+
if (!updateOk) {
|
|
4910
|
+
if (updateErr?.message?.includes("does not exist")) {
|
|
4911
|
+
if (config.verbose) {
|
|
4870
4912
|
console.warn(
|
|
4871
|
-
`[EventualConsistency]
|
|
4913
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Record ${originalId} doesn't exist. Skipping consolidation. ${transactions.length} transactions will remain pending until record is created.`
|
|
4872
4914
|
);
|
|
4873
4915
|
}
|
|
4874
|
-
return
|
|
4875
|
-
} else if (this.config.lateArrivalStrategy === "warn") {
|
|
4876
|
-
console.warn(
|
|
4877
|
-
`[EventualConsistency] Late arrival detected: transaction for ${cohortInfo.hour} is ${hoursLate}h late (watermark: ${this.config.consolidationWindow}h). Processing anyway, but consolidation may not pick it up.`
|
|
4878
|
-
);
|
|
4916
|
+
return consolidatedValue;
|
|
4879
4917
|
}
|
|
4918
|
+
console.error(
|
|
4919
|
+
`[EventualConsistency] ${config.resource}.${config.field} - FAILED to update ${originalId}: ${updateErr?.message || updateErr}`,
|
|
4920
|
+
{ error: updateErr, consolidatedValue, currentValue }
|
|
4921
|
+
);
|
|
4922
|
+
throw updateErr;
|
|
4880
4923
|
}
|
|
4881
|
-
|
|
4882
|
-
|
|
4883
|
-
|
|
4884
|
-
|
|
4885
|
-
|
|
4886
|
-
operation: data.operation || "set",
|
|
4887
|
-
timestamp: now.toISOString(),
|
|
4888
|
-
cohortDate: cohortInfo.date,
|
|
4889
|
-
cohortHour: cohortInfo.hour,
|
|
4890
|
-
cohortMonth: cohortInfo.month,
|
|
4891
|
-
source: data.source || "unknown",
|
|
4892
|
-
applied: false
|
|
4893
|
-
};
|
|
4894
|
-
if (this.config.batchTransactions) {
|
|
4895
|
-
handler.pendingTransactions.set(transaction.id, transaction);
|
|
4896
|
-
if (this.config.verbose) {
|
|
4897
|
-
console.log(
|
|
4898
|
-
`[EventualConsistency] ${handler.resource}.${handler.field} - Transaction batched: ${data.operation} ${data.value} for ${data.originalId} (batch: ${handler.pendingTransactions.size}/${this.config.batchSize})`
|
|
4924
|
+
if (updateOk) {
|
|
4925
|
+
const transactionsToUpdate = transactions.filter((txn) => txn.id !== "__synthetic__");
|
|
4926
|
+
const { results, errors } = await PromisePool.for(transactionsToUpdate).withConcurrency(10).process(async (txn) => {
|
|
4927
|
+
const [ok2, err2] = await tryFn(
|
|
4928
|
+
() => transactionResource.update(txn.id, { applied: true })
|
|
4899
4929
|
);
|
|
4930
|
+
if (!ok2 && config.verbose) {
|
|
4931
|
+
console.warn(`[EventualConsistency] Failed to mark transaction ${txn.id} as applied:`, err2?.message);
|
|
4932
|
+
}
|
|
4933
|
+
return ok2;
|
|
4934
|
+
});
|
|
4935
|
+
if (errors && errors.length > 0 && config.verbose) {
|
|
4936
|
+
console.warn(`[EventualConsistency] ${errors.length} transactions failed to mark as applied`);
|
|
4900
4937
|
}
|
|
4901
|
-
if (
|
|
4902
|
-
await
|
|
4938
|
+
if (config.enableAnalytics && transactionsToUpdate.length > 0 && updateAnalyticsFn) {
|
|
4939
|
+
await updateAnalyticsFn(transactionsToUpdate);
|
|
4903
4940
|
}
|
|
4904
|
-
|
|
4905
|
-
|
|
4906
|
-
|
|
4907
|
-
|
|
4908
|
-
|
|
4909
|
-
|
|
4941
|
+
if (targetResource && targetResource.cache && typeof targetResource.cache.delete === "function") {
|
|
4942
|
+
try {
|
|
4943
|
+
const cacheKey = await targetResource.cacheKeyFor({ id: originalId });
|
|
4944
|
+
await targetResource.cache.delete(cacheKey);
|
|
4945
|
+
if (config.verbose) {
|
|
4946
|
+
console.log(
|
|
4947
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Cache invalidated for ${originalId}`
|
|
4948
|
+
);
|
|
4949
|
+
}
|
|
4950
|
+
} catch (cacheErr) {
|
|
4951
|
+
if (config.verbose) {
|
|
4952
|
+
console.warn(
|
|
4953
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Failed to invalidate cache for ${originalId}: ${cacheErr?.message}`
|
|
4954
|
+
);
|
|
4955
|
+
}
|
|
4956
|
+
}
|
|
4910
4957
|
}
|
|
4911
4958
|
}
|
|
4912
|
-
return
|
|
4959
|
+
return consolidatedValue;
|
|
4960
|
+
} finally {
|
|
4961
|
+
const [lockReleased, lockReleaseErr] = await tryFn(() => lockResource.delete(lockId));
|
|
4962
|
+
if (!lockReleased && config.verbose) {
|
|
4963
|
+
console.warn(`[EventualConsistency] Failed to release lock ${lockId}:`, lockReleaseErr?.message);
|
|
4964
|
+
}
|
|
4913
4965
|
}
|
|
4914
|
-
|
|
4915
|
-
|
|
4916
|
-
|
|
4917
|
-
|
|
4918
|
-
|
|
4919
|
-
|
|
4920
|
-
|
|
4921
|
-
|
|
4922
|
-
|
|
4923
|
-
|
|
4924
|
-
|
|
4925
|
-
|
|
4926
|
-
|
|
4966
|
+
}
|
|
4967
|
+
async function getConsolidatedValue(originalId, options, transactionResource, targetResource, config) {
|
|
4968
|
+
const includeApplied = options.includeApplied || false;
|
|
4969
|
+
const startDate = options.startDate;
|
|
4970
|
+
const endDate = options.endDate;
|
|
4971
|
+
const query = { originalId };
|
|
4972
|
+
if (!includeApplied) {
|
|
4973
|
+
query.applied = false;
|
|
4974
|
+
}
|
|
4975
|
+
const [ok, err, transactions] = await tryFn(
|
|
4976
|
+
() => transactionResource.query(query)
|
|
4977
|
+
);
|
|
4978
|
+
if (!ok || !transactions || transactions.length === 0) {
|
|
4979
|
+
const [recordOk2, recordErr2, record2] = await tryFn(
|
|
4980
|
+
() => targetResource.get(originalId)
|
|
4981
|
+
);
|
|
4982
|
+
if (recordOk2 && record2) {
|
|
4983
|
+
return record2[config.field] || 0;
|
|
4927
4984
|
}
|
|
4985
|
+
return 0;
|
|
4928
4986
|
}
|
|
4929
|
-
|
|
4930
|
-
|
|
4931
|
-
|
|
4932
|
-
|
|
4933
|
-
|
|
4934
|
-
|
|
4935
|
-
|
|
4936
|
-
|
|
4937
|
-
return {
|
|
4938
|
-
date: `${year}-${month}-${day}`,
|
|
4939
|
-
hour: `${year}-${month}-${day}T${hour}`,
|
|
4940
|
-
// ISO-like format for hour partition
|
|
4941
|
-
month: `${year}-${month}`
|
|
4942
|
-
};
|
|
4987
|
+
let filtered = transactions;
|
|
4988
|
+
if (startDate || endDate) {
|
|
4989
|
+
filtered = transactions.filter((t) => {
|
|
4990
|
+
const timestamp = new Date(t.timestamp);
|
|
4991
|
+
if (startDate && timestamp < new Date(startDate)) return false;
|
|
4992
|
+
if (endDate && timestamp > new Date(endDate)) return false;
|
|
4993
|
+
return true;
|
|
4994
|
+
});
|
|
4943
4995
|
}
|
|
4944
|
-
|
|
4945
|
-
|
|
4946
|
-
|
|
4947
|
-
|
|
4948
|
-
|
|
4949
|
-
|
|
4950
|
-
|
|
4951
|
-
|
|
4952
|
-
|
|
4953
|
-
|
|
4954
|
-
|
|
4955
|
-
|
|
4956
|
-
|
|
4957
|
-
|
|
4958
|
-
|
|
4959
|
-
|
|
4960
|
-
|
|
4961
|
-
|
|
4962
|
-
|
|
4963
|
-
|
|
4996
|
+
const [recordOk, recordErr, record] = await tryFn(
|
|
4997
|
+
() => targetResource.get(originalId)
|
|
4998
|
+
);
|
|
4999
|
+
const currentValue = recordOk && record ? record[config.field] || 0 : 0;
|
|
5000
|
+
const hasSetOperation = filtered.some((t) => t.operation === "set");
|
|
5001
|
+
if (currentValue !== 0 && !hasSetOperation) {
|
|
5002
|
+
filtered.unshift(createSyntheticSetTransaction(currentValue));
|
|
5003
|
+
}
|
|
5004
|
+
filtered.sort(
|
|
5005
|
+
(a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()
|
|
5006
|
+
);
|
|
5007
|
+
return config.reducer(filtered);
|
|
5008
|
+
}
|
|
5009
|
+
async function getCohortStats(cohortDate, transactionResource) {
|
|
5010
|
+
const [ok, err, transactions] = await tryFn(
|
|
5011
|
+
() => transactionResource.query({
|
|
5012
|
+
cohortDate
|
|
5013
|
+
})
|
|
5014
|
+
);
|
|
5015
|
+
if (!ok) return null;
|
|
5016
|
+
const stats = {
|
|
5017
|
+
date: cohortDate,
|
|
5018
|
+
transactionCount: transactions.length,
|
|
5019
|
+
totalValue: 0,
|
|
5020
|
+
byOperation: { set: 0, add: 0, sub: 0 },
|
|
5021
|
+
byOriginalId: {}
|
|
5022
|
+
};
|
|
5023
|
+
for (const txn of transactions) {
|
|
5024
|
+
stats.totalValue += txn.value || 0;
|
|
5025
|
+
stats.byOperation[txn.operation] = (stats.byOperation[txn.operation] || 0) + 1;
|
|
5026
|
+
if (!stats.byOriginalId[txn.originalId]) {
|
|
5027
|
+
stats.byOriginalId[txn.originalId] = {
|
|
5028
|
+
count: 0,
|
|
5029
|
+
value: 0
|
|
4964
5030
|
};
|
|
4965
|
-
|
|
4966
|
-
|
|
4967
|
-
|
|
5031
|
+
}
|
|
5032
|
+
stats.byOriginalId[txn.originalId].count++;
|
|
5033
|
+
stats.byOriginalId[txn.originalId].value += txn.value || 0;
|
|
5034
|
+
}
|
|
5035
|
+
return stats;
|
|
5036
|
+
}
|
|
5037
|
+
async function recalculateRecord(originalId, transactionResource, targetResource, lockResource, consolidateRecordFn, config) {
|
|
5038
|
+
await cleanupStaleLocks(lockResource, config);
|
|
5039
|
+
const lockId = `lock-recalculate-${originalId}`;
|
|
5040
|
+
const [lockAcquired, lockErr, lock] = await tryFn(
|
|
5041
|
+
() => lockResource.insert({
|
|
5042
|
+
id: lockId,
|
|
5043
|
+
lockedAt: Date.now(),
|
|
5044
|
+
workerId: process.pid ? String(process.pid) : "unknown"
|
|
5045
|
+
})
|
|
5046
|
+
);
|
|
5047
|
+
if (!lockAcquired) {
|
|
5048
|
+
if (config.verbose) {
|
|
5049
|
+
console.log(`[EventualConsistency] Recalculate lock for ${originalId} already held, skipping`);
|
|
5050
|
+
}
|
|
5051
|
+
throw new Error(`Cannot recalculate ${originalId}: lock already held by another worker`);
|
|
5052
|
+
}
|
|
5053
|
+
try {
|
|
5054
|
+
if (config.verbose) {
|
|
5055
|
+
console.log(
|
|
5056
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Starting recalculation for ${originalId} (resetting all transactions to pending)`
|
|
5057
|
+
);
|
|
5058
|
+
}
|
|
5059
|
+
const [allOk, allErr, allTransactions] = await tryFn(
|
|
5060
|
+
() => transactionResource.query({
|
|
5061
|
+
originalId
|
|
5062
|
+
})
|
|
5063
|
+
);
|
|
5064
|
+
if (!allOk || !allTransactions || allTransactions.length === 0) {
|
|
5065
|
+
if (config.verbose) {
|
|
5066
|
+
console.log(
|
|
5067
|
+
`[EventualConsistency] ${config.resource}.${config.field} - No transactions found for ${originalId}, nothing to recalculate`
|
|
4968
5068
|
);
|
|
4969
5069
|
}
|
|
4970
|
-
return
|
|
5070
|
+
return 0;
|
|
4971
5071
|
}
|
|
4972
|
-
|
|
4973
|
-
startConsolidationTimer() {
|
|
4974
|
-
const intervalMs = this.config.consolidationInterval * 1e3;
|
|
4975
|
-
if (this.config.verbose) {
|
|
4976
|
-
const nextRun = new Date(Date.now() + intervalMs);
|
|
5072
|
+
if (config.verbose) {
|
|
4977
5073
|
console.log(
|
|
4978
|
-
`[EventualConsistency] ${
|
|
5074
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Found ${allTransactions.length} total transactions for ${originalId}, marking all as pending...`
|
|
4979
5075
|
);
|
|
4980
5076
|
}
|
|
4981
|
-
|
|
4982
|
-
|
|
4983
|
-
|
|
4984
|
-
|
|
4985
|
-
|
|
4986
|
-
|
|
4987
|
-
|
|
4988
|
-
|
|
5077
|
+
const transactionsToReset = allTransactions.filter((txn) => txn.source !== "anchor");
|
|
5078
|
+
const { results, errors } = await PromisePool.for(transactionsToReset).withConcurrency(10).process(async (txn) => {
|
|
5079
|
+
const [ok, err] = await tryFn(
|
|
5080
|
+
() => transactionResource.update(txn.id, { applied: false })
|
|
5081
|
+
);
|
|
5082
|
+
if (!ok && config.verbose) {
|
|
5083
|
+
console.warn(`[EventualConsistency] Failed to reset transaction ${txn.id}:`, err?.message);
|
|
5084
|
+
}
|
|
5085
|
+
return ok;
|
|
5086
|
+
});
|
|
5087
|
+
if (errors && errors.length > 0) {
|
|
5088
|
+
console.warn(
|
|
5089
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Failed to reset ${errors.length} transactions during recalculation`
|
|
5090
|
+
);
|
|
5091
|
+
}
|
|
5092
|
+
if (config.verbose) {
|
|
4989
5093
|
console.log(
|
|
4990
|
-
`[EventualConsistency] ${
|
|
5094
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Reset ${results.length} transactions to pending, now resetting record value and running consolidation...`
|
|
4991
5095
|
);
|
|
4992
5096
|
}
|
|
4993
|
-
|
|
4994
|
-
|
|
4995
|
-
|
|
4996
|
-
|
|
4997
|
-
|
|
4998
|
-
|
|
4999
|
-
|
|
5000
|
-
|
|
5001
|
-
|
|
5002
|
-
const oldLockResource = this.lockResource;
|
|
5003
|
-
const oldAnalyticsResource = this.analyticsResource;
|
|
5004
|
-
this.config.resource = resourceName;
|
|
5005
|
-
this.config.field = fieldName;
|
|
5006
|
-
this.transactionResource = handler.transactionResource;
|
|
5007
|
-
this.targetResource = handler.targetResource;
|
|
5008
|
-
this.lockResource = handler.lockResource;
|
|
5009
|
-
this.analyticsResource = handler.analyticsResource;
|
|
5010
|
-
try {
|
|
5011
|
-
await this.runConsolidation();
|
|
5012
|
-
} finally {
|
|
5013
|
-
this.config.resource = oldResource;
|
|
5014
|
-
this.config.field = oldField;
|
|
5015
|
-
this.transactionResource = oldTransactionResource;
|
|
5016
|
-
this.targetResource = oldTargetResource;
|
|
5017
|
-
this.lockResource = oldLockResource;
|
|
5018
|
-
this.analyticsResource = oldAnalyticsResource;
|
|
5097
|
+
const [resetOk, resetErr] = await tryFn(
|
|
5098
|
+
() => targetResource.update(originalId, {
|
|
5099
|
+
[config.field]: 0
|
|
5100
|
+
})
|
|
5101
|
+
);
|
|
5102
|
+
if (!resetOk && config.verbose) {
|
|
5103
|
+
console.warn(
|
|
5104
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Failed to reset record value for ${originalId}: ${resetErr?.message}`
|
|
5105
|
+
);
|
|
5019
5106
|
}
|
|
5020
|
-
|
|
5021
|
-
|
|
5022
|
-
const startTime = Date.now();
|
|
5023
|
-
if (this.config.verbose) {
|
|
5107
|
+
const consolidatedValue = await consolidateRecordFn(originalId);
|
|
5108
|
+
if (config.verbose) {
|
|
5024
5109
|
console.log(
|
|
5025
|
-
`[EventualConsistency] ${
|
|
5110
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Recalculation complete for ${originalId}: final value = ${consolidatedValue}`
|
|
5026
5111
|
);
|
|
5027
5112
|
}
|
|
5028
|
-
|
|
5029
|
-
|
|
5030
|
-
|
|
5031
|
-
|
|
5032
|
-
|
|
5033
|
-
|
|
5034
|
-
|
|
5035
|
-
|
|
5113
|
+
return consolidatedValue;
|
|
5114
|
+
} finally {
|
|
5115
|
+
const [lockReleased, lockReleaseErr] = await tryFn(() => lockResource.delete(lockId));
|
|
5116
|
+
if (!lockReleased && config.verbose) {
|
|
5117
|
+
console.warn(`[EventualConsistency] Failed to release recalculate lock ${lockId}:`, lockReleaseErr?.message);
|
|
5118
|
+
}
|
|
5119
|
+
}
|
|
5120
|
+
}
|
|
5121
|
+
|
|
5122
|
+
function startGarbageCollectionTimer(handler, resourceName, fieldName, runGCCallback, config) {
|
|
5123
|
+
const gcIntervalMs = config.gcInterval * 1e3;
|
|
5124
|
+
handler.gcTimer = setInterval(async () => {
|
|
5125
|
+
await runGCCallback(handler, resourceName, fieldName);
|
|
5126
|
+
}, gcIntervalMs);
|
|
5127
|
+
return handler.gcTimer;
|
|
5128
|
+
}
|
|
5129
|
+
async function runGarbageCollection(transactionResource, lockResource, config, emitFn) {
|
|
5130
|
+
const gcLockId = `lock-gc-${config.resource}-${config.field}`;
|
|
5131
|
+
const [lockAcquired] = await tryFn(
|
|
5132
|
+
() => lockResource.insert({
|
|
5133
|
+
id: gcLockId,
|
|
5134
|
+
lockedAt: Date.now(),
|
|
5135
|
+
workerId: process.pid ? String(process.pid) : "unknown"
|
|
5136
|
+
})
|
|
5137
|
+
);
|
|
5138
|
+
if (!lockAcquired) {
|
|
5139
|
+
if (config.verbose) {
|
|
5140
|
+
console.log(`[EventualConsistency] GC already running in another container`);
|
|
5141
|
+
}
|
|
5142
|
+
return;
|
|
5143
|
+
}
|
|
5144
|
+
try {
|
|
5145
|
+
const now = Date.now();
|
|
5146
|
+
const retentionMs = config.transactionRetention * 24 * 60 * 60 * 1e3;
|
|
5147
|
+
const cutoffDate = new Date(now - retentionMs);
|
|
5148
|
+
const cutoffIso = cutoffDate.toISOString();
|
|
5149
|
+
if (config.verbose) {
|
|
5150
|
+
console.log(`[EventualConsistency] Running GC for transactions older than ${cutoffIso} (${config.transactionRetention} days)`);
|
|
5151
|
+
}
|
|
5152
|
+
const [ok, err, oldTransactions] = await tryFn(
|
|
5153
|
+
() => transactionResource.query({
|
|
5154
|
+
applied: true,
|
|
5155
|
+
timestamp: { "<": cutoffIso }
|
|
5156
|
+
})
|
|
5157
|
+
);
|
|
5158
|
+
if (!ok) {
|
|
5159
|
+
if (config.verbose) {
|
|
5160
|
+
console.warn(`[EventualConsistency] GC failed to query transactions:`, err?.message);
|
|
5036
5161
|
}
|
|
5037
|
-
|
|
5038
|
-
|
|
5039
|
-
|
|
5040
|
-
|
|
5162
|
+
return;
|
|
5163
|
+
}
|
|
5164
|
+
if (!oldTransactions || oldTransactions.length === 0) {
|
|
5165
|
+
if (config.verbose) {
|
|
5166
|
+
console.log(`[EventualConsistency] No old transactions to clean up`);
|
|
5041
5167
|
}
|
|
5042
|
-
|
|
5043
|
-
|
|
5044
|
-
|
|
5045
|
-
|
|
5046
|
-
|
|
5047
|
-
|
|
5048
|
-
|
|
5049
|
-
|
|
5050
|
-
|
|
5051
|
-
|
|
5168
|
+
return;
|
|
5169
|
+
}
|
|
5170
|
+
if (config.verbose) {
|
|
5171
|
+
console.log(`[EventualConsistency] Deleting ${oldTransactions.length} old transactions`);
|
|
5172
|
+
}
|
|
5173
|
+
const { results, errors } = await PromisePool.for(oldTransactions).withConcurrency(10).process(async (txn) => {
|
|
5174
|
+
const [deleted] = await tryFn(() => transactionResource.delete(txn.id));
|
|
5175
|
+
return deleted;
|
|
5176
|
+
});
|
|
5177
|
+
if (config.verbose) {
|
|
5178
|
+
console.log(`[EventualConsistency] GC completed: ${results.length} deleted, ${errors.length} errors`);
|
|
5179
|
+
}
|
|
5180
|
+
if (emitFn) {
|
|
5181
|
+
emitFn("eventual-consistency.gc-completed", {
|
|
5182
|
+
resource: config.resource,
|
|
5183
|
+
field: config.field,
|
|
5184
|
+
deletedCount: results.length,
|
|
5185
|
+
errorCount: errors.length
|
|
5186
|
+
});
|
|
5187
|
+
}
|
|
5188
|
+
} catch (error) {
|
|
5189
|
+
if (config.verbose) {
|
|
5190
|
+
console.warn(`[EventualConsistency] GC error:`, error.message);
|
|
5191
|
+
}
|
|
5192
|
+
if (emitFn) {
|
|
5193
|
+
emitFn("eventual-consistency.gc-error", error);
|
|
5194
|
+
}
|
|
5195
|
+
} finally {
|
|
5196
|
+
await tryFn(() => lockResource.delete(gcLockId));
|
|
5197
|
+
}
|
|
5198
|
+
}
|
|
5199
|
+
|
|
5200
|
+
async function updateAnalytics(transactions, analyticsResource, config) {
|
|
5201
|
+
if (!analyticsResource || transactions.length === 0) return;
|
|
5202
|
+
if (config.verbose) {
|
|
5203
|
+
console.log(
|
|
5204
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Updating analytics for ${transactions.length} transactions...`
|
|
5205
|
+
);
|
|
5206
|
+
}
|
|
5207
|
+
try {
|
|
5208
|
+
const byHour = groupByCohort(transactions, "cohortHour");
|
|
5209
|
+
const cohortCount = Object.keys(byHour).length;
|
|
5210
|
+
if (config.verbose) {
|
|
5211
|
+
console.log(
|
|
5212
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Updating ${cohortCount} hourly analytics cohorts...`
|
|
5052
5213
|
);
|
|
5053
|
-
|
|
5054
|
-
|
|
5055
|
-
|
|
5056
|
-
|
|
5057
|
-
|
|
5058
|
-
|
|
5059
|
-
|
|
5060
|
-
return;
|
|
5061
|
-
}
|
|
5062
|
-
const uniqueIds = [...new Set(transactions.map((t) => t.originalId))];
|
|
5063
|
-
if (this.config.verbose) {
|
|
5214
|
+
}
|
|
5215
|
+
for (const [cohort, txns] of Object.entries(byHour)) {
|
|
5216
|
+
await upsertAnalytics("hour", cohort, txns, analyticsResource, config);
|
|
5217
|
+
}
|
|
5218
|
+
if (config.analyticsConfig.rollupStrategy === "incremental") {
|
|
5219
|
+
const uniqueHours = Object.keys(byHour);
|
|
5220
|
+
if (config.verbose) {
|
|
5064
5221
|
console.log(
|
|
5065
|
-
`[EventualConsistency] ${
|
|
5066
|
-
);
|
|
5067
|
-
}
|
|
5068
|
-
const { results, errors } = await PromisePool.for(uniqueIds).withConcurrency(this.config.consolidationConcurrency).process(async (id) => {
|
|
5069
|
-
return await this.consolidateRecord(id);
|
|
5070
|
-
});
|
|
5071
|
-
const duration = Date.now() - startTime;
|
|
5072
|
-
if (errors && errors.length > 0) {
|
|
5073
|
-
console.error(
|
|
5074
|
-
`[EventualConsistency] ${this.config.resource}.${this.config.field} - Consolidation completed with ${errors.length} errors in ${duration}ms:`,
|
|
5075
|
-
errors
|
|
5222
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Rolling up ${uniqueHours.length} hours to daily/monthly analytics...`
|
|
5076
5223
|
);
|
|
5077
5224
|
}
|
|
5078
|
-
|
|
5079
|
-
|
|
5080
|
-
`[EventualConsistency] ${this.config.resource}.${this.config.field} - Consolidation complete: ${results.length} records consolidated in ${duration}ms (${errors.length} errors). Next run in ${this.config.consolidationInterval}s`
|
|
5081
|
-
);
|
|
5225
|
+
for (const cohortHour of uniqueHours) {
|
|
5226
|
+
await rollupAnalytics(cohortHour, analyticsResource, config);
|
|
5082
5227
|
}
|
|
5083
|
-
|
|
5084
|
-
|
|
5085
|
-
|
|
5086
|
-
|
|
5087
|
-
successCount: results.length,
|
|
5088
|
-
errorCount: errors.length,
|
|
5089
|
-
duration
|
|
5090
|
-
});
|
|
5091
|
-
} catch (error) {
|
|
5092
|
-
const duration = Date.now() - startTime;
|
|
5093
|
-
console.error(
|
|
5094
|
-
`[EventualConsistency] ${this.config.resource}.${this.config.field} - Consolidation error after ${duration}ms:`,
|
|
5095
|
-
error
|
|
5228
|
+
}
|
|
5229
|
+
if (config.verbose) {
|
|
5230
|
+
console.log(
|
|
5231
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Analytics update complete for ${cohortCount} cohorts`
|
|
5096
5232
|
);
|
|
5097
|
-
this.emit("eventual-consistency.consolidation-error", error);
|
|
5098
5233
|
}
|
|
5234
|
+
} catch (error) {
|
|
5235
|
+
console.warn(
|
|
5236
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Analytics update error:`,
|
|
5237
|
+
error.message
|
|
5238
|
+
);
|
|
5099
5239
|
}
|
|
5100
|
-
|
|
5101
|
-
|
|
5102
|
-
|
|
5103
|
-
|
|
5104
|
-
|
|
5105
|
-
|
|
5106
|
-
|
|
5107
|
-
|
|
5240
|
+
}
|
|
5241
|
+
async function upsertAnalytics(period, cohort, transactions, analyticsResource, config) {
|
|
5242
|
+
const id = `${period}-${cohort}`;
|
|
5243
|
+
const transactionCount = transactions.length;
|
|
5244
|
+
const signedValues = transactions.map((t) => {
|
|
5245
|
+
if (t.operation === "sub") return -t.value;
|
|
5246
|
+
return t.value;
|
|
5247
|
+
});
|
|
5248
|
+
const totalValue = signedValues.reduce((sum, v) => sum + v, 0);
|
|
5249
|
+
const avgValue = totalValue / transactionCount;
|
|
5250
|
+
const minValue = Math.min(...signedValues);
|
|
5251
|
+
const maxValue = Math.max(...signedValues);
|
|
5252
|
+
const operations = calculateOperationBreakdown(transactions);
|
|
5253
|
+
const recordCount = new Set(transactions.map((t) => t.originalId)).size;
|
|
5254
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
5255
|
+
const [existingOk, existingErr, existing] = await tryFn(
|
|
5256
|
+
() => analyticsResource.get(id)
|
|
5257
|
+
);
|
|
5258
|
+
if (existingOk && existing) {
|
|
5259
|
+
const newTransactionCount = existing.transactionCount + transactionCount;
|
|
5260
|
+
const newTotalValue = existing.totalValue + totalValue;
|
|
5261
|
+
const newAvgValue = newTotalValue / newTransactionCount;
|
|
5262
|
+
const newMinValue = Math.min(existing.minValue, minValue);
|
|
5263
|
+
const newMaxValue = Math.max(existing.maxValue, maxValue);
|
|
5264
|
+
const newOperations = { ...existing.operations };
|
|
5265
|
+
for (const [op, stats] of Object.entries(operations)) {
|
|
5266
|
+
if (!newOperations[op]) {
|
|
5267
|
+
newOperations[op] = { count: 0, sum: 0 };
|
|
5268
|
+
}
|
|
5269
|
+
newOperations[op].count += stats.count;
|
|
5270
|
+
newOperations[op].sum += stats.sum;
|
|
5271
|
+
}
|
|
5272
|
+
const newRecordCount = Math.max(existing.recordCount, recordCount);
|
|
5273
|
+
await tryFn(
|
|
5274
|
+
() => analyticsResource.update(id, {
|
|
5275
|
+
transactionCount: newTransactionCount,
|
|
5276
|
+
totalValue: newTotalValue,
|
|
5277
|
+
avgValue: newAvgValue,
|
|
5278
|
+
minValue: newMinValue,
|
|
5279
|
+
maxValue: newMaxValue,
|
|
5280
|
+
operations: newOperations,
|
|
5281
|
+
recordCount: newRecordCount,
|
|
5282
|
+
updatedAt: now
|
|
5108
5283
|
})
|
|
5109
5284
|
);
|
|
5110
|
-
|
|
5111
|
-
|
|
5112
|
-
|
|
5113
|
-
|
|
5114
|
-
|
|
5115
|
-
|
|
5116
|
-
|
|
5117
|
-
|
|
5118
|
-
|
|
5119
|
-
|
|
5120
|
-
|
|
5121
|
-
|
|
5122
|
-
|
|
5123
|
-
|
|
5124
|
-
|
|
5125
|
-
)
|
|
5126
|
-
|
|
5127
|
-
|
|
5128
|
-
|
|
5129
|
-
|
|
5130
|
-
|
|
5131
|
-
|
|
5132
|
-
|
|
5133
|
-
|
|
5134
|
-
|
|
5135
|
-
|
|
5136
|
-
|
|
5137
|
-
|
|
5138
|
-
|
|
5139
|
-
|
|
5140
|
-
|
|
5141
|
-
|
|
5142
|
-
|
|
5143
|
-
|
|
5144
|
-
|
|
5145
|
-
|
|
5146
|
-
|
|
5147
|
-
|
|
5148
|
-
|
|
5149
|
-
|
|
5150
|
-
|
|
5151
|
-
|
|
5152
|
-
|
|
5153
|
-
|
|
5154
|
-
|
|
5155
|
-
|
|
5156
|
-
|
|
5157
|
-
|
|
5158
|
-
|
|
5159
|
-
|
|
5160
|
-
|
|
5161
|
-
|
|
5162
|
-
|
|
5163
|
-
|
|
5164
|
-
|
|
5165
|
-
|
|
5166
|
-
|
|
5167
|
-
|
|
5168
|
-
|
|
5169
|
-
|
|
5170
|
-
|
|
5171
|
-
|
|
5172
|
-
|
|
5173
|
-
|
|
5174
|
-
|
|
5175
|
-
|
|
5176
|
-
|
|
5177
|
-
|
|
5178
|
-
|
|
5179
|
-
|
|
5180
|
-
|
|
5181
|
-
|
|
5182
|
-
|
|
5183
|
-
|
|
5184
|
-
|
|
5185
|
-
|
|
5186
|
-
|
|
5187
|
-
|
|
5188
|
-
|
|
5189
|
-
|
|
5190
|
-
|
|
5191
|
-
|
|
5192
|
-
|
|
5193
|
-
|
|
5194
|
-
|
|
5195
|
-
|
|
5196
|
-
|
|
5197
|
-
|
|
5198
|
-
|
|
5199
|
-
|
|
5200
|
-
|
|
5201
|
-
|
|
5202
|
-
|
|
5203
|
-
|
|
5204
|
-
|
|
5205
|
-
|
|
5206
|
-
|
|
5207
|
-
|
|
5208
|
-
|
|
5209
|
-
|
|
5210
|
-
|
|
5211
|
-
|
|
5212
|
-
|
|
5213
|
-
|
|
5214
|
-
|
|
5215
|
-
|
|
5216
|
-
|
|
5217
|
-
|
|
5218
|
-
|
|
5219
|
-
|
|
5220
|
-
|
|
5221
|
-
|
|
5222
|
-
|
|
5223
|
-
|
|
5224
|
-
|
|
5225
|
-
|
|
5226
|
-
|
|
5227
|
-
|
|
5228
|
-
|
|
5229
|
-
|
|
5230
|
-
|
|
5231
|
-
|
|
5232
|
-
|
|
5233
|
-
|
|
5234
|
-
|
|
5235
|
-
|
|
5236
|
-
|
|
5237
|
-
|
|
5238
|
-
|
|
5239
|
-
|
|
5240
|
-
|
|
5285
|
+
} else {
|
|
5286
|
+
await tryFn(
|
|
5287
|
+
() => analyticsResource.insert({
|
|
5288
|
+
id,
|
|
5289
|
+
period,
|
|
5290
|
+
cohort,
|
|
5291
|
+
transactionCount,
|
|
5292
|
+
totalValue,
|
|
5293
|
+
avgValue,
|
|
5294
|
+
minValue,
|
|
5295
|
+
maxValue,
|
|
5296
|
+
operations,
|
|
5297
|
+
recordCount,
|
|
5298
|
+
consolidatedAt: now,
|
|
5299
|
+
updatedAt: now
|
|
5300
|
+
})
|
|
5301
|
+
);
|
|
5302
|
+
}
|
|
5303
|
+
}
|
|
5304
|
+
function calculateOperationBreakdown(transactions) {
|
|
5305
|
+
const breakdown = {};
|
|
5306
|
+
for (const txn of transactions) {
|
|
5307
|
+
const op = txn.operation;
|
|
5308
|
+
if (!breakdown[op]) {
|
|
5309
|
+
breakdown[op] = { count: 0, sum: 0 };
|
|
5310
|
+
}
|
|
5311
|
+
breakdown[op].count++;
|
|
5312
|
+
const signedValue = op === "sub" ? -txn.value : txn.value;
|
|
5313
|
+
breakdown[op].sum += signedValue;
|
|
5314
|
+
}
|
|
5315
|
+
return breakdown;
|
|
5316
|
+
}
|
|
5317
|
+
async function rollupAnalytics(cohortHour, analyticsResource, config) {
|
|
5318
|
+
const cohortDate = cohortHour.substring(0, 10);
|
|
5319
|
+
const cohortMonth = cohortHour.substring(0, 7);
|
|
5320
|
+
await rollupPeriod("day", cohortDate, cohortDate, analyticsResource);
|
|
5321
|
+
await rollupPeriod("month", cohortMonth, cohortMonth, analyticsResource);
|
|
5322
|
+
}
|
|
5323
|
+
async function rollupPeriod(period, cohort, sourcePrefix, analyticsResource, config) {
|
|
5324
|
+
const sourcePeriod = period === "day" ? "hour" : "day";
|
|
5325
|
+
const [ok, err, allAnalytics] = await tryFn(
|
|
5326
|
+
() => analyticsResource.list()
|
|
5327
|
+
);
|
|
5328
|
+
if (!ok || !allAnalytics) return;
|
|
5329
|
+
const sourceAnalytics = allAnalytics.filter(
|
|
5330
|
+
(a) => a.period === sourcePeriod && a.cohort.startsWith(sourcePrefix)
|
|
5331
|
+
);
|
|
5332
|
+
if (sourceAnalytics.length === 0) return;
|
|
5333
|
+
const transactionCount = sourceAnalytics.reduce((sum, a) => sum + a.transactionCount, 0);
|
|
5334
|
+
const totalValue = sourceAnalytics.reduce((sum, a) => sum + a.totalValue, 0);
|
|
5335
|
+
const avgValue = totalValue / transactionCount;
|
|
5336
|
+
const minValue = Math.min(...sourceAnalytics.map((a) => a.minValue));
|
|
5337
|
+
const maxValue = Math.max(...sourceAnalytics.map((a) => a.maxValue));
|
|
5338
|
+
const operations = {};
|
|
5339
|
+
for (const analytics of sourceAnalytics) {
|
|
5340
|
+
for (const [op, stats] of Object.entries(analytics.operations || {})) {
|
|
5341
|
+
if (!operations[op]) {
|
|
5342
|
+
operations[op] = { count: 0, sum: 0 };
|
|
5343
|
+
}
|
|
5344
|
+
operations[op].count += stats.count;
|
|
5345
|
+
operations[op].sum += stats.sum;
|
|
5346
|
+
}
|
|
5347
|
+
}
|
|
5348
|
+
const recordCount = Math.max(...sourceAnalytics.map((a) => a.recordCount));
|
|
5349
|
+
const id = `${period}-${cohort}`;
|
|
5350
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
5351
|
+
const [existingOk, existingErr, existing] = await tryFn(
|
|
5352
|
+
() => analyticsResource.get(id)
|
|
5353
|
+
);
|
|
5354
|
+
if (existingOk && existing) {
|
|
5355
|
+
await tryFn(
|
|
5356
|
+
() => analyticsResource.update(id, {
|
|
5357
|
+
transactionCount,
|
|
5358
|
+
totalValue,
|
|
5359
|
+
avgValue,
|
|
5360
|
+
minValue,
|
|
5361
|
+
maxValue,
|
|
5362
|
+
operations,
|
|
5363
|
+
recordCount,
|
|
5364
|
+
updatedAt: now
|
|
5365
|
+
})
|
|
5366
|
+
);
|
|
5367
|
+
} else {
|
|
5368
|
+
await tryFn(
|
|
5369
|
+
() => analyticsResource.insert({
|
|
5370
|
+
id,
|
|
5371
|
+
period,
|
|
5372
|
+
cohort,
|
|
5373
|
+
transactionCount,
|
|
5374
|
+
totalValue,
|
|
5375
|
+
avgValue,
|
|
5376
|
+
minValue,
|
|
5377
|
+
maxValue,
|
|
5378
|
+
operations,
|
|
5379
|
+
recordCount,
|
|
5380
|
+
consolidatedAt: now,
|
|
5381
|
+
updatedAt: now
|
|
5382
|
+
})
|
|
5383
|
+
);
|
|
5384
|
+
}
|
|
5385
|
+
}
|
|
5386
|
+
function fillGaps(data, period, startDate, endDate) {
|
|
5387
|
+
if (!data || data.length === 0) {
|
|
5388
|
+
data = [];
|
|
5389
|
+
}
|
|
5390
|
+
const dataMap = /* @__PURE__ */ new Map();
|
|
5391
|
+
data.forEach((item) => {
|
|
5392
|
+
dataMap.set(item.cohort, item);
|
|
5393
|
+
});
|
|
5394
|
+
const result = [];
|
|
5395
|
+
const emptyRecord = {
|
|
5396
|
+
count: 0,
|
|
5397
|
+
sum: 0,
|
|
5398
|
+
avg: 0,
|
|
5399
|
+
min: 0,
|
|
5400
|
+
max: 0,
|
|
5401
|
+
recordCount: 0
|
|
5402
|
+
};
|
|
5403
|
+
if (period === "hour") {
|
|
5404
|
+
const start = /* @__PURE__ */ new Date(startDate + "T00:00:00Z");
|
|
5405
|
+
const end = /* @__PURE__ */ new Date(endDate + "T23:59:59Z");
|
|
5406
|
+
for (let dt = new Date(start); dt <= end; dt.setHours(dt.getHours() + 1)) {
|
|
5407
|
+
const cohort = dt.toISOString().substring(0, 13);
|
|
5408
|
+
result.push(dataMap.get(cohort) || { cohort, ...emptyRecord });
|
|
5409
|
+
}
|
|
5410
|
+
} else if (period === "day") {
|
|
5411
|
+
const start = new Date(startDate);
|
|
5412
|
+
const end = new Date(endDate);
|
|
5413
|
+
for (let dt = new Date(start); dt <= end; dt.setDate(dt.getDate() + 1)) {
|
|
5414
|
+
const cohort = dt.toISOString().substring(0, 10);
|
|
5415
|
+
result.push(dataMap.get(cohort) || { cohort, ...emptyRecord });
|
|
5416
|
+
}
|
|
5417
|
+
} else if (period === "month") {
|
|
5418
|
+
const startYear = parseInt(startDate.substring(0, 4));
|
|
5419
|
+
const startMonth = parseInt(startDate.substring(5, 7));
|
|
5420
|
+
const endYear = parseInt(endDate.substring(0, 4));
|
|
5421
|
+
const endMonth = parseInt(endDate.substring(5, 7));
|
|
5422
|
+
for (let year = startYear; year <= endYear; year++) {
|
|
5423
|
+
const firstMonth = year === startYear ? startMonth : 1;
|
|
5424
|
+
const lastMonth = year === endYear ? endMonth : 12;
|
|
5425
|
+
for (let month = firstMonth; month <= lastMonth; month++) {
|
|
5426
|
+
const cohort = `${year}-${month.toString().padStart(2, "0")}`;
|
|
5427
|
+
result.push(dataMap.get(cohort) || { cohort, ...emptyRecord });
|
|
5241
5428
|
}
|
|
5242
|
-
|
|
5243
|
-
|
|
5429
|
+
}
|
|
5430
|
+
}
|
|
5431
|
+
return result;
|
|
5432
|
+
}
|
|
5433
|
+
async function getAnalytics(resourceName, field, options, fieldHandlers) {
|
|
5434
|
+
const resourceHandlers = fieldHandlers.get(resourceName);
|
|
5435
|
+
if (!resourceHandlers) {
|
|
5436
|
+
throw new Error(`No eventual consistency configured for resource: ${resourceName}`);
|
|
5437
|
+
}
|
|
5438
|
+
const handler = resourceHandlers.get(field);
|
|
5439
|
+
if (!handler) {
|
|
5440
|
+
throw new Error(`No eventual consistency configured for field: ${resourceName}.${field}`);
|
|
5441
|
+
}
|
|
5442
|
+
if (!handler.analyticsResource) {
|
|
5443
|
+
throw new Error("Analytics not enabled for this plugin");
|
|
5444
|
+
}
|
|
5445
|
+
const { period = "day", date, startDate, endDate, month, year, breakdown = false } = options;
|
|
5446
|
+
const [ok, err, allAnalytics] = await tryFn(
|
|
5447
|
+
() => handler.analyticsResource.list()
|
|
5448
|
+
);
|
|
5449
|
+
if (!ok || !allAnalytics) {
|
|
5450
|
+
return [];
|
|
5451
|
+
}
|
|
5452
|
+
let filtered = allAnalytics.filter((a) => a.period === period);
|
|
5453
|
+
if (date) {
|
|
5454
|
+
if (period === "hour") {
|
|
5455
|
+
filtered = filtered.filter((a) => a.cohort.startsWith(date));
|
|
5456
|
+
} else {
|
|
5457
|
+
filtered = filtered.filter((a) => a.cohort === date);
|
|
5458
|
+
}
|
|
5459
|
+
} else if (startDate && endDate) {
|
|
5460
|
+
filtered = filtered.filter((a) => a.cohort >= startDate && a.cohort <= endDate);
|
|
5461
|
+
} else if (month) {
|
|
5462
|
+
filtered = filtered.filter((a) => a.cohort.startsWith(month));
|
|
5463
|
+
} else if (year) {
|
|
5464
|
+
filtered = filtered.filter((a) => a.cohort.startsWith(String(year)));
|
|
5465
|
+
}
|
|
5466
|
+
filtered.sort((a, b) => a.cohort.localeCompare(b.cohort));
|
|
5467
|
+
if (breakdown === "operations") {
|
|
5468
|
+
return filtered.map((a) => ({
|
|
5469
|
+
cohort: a.cohort,
|
|
5470
|
+
...a.operations
|
|
5471
|
+
}));
|
|
5472
|
+
}
|
|
5473
|
+
return filtered.map((a) => ({
|
|
5474
|
+
cohort: a.cohort,
|
|
5475
|
+
count: a.transactionCount,
|
|
5476
|
+
sum: a.totalValue,
|
|
5477
|
+
avg: a.avgValue,
|
|
5478
|
+
min: a.minValue,
|
|
5479
|
+
max: a.maxValue,
|
|
5480
|
+
operations: a.operations,
|
|
5481
|
+
recordCount: a.recordCount
|
|
5482
|
+
}));
|
|
5483
|
+
}
|
|
5484
|
+
async function getMonthByDay(resourceName, field, month, options, fieldHandlers) {
|
|
5485
|
+
const year = parseInt(month.substring(0, 4));
|
|
5486
|
+
const monthNum = parseInt(month.substring(5, 7));
|
|
5487
|
+
const firstDay = new Date(year, monthNum - 1, 1);
|
|
5488
|
+
const lastDay = new Date(year, monthNum, 0);
|
|
5489
|
+
const startDate = firstDay.toISOString().substring(0, 10);
|
|
5490
|
+
const endDate = lastDay.toISOString().substring(0, 10);
|
|
5491
|
+
const data = await getAnalytics(resourceName, field, {
|
|
5492
|
+
period: "day",
|
|
5493
|
+
startDate,
|
|
5494
|
+
endDate
|
|
5495
|
+
}, fieldHandlers);
|
|
5496
|
+
if (options.fillGaps) {
|
|
5497
|
+
return fillGaps(data, "day", startDate, endDate);
|
|
5498
|
+
}
|
|
5499
|
+
return data;
|
|
5500
|
+
}
|
|
5501
|
+
async function getDayByHour(resourceName, field, date, options, fieldHandlers) {
|
|
5502
|
+
const data = await getAnalytics(resourceName, field, {
|
|
5503
|
+
period: "hour",
|
|
5504
|
+
date
|
|
5505
|
+
}, fieldHandlers);
|
|
5506
|
+
if (options.fillGaps) {
|
|
5507
|
+
return fillGaps(data, "hour", date, date);
|
|
5508
|
+
}
|
|
5509
|
+
return data;
|
|
5510
|
+
}
|
|
5511
|
+
async function getLastNDays(resourceName, field, days, options, fieldHandlers) {
|
|
5512
|
+
const dates = Array.from({ length: days }, (_, i) => {
|
|
5513
|
+
const date = /* @__PURE__ */ new Date();
|
|
5514
|
+
date.setDate(date.getDate() - i);
|
|
5515
|
+
return date.toISOString().substring(0, 10);
|
|
5516
|
+
}).reverse();
|
|
5517
|
+
const data = await getAnalytics(resourceName, field, {
|
|
5518
|
+
period: "day",
|
|
5519
|
+
startDate: dates[0],
|
|
5520
|
+
endDate: dates[dates.length - 1]
|
|
5521
|
+
}, fieldHandlers);
|
|
5522
|
+
if (options.fillGaps) {
|
|
5523
|
+
return fillGaps(data, "day", dates[0], dates[dates.length - 1]);
|
|
5524
|
+
}
|
|
5525
|
+
return data;
|
|
5526
|
+
}
|
|
5527
|
+
async function getYearByMonth(resourceName, field, year, options, fieldHandlers) {
|
|
5528
|
+
const data = await getAnalytics(resourceName, field, {
|
|
5529
|
+
period: "month",
|
|
5530
|
+
year
|
|
5531
|
+
}, fieldHandlers);
|
|
5532
|
+
if (options.fillGaps) {
|
|
5533
|
+
const startDate = `${year}-01`;
|
|
5534
|
+
const endDate = `${year}-12`;
|
|
5535
|
+
return fillGaps(data, "month", startDate, endDate);
|
|
5536
|
+
}
|
|
5537
|
+
return data;
|
|
5538
|
+
}
|
|
5539
|
+
async function getMonthByHour(resourceName, field, month, options, fieldHandlers) {
|
|
5540
|
+
let year, monthNum;
|
|
5541
|
+
if (month === "last") {
|
|
5542
|
+
const now = /* @__PURE__ */ new Date();
|
|
5543
|
+
now.setMonth(now.getMonth() - 1);
|
|
5544
|
+
year = now.getFullYear();
|
|
5545
|
+
monthNum = now.getMonth() + 1;
|
|
5546
|
+
} else {
|
|
5547
|
+
year = parseInt(month.substring(0, 4));
|
|
5548
|
+
monthNum = parseInt(month.substring(5, 7));
|
|
5549
|
+
}
|
|
5550
|
+
const firstDay = new Date(year, monthNum - 1, 1);
|
|
5551
|
+
const lastDay = new Date(year, monthNum, 0);
|
|
5552
|
+
const startDate = firstDay.toISOString().substring(0, 10);
|
|
5553
|
+
const endDate = lastDay.toISOString().substring(0, 10);
|
|
5554
|
+
const data = await getAnalytics(resourceName, field, {
|
|
5555
|
+
period: "hour",
|
|
5556
|
+
startDate,
|
|
5557
|
+
endDate
|
|
5558
|
+
}, fieldHandlers);
|
|
5559
|
+
if (options.fillGaps) {
|
|
5560
|
+
return fillGaps(data, "hour", startDate, endDate);
|
|
5561
|
+
}
|
|
5562
|
+
return data;
|
|
5563
|
+
}
|
|
5564
|
+
async function getTopRecords(resourceName, field, options, fieldHandlers) {
|
|
5565
|
+
const resourceHandlers = fieldHandlers.get(resourceName);
|
|
5566
|
+
if (!resourceHandlers) {
|
|
5567
|
+
throw new Error(`No eventual consistency configured for resource: ${resourceName}`);
|
|
5568
|
+
}
|
|
5569
|
+
const handler = resourceHandlers.get(field);
|
|
5570
|
+
if (!handler) {
|
|
5571
|
+
throw new Error(`No eventual consistency configured for field: ${resourceName}.${field}`);
|
|
5572
|
+
}
|
|
5573
|
+
if (!handler.transactionResource) {
|
|
5574
|
+
throw new Error("Transaction resource not initialized");
|
|
5575
|
+
}
|
|
5576
|
+
const { period = "day", date, metric = "transactionCount", limit = 10 } = options;
|
|
5577
|
+
const [ok, err, transactions] = await tryFn(
|
|
5578
|
+
() => handler.transactionResource.list()
|
|
5579
|
+
);
|
|
5580
|
+
if (!ok || !transactions) {
|
|
5581
|
+
return [];
|
|
5582
|
+
}
|
|
5583
|
+
let filtered = transactions;
|
|
5584
|
+
if (date) {
|
|
5585
|
+
if (period === "hour") {
|
|
5586
|
+
filtered = transactions.filter((t) => t.cohortHour && t.cohortHour.startsWith(date));
|
|
5587
|
+
} else if (period === "day") {
|
|
5588
|
+
filtered = transactions.filter((t) => t.cohortDate === date);
|
|
5589
|
+
} else if (period === "month") {
|
|
5590
|
+
filtered = transactions.filter((t) => t.cohortMonth && t.cohortMonth.startsWith(date));
|
|
5591
|
+
}
|
|
5592
|
+
}
|
|
5593
|
+
const byRecord = {};
|
|
5594
|
+
for (const txn of filtered) {
|
|
5595
|
+
const recordId = txn.originalId;
|
|
5596
|
+
if (!byRecord[recordId]) {
|
|
5597
|
+
byRecord[recordId] = { count: 0, sum: 0 };
|
|
5598
|
+
}
|
|
5599
|
+
byRecord[recordId].count++;
|
|
5600
|
+
byRecord[recordId].sum += txn.value;
|
|
5601
|
+
}
|
|
5602
|
+
const records = Object.entries(byRecord).map(([recordId, stats]) => ({
|
|
5603
|
+
recordId,
|
|
5604
|
+
count: stats.count,
|
|
5605
|
+
sum: stats.sum
|
|
5606
|
+
}));
|
|
5607
|
+
records.sort((a, b) => {
|
|
5608
|
+
if (metric === "transactionCount") {
|
|
5609
|
+
return b.count - a.count;
|
|
5610
|
+
} else if (metric === "totalValue") {
|
|
5611
|
+
return b.sum - a.sum;
|
|
5612
|
+
}
|
|
5613
|
+
return 0;
|
|
5614
|
+
});
|
|
5615
|
+
return records.slice(0, limit);
|
|
5616
|
+
}
|
|
5617
|
+
|
|
5618
|
+
function addHelperMethods(resource, plugin, config) {
|
|
5619
|
+
resource.set = async (id, field, value) => {
|
|
5620
|
+
const { plugin: handler } = resolveFieldAndPlugin(resource, field, value);
|
|
5621
|
+
const now = /* @__PURE__ */ new Date();
|
|
5622
|
+
const cohortInfo = getCohortInfo(now, config.cohort.timezone, config.verbose);
|
|
5623
|
+
const transaction = {
|
|
5624
|
+
id: idGenerator(),
|
|
5625
|
+
originalId: id,
|
|
5626
|
+
field: handler.field,
|
|
5627
|
+
value,
|
|
5628
|
+
operation: "set",
|
|
5629
|
+
timestamp: now.toISOString(),
|
|
5630
|
+
cohortDate: cohortInfo.date,
|
|
5631
|
+
cohortHour: cohortInfo.hour,
|
|
5632
|
+
cohortMonth: cohortInfo.month,
|
|
5633
|
+
source: "set",
|
|
5634
|
+
applied: false
|
|
5635
|
+
};
|
|
5636
|
+
await handler.transactionResource.insert(transaction);
|
|
5637
|
+
if (config.mode === "sync") {
|
|
5638
|
+
return await plugin._syncModeConsolidate(handler, id, field);
|
|
5639
|
+
}
|
|
5640
|
+
return value;
|
|
5641
|
+
};
|
|
5642
|
+
resource.add = async (id, field, amount) => {
|
|
5643
|
+
const { plugin: handler } = resolveFieldAndPlugin(resource, field, amount);
|
|
5644
|
+
const now = /* @__PURE__ */ new Date();
|
|
5645
|
+
const cohortInfo = getCohortInfo(now, config.cohort.timezone, config.verbose);
|
|
5646
|
+
const transaction = {
|
|
5647
|
+
id: idGenerator(),
|
|
5648
|
+
originalId: id,
|
|
5649
|
+
field: handler.field,
|
|
5650
|
+
value: amount,
|
|
5651
|
+
operation: "add",
|
|
5652
|
+
timestamp: now.toISOString(),
|
|
5653
|
+
cohortDate: cohortInfo.date,
|
|
5654
|
+
cohortHour: cohortInfo.hour,
|
|
5655
|
+
cohortMonth: cohortInfo.month,
|
|
5656
|
+
source: "add",
|
|
5657
|
+
applied: false
|
|
5658
|
+
};
|
|
5659
|
+
await handler.transactionResource.insert(transaction);
|
|
5660
|
+
if (config.mode === "sync") {
|
|
5661
|
+
return await plugin._syncModeConsolidate(handler, id, field);
|
|
5662
|
+
}
|
|
5663
|
+
const [ok, err, record] = await tryFn(() => handler.targetResource.get(id));
|
|
5664
|
+
const currentValue = ok && record ? record[field] || 0 : 0;
|
|
5665
|
+
return currentValue + amount;
|
|
5666
|
+
};
|
|
5667
|
+
resource.sub = async (id, field, amount) => {
|
|
5668
|
+
const { plugin: handler } = resolveFieldAndPlugin(resource, field, amount);
|
|
5669
|
+
const now = /* @__PURE__ */ new Date();
|
|
5670
|
+
const cohortInfo = getCohortInfo(now, config.cohort.timezone, config.verbose);
|
|
5671
|
+
const transaction = {
|
|
5672
|
+
id: idGenerator(),
|
|
5673
|
+
originalId: id,
|
|
5674
|
+
field: handler.field,
|
|
5675
|
+
value: amount,
|
|
5676
|
+
operation: "sub",
|
|
5677
|
+
timestamp: now.toISOString(),
|
|
5678
|
+
cohortDate: cohortInfo.date,
|
|
5679
|
+
cohortHour: cohortInfo.hour,
|
|
5680
|
+
cohortMonth: cohortInfo.month,
|
|
5681
|
+
source: "sub",
|
|
5682
|
+
applied: false
|
|
5683
|
+
};
|
|
5684
|
+
await handler.transactionResource.insert(transaction);
|
|
5685
|
+
if (config.mode === "sync") {
|
|
5686
|
+
return await plugin._syncModeConsolidate(handler, id, field);
|
|
5687
|
+
}
|
|
5688
|
+
const [ok, err, record] = await tryFn(() => handler.targetResource.get(id));
|
|
5689
|
+
const currentValue = ok && record ? record[field] || 0 : 0;
|
|
5690
|
+
return currentValue - amount;
|
|
5691
|
+
};
|
|
5692
|
+
resource.consolidate = async (id, field) => {
|
|
5693
|
+
if (!field) {
|
|
5694
|
+
throw new Error(`Field parameter is required: consolidate(id, field)`);
|
|
5695
|
+
}
|
|
5696
|
+
const handler = resource._eventualConsistencyPlugins[field];
|
|
5697
|
+
if (!handler) {
|
|
5698
|
+
const availableFields = Object.keys(resource._eventualConsistencyPlugins).join(", ");
|
|
5699
|
+
throw new Error(
|
|
5700
|
+
`No eventual consistency plugin found for field "${field}". Available fields: ${availableFields}`
|
|
5244
5701
|
);
|
|
5245
|
-
|
|
5246
|
-
|
|
5247
|
-
|
|
5248
|
-
|
|
5249
|
-
|
|
5250
|
-
|
|
5251
|
-
|
|
5252
|
-
|
|
5253
|
-
|
|
5254
|
-
}
|
|
5255
|
-
const [updateOk, updateErr] = await tryFn(
|
|
5256
|
-
() => this.targetResource.update(originalId, {
|
|
5257
|
-
[this.config.field]: consolidatedValue
|
|
5258
|
-
})
|
|
5702
|
+
}
|
|
5703
|
+
return await plugin._consolidateWithHandler(handler, id);
|
|
5704
|
+
};
|
|
5705
|
+
resource.getConsolidatedValue = async (id, field, options = {}) => {
|
|
5706
|
+
const handler = resource._eventualConsistencyPlugins[field];
|
|
5707
|
+
if (!handler) {
|
|
5708
|
+
const availableFields = Object.keys(resource._eventualConsistencyPlugins).join(", ");
|
|
5709
|
+
throw new Error(
|
|
5710
|
+
`No eventual consistency plugin found for field "${field}". Available fields: ${availableFields}`
|
|
5259
5711
|
);
|
|
5260
|
-
if (!updateOk) {
|
|
5261
|
-
if (updateErr?.message?.includes("does not exist")) {
|
|
5262
|
-
if (this.config.verbose) {
|
|
5263
|
-
console.warn(
|
|
5264
|
-
`[EventualConsistency] ${this.config.resource}.${this.config.field} - Record ${originalId} doesn't exist. Skipping consolidation. ${transactions.length} transactions will remain pending until record is created.`
|
|
5265
|
-
);
|
|
5266
|
-
}
|
|
5267
|
-
return consolidatedValue;
|
|
5268
|
-
}
|
|
5269
|
-
console.error(
|
|
5270
|
-
`[EventualConsistency] ${this.config.resource}.${this.config.field} - FAILED to update ${originalId}: ${updateErr?.message || updateErr}`,
|
|
5271
|
-
{ error: updateErr, consolidatedValue, currentValue }
|
|
5272
|
-
);
|
|
5273
|
-
throw updateErr;
|
|
5274
|
-
}
|
|
5275
|
-
if (updateOk) {
|
|
5276
|
-
const transactionsToUpdate = transactions.filter((txn) => txn.id !== "__synthetic__");
|
|
5277
|
-
const { results, errors } = await PromisePool.for(transactionsToUpdate).withConcurrency(10).process(async (txn) => {
|
|
5278
|
-
const [ok2, err2] = await tryFn(
|
|
5279
|
-
() => this.transactionResource.update(txn.id, { applied: true })
|
|
5280
|
-
);
|
|
5281
|
-
if (!ok2 && this.config.verbose) {
|
|
5282
|
-
console.warn(`[EventualConsistency] Failed to mark transaction ${txn.id} as applied:`, err2?.message);
|
|
5283
|
-
}
|
|
5284
|
-
return ok2;
|
|
5285
|
-
});
|
|
5286
|
-
if (errors && errors.length > 0 && this.config.verbose) {
|
|
5287
|
-
console.warn(`[EventualConsistency] ${errors.length} transactions failed to mark as applied`);
|
|
5288
|
-
}
|
|
5289
|
-
if (this.config.enableAnalytics && transactionsToUpdate.length > 0) {
|
|
5290
|
-
await this.updateAnalytics(transactionsToUpdate);
|
|
5291
|
-
}
|
|
5292
|
-
if (this.targetResource && this.targetResource.cache && typeof this.targetResource.cache.delete === "function") {
|
|
5293
|
-
try {
|
|
5294
|
-
const cacheKey = await this.targetResource.cacheKeyFor({ id: originalId });
|
|
5295
|
-
await this.targetResource.cache.delete(cacheKey);
|
|
5296
|
-
if (this.config.verbose) {
|
|
5297
|
-
console.log(
|
|
5298
|
-
`[EventualConsistency] ${this.config.resource}.${this.config.field} - Cache invalidated for ${originalId}`
|
|
5299
|
-
);
|
|
5300
|
-
}
|
|
5301
|
-
} catch (cacheErr) {
|
|
5302
|
-
if (this.config.verbose) {
|
|
5303
|
-
console.warn(
|
|
5304
|
-
`[EventualConsistency] ${this.config.resource}.${this.config.field} - Failed to invalidate cache for ${originalId}: ${cacheErr?.message}`
|
|
5305
|
-
);
|
|
5306
|
-
}
|
|
5307
|
-
}
|
|
5308
|
-
}
|
|
5309
|
-
}
|
|
5310
|
-
return consolidatedValue;
|
|
5311
|
-
} finally {
|
|
5312
|
-
const [lockReleased, lockReleaseErr] = await tryFn(() => this.lockResource.delete(lockId));
|
|
5313
|
-
if (!lockReleased && this.config.verbose) {
|
|
5314
|
-
console.warn(`[EventualConsistency] Failed to release lock ${lockId}:`, lockReleaseErr?.message);
|
|
5315
|
-
}
|
|
5316
5712
|
}
|
|
5317
|
-
|
|
5318
|
-
|
|
5319
|
-
|
|
5320
|
-
|
|
5321
|
-
|
|
5322
|
-
const query = { originalId };
|
|
5323
|
-
if (!includeApplied) {
|
|
5324
|
-
query.applied = false;
|
|
5713
|
+
return await plugin._getConsolidatedValueWithHandler(handler, id, options);
|
|
5714
|
+
};
|
|
5715
|
+
resource.recalculate = async (id, field) => {
|
|
5716
|
+
if (!field) {
|
|
5717
|
+
throw new Error(`Field parameter is required: recalculate(id, field)`);
|
|
5325
5718
|
}
|
|
5326
|
-
const
|
|
5327
|
-
|
|
5328
|
-
|
|
5329
|
-
|
|
5330
|
-
|
|
5331
|
-
() => this.targetResource.get(originalId)
|
|
5719
|
+
const handler = resource._eventualConsistencyPlugins[field];
|
|
5720
|
+
if (!handler) {
|
|
5721
|
+
const availableFields = Object.keys(resource._eventualConsistencyPlugins).join(", ");
|
|
5722
|
+
throw new Error(
|
|
5723
|
+
`No eventual consistency plugin found for field "${field}". Available fields: ${availableFields}`
|
|
5332
5724
|
);
|
|
5333
|
-
|
|
5334
|
-
|
|
5725
|
+
}
|
|
5726
|
+
return await plugin._recalculateWithHandler(handler, id);
|
|
5727
|
+
};
|
|
5728
|
+
}
|
|
5729
|
+
|
|
5730
|
+
async function onSetup(database, fieldHandlers, completeFieldSetupFn, watchForResourceFn) {
|
|
5731
|
+
for (const [resourceName, resourceHandlers] of fieldHandlers) {
|
|
5732
|
+
const targetResource = database.resources[resourceName];
|
|
5733
|
+
if (!targetResource) {
|
|
5734
|
+
for (const handler of resourceHandlers.values()) {
|
|
5735
|
+
handler.deferredSetup = true;
|
|
5335
5736
|
}
|
|
5336
|
-
|
|
5737
|
+
watchForResourceFn(resourceName);
|
|
5738
|
+
continue;
|
|
5337
5739
|
}
|
|
5338
|
-
|
|
5339
|
-
|
|
5340
|
-
|
|
5341
|
-
const timestamp = new Date(t.timestamp);
|
|
5342
|
-
if (startDate && timestamp < new Date(startDate)) return false;
|
|
5343
|
-
if (endDate && timestamp > new Date(endDate)) return false;
|
|
5344
|
-
return true;
|
|
5345
|
-
});
|
|
5740
|
+
for (const [fieldName, handler] of resourceHandlers) {
|
|
5741
|
+
handler.targetResource = targetResource;
|
|
5742
|
+
await completeFieldSetupFn(handler);
|
|
5346
5743
|
}
|
|
5347
|
-
|
|
5348
|
-
|
|
5349
|
-
|
|
5350
|
-
|
|
5351
|
-
|
|
5352
|
-
|
|
5353
|
-
|
|
5744
|
+
}
|
|
5745
|
+
}
|
|
5746
|
+
function watchForResource(resourceName, database, fieldHandlers, completeFieldSetupFn) {
|
|
5747
|
+
const hookCallback = async ({ resource, config }) => {
|
|
5748
|
+
if (config.name === resourceName) {
|
|
5749
|
+
const resourceHandlers = fieldHandlers.get(resourceName);
|
|
5750
|
+
if (!resourceHandlers) return;
|
|
5751
|
+
for (const [fieldName, handler] of resourceHandlers) {
|
|
5752
|
+
if (handler.deferredSetup) {
|
|
5753
|
+
handler.targetResource = resource;
|
|
5754
|
+
handler.deferredSetup = false;
|
|
5755
|
+
await completeFieldSetupFn(handler);
|
|
5756
|
+
}
|
|
5757
|
+
}
|
|
5354
5758
|
}
|
|
5355
|
-
|
|
5356
|
-
|
|
5759
|
+
};
|
|
5760
|
+
database.addHook("afterCreateResource", hookCallback);
|
|
5761
|
+
}
|
|
5762
|
+
async function completeFieldSetup(handler, database, config, plugin) {
|
|
5763
|
+
if (!handler.targetResource) return;
|
|
5764
|
+
const resourceName = handler.resource;
|
|
5765
|
+
const fieldName = handler.field;
|
|
5766
|
+
const transactionResourceName = `${resourceName}_transactions_${fieldName}`;
|
|
5767
|
+
const partitionConfig = createPartitionConfig();
|
|
5768
|
+
const [ok, err, transactionResource] = await tryFn(
|
|
5769
|
+
() => database.createResource({
|
|
5770
|
+
name: transactionResourceName,
|
|
5771
|
+
attributes: {
|
|
5772
|
+
id: "string|required",
|
|
5773
|
+
originalId: "string|required",
|
|
5774
|
+
field: "string|required",
|
|
5775
|
+
value: "number|required",
|
|
5776
|
+
operation: "string|required",
|
|
5777
|
+
timestamp: "string|required",
|
|
5778
|
+
cohortDate: "string|required",
|
|
5779
|
+
cohortHour: "string|required",
|
|
5780
|
+
cohortMonth: "string|optional",
|
|
5781
|
+
source: "string|optional",
|
|
5782
|
+
applied: "boolean|optional"
|
|
5783
|
+
},
|
|
5784
|
+
behavior: "body-overflow",
|
|
5785
|
+
timestamps: true,
|
|
5786
|
+
partitions: partitionConfig,
|
|
5787
|
+
asyncPartitions: true,
|
|
5788
|
+
createdBy: "EventualConsistencyPlugin"
|
|
5789
|
+
})
|
|
5790
|
+
);
|
|
5791
|
+
if (!ok && !database.resources[transactionResourceName]) {
|
|
5792
|
+
throw new Error(`Failed to create transaction resource for ${resourceName}.${fieldName}: ${err?.message}`);
|
|
5793
|
+
}
|
|
5794
|
+
handler.transactionResource = ok ? transactionResource : database.resources[transactionResourceName];
|
|
5795
|
+
const lockResourceName = `${resourceName}_consolidation_locks_${fieldName}`;
|
|
5796
|
+
const [lockOk, lockErr, lockResource] = await tryFn(
|
|
5797
|
+
() => database.createResource({
|
|
5798
|
+
name: lockResourceName,
|
|
5799
|
+
attributes: {
|
|
5800
|
+
id: "string|required",
|
|
5801
|
+
lockedAt: "number|required",
|
|
5802
|
+
workerId: "string|optional"
|
|
5803
|
+
},
|
|
5804
|
+
behavior: "body-only",
|
|
5805
|
+
timestamps: false,
|
|
5806
|
+
createdBy: "EventualConsistencyPlugin"
|
|
5807
|
+
})
|
|
5808
|
+
);
|
|
5809
|
+
if (!lockOk && !database.resources[lockResourceName]) {
|
|
5810
|
+
throw new Error(`Failed to create lock resource for ${resourceName}.${fieldName}: ${lockErr?.message}`);
|
|
5811
|
+
}
|
|
5812
|
+
handler.lockResource = lockOk ? lockResource : database.resources[lockResourceName];
|
|
5813
|
+
if (config.enableAnalytics) {
|
|
5814
|
+
await createAnalyticsResource(handler, database, resourceName, fieldName);
|
|
5815
|
+
}
|
|
5816
|
+
addHelperMethodsForHandler(handler, plugin, config);
|
|
5817
|
+
if (config.verbose) {
|
|
5818
|
+
console.log(
|
|
5819
|
+
`[EventualConsistency] ${resourceName}.${fieldName} - Setup complete. Resources: ${transactionResourceName}, ${lockResourceName}${config.enableAnalytics ? `, ${resourceName}_analytics_${fieldName}` : ""}`
|
|
5357
5820
|
);
|
|
5358
|
-
return this.config.reducer(filtered);
|
|
5359
5821
|
}
|
|
5360
|
-
|
|
5361
|
-
|
|
5362
|
-
|
|
5363
|
-
|
|
5364
|
-
|
|
5365
|
-
|
|
5366
|
-
|
|
5367
|
-
|
|
5368
|
-
|
|
5369
|
-
|
|
5370
|
-
|
|
5371
|
-
|
|
5372
|
-
|
|
5373
|
-
|
|
5374
|
-
|
|
5375
|
-
|
|
5376
|
-
|
|
5377
|
-
|
|
5378
|
-
|
|
5379
|
-
|
|
5380
|
-
|
|
5381
|
-
|
|
5382
|
-
|
|
5822
|
+
}
|
|
5823
|
+
async function createAnalyticsResource(handler, database, resourceName, fieldName) {
|
|
5824
|
+
const analyticsResourceName = `${resourceName}_analytics_${fieldName}`;
|
|
5825
|
+
const [ok, err, analyticsResource] = await tryFn(
|
|
5826
|
+
() => database.createResource({
|
|
5827
|
+
name: analyticsResourceName,
|
|
5828
|
+
attributes: {
|
|
5829
|
+
id: "string|required",
|
|
5830
|
+
period: "string|required",
|
|
5831
|
+
cohort: "string|required",
|
|
5832
|
+
transactionCount: "number|required",
|
|
5833
|
+
totalValue: "number|required",
|
|
5834
|
+
avgValue: "number|required",
|
|
5835
|
+
minValue: "number|required",
|
|
5836
|
+
maxValue: "number|required",
|
|
5837
|
+
operations: "object|optional",
|
|
5838
|
+
recordCount: "number|required",
|
|
5839
|
+
consolidatedAt: "string|required",
|
|
5840
|
+
updatedAt: "string|required"
|
|
5841
|
+
},
|
|
5842
|
+
behavior: "body-overflow",
|
|
5843
|
+
timestamps: false,
|
|
5844
|
+
createdBy: "EventualConsistencyPlugin"
|
|
5845
|
+
})
|
|
5846
|
+
);
|
|
5847
|
+
if (!ok && !database.resources[analyticsResourceName]) {
|
|
5848
|
+
throw new Error(`Failed to create analytics resource for ${resourceName}.${fieldName}: ${err?.message}`);
|
|
5849
|
+
}
|
|
5850
|
+
handler.analyticsResource = ok ? analyticsResource : database.resources[analyticsResourceName];
|
|
5851
|
+
}
|
|
5852
|
+
function addHelperMethodsForHandler(handler, plugin, config) {
|
|
5853
|
+
const resource = handler.targetResource;
|
|
5854
|
+
const fieldName = handler.field;
|
|
5855
|
+
if (!resource._eventualConsistencyPlugins) {
|
|
5856
|
+
resource._eventualConsistencyPlugins = {};
|
|
5857
|
+
}
|
|
5858
|
+
resource._eventualConsistencyPlugins[fieldName] = handler;
|
|
5859
|
+
if (!resource.add) {
|
|
5860
|
+
addHelperMethods(resource, plugin, config);
|
|
5861
|
+
}
|
|
5862
|
+
}
|
|
5863
|
+
async function onStart(fieldHandlers, config, runConsolidationFn, runGCFn, emitFn) {
|
|
5864
|
+
for (const [resourceName, resourceHandlers] of fieldHandlers) {
|
|
5865
|
+
for (const [fieldName, handler] of resourceHandlers) {
|
|
5866
|
+
if (!handler.deferredSetup) {
|
|
5867
|
+
if (config.autoConsolidate && config.mode === "async") {
|
|
5868
|
+
startConsolidationTimer(handler, resourceName, fieldName, runConsolidationFn, config);
|
|
5869
|
+
}
|
|
5870
|
+
if (config.transactionRetention && config.transactionRetention > 0) {
|
|
5871
|
+
startGarbageCollectionTimer(handler, resourceName, fieldName, runGCFn, config);
|
|
5872
|
+
}
|
|
5873
|
+
if (emitFn) {
|
|
5874
|
+
emitFn("eventual-consistency.started", {
|
|
5875
|
+
resource: resourceName,
|
|
5876
|
+
field: fieldName,
|
|
5877
|
+
cohort: config.cohort
|
|
5878
|
+
});
|
|
5879
|
+
}
|
|
5383
5880
|
}
|
|
5384
|
-
stats.byOriginalId[txn.originalId].count++;
|
|
5385
|
-
stats.byOriginalId[txn.originalId].value += txn.value || 0;
|
|
5386
5881
|
}
|
|
5387
|
-
return stats;
|
|
5388
5882
|
}
|
|
5389
|
-
|
|
5390
|
-
|
|
5391
|
-
|
|
5392
|
-
|
|
5393
|
-
|
|
5394
|
-
|
|
5395
|
-
|
|
5396
|
-
|
|
5397
|
-
|
|
5398
|
-
|
|
5399
|
-
|
|
5400
|
-
|
|
5401
|
-
|
|
5402
|
-
|
|
5403
|
-
}
|
|
5404
|
-
|
|
5405
|
-
|
|
5406
|
-
|
|
5407
|
-
|
|
5883
|
+
}
|
|
5884
|
+
async function onStop(fieldHandlers, emitFn) {
|
|
5885
|
+
for (const [resourceName, resourceHandlers] of fieldHandlers) {
|
|
5886
|
+
for (const [fieldName, handler] of resourceHandlers) {
|
|
5887
|
+
if (handler.consolidationTimer) {
|
|
5888
|
+
clearInterval(handler.consolidationTimer);
|
|
5889
|
+
handler.consolidationTimer = null;
|
|
5890
|
+
}
|
|
5891
|
+
if (handler.gcTimer) {
|
|
5892
|
+
clearInterval(handler.gcTimer);
|
|
5893
|
+
handler.gcTimer = null;
|
|
5894
|
+
}
|
|
5895
|
+
if (handler.pendingTransactions && handler.pendingTransactions.size > 0) {
|
|
5896
|
+
await flushPendingTransactions(handler);
|
|
5897
|
+
}
|
|
5898
|
+
if (emitFn) {
|
|
5899
|
+
emitFn("eventual-consistency.stopped", {
|
|
5900
|
+
resource: resourceName,
|
|
5901
|
+
field: fieldName
|
|
5902
|
+
});
|
|
5408
5903
|
}
|
|
5409
|
-
return;
|
|
5410
5904
|
}
|
|
5411
|
-
|
|
5412
|
-
|
|
5413
|
-
|
|
5414
|
-
|
|
5415
|
-
|
|
5416
|
-
|
|
5417
|
-
|
|
5418
|
-
|
|
5419
|
-
|
|
5420
|
-
|
|
5421
|
-
|
|
5422
|
-
|
|
5423
|
-
|
|
5424
|
-
|
|
5425
|
-
|
|
5426
|
-
console.warn(`[EventualConsistency] ${errors.length} stale locks failed to delete`);
|
|
5427
|
-
}
|
|
5428
|
-
} catch (error) {
|
|
5429
|
-
if (this.config.verbose) {
|
|
5430
|
-
console.warn(`[EventualConsistency] Error cleaning up stale locks:`, error.message);
|
|
5905
|
+
}
|
|
5906
|
+
}
|
|
5907
|
+
|
|
5908
|
+
class EventualConsistencyPlugin extends Plugin {
|
|
5909
|
+
constructor(options = {}) {
|
|
5910
|
+
super(options);
|
|
5911
|
+
validateResourcesConfig(options.resources);
|
|
5912
|
+
const detectedTimezone = detectTimezone();
|
|
5913
|
+
const timezoneAutoDetected = !options.cohort?.timezone;
|
|
5914
|
+
this.config = createConfig(options, detectedTimezone);
|
|
5915
|
+
this.fieldHandlers = /* @__PURE__ */ new Map();
|
|
5916
|
+
for (const [resourceName, fields] of Object.entries(options.resources)) {
|
|
5917
|
+
const resourceHandlers = /* @__PURE__ */ new Map();
|
|
5918
|
+
for (const fieldName of fields) {
|
|
5919
|
+
resourceHandlers.set(fieldName, createFieldHandler(resourceName, fieldName));
|
|
5431
5920
|
}
|
|
5432
|
-
|
|
5433
|
-
await tryFn(() => this.lockResource.delete(cleanupLockId));
|
|
5921
|
+
this.fieldHandlers.set(resourceName, resourceHandlers);
|
|
5434
5922
|
}
|
|
5923
|
+
logConfigWarnings(this.config);
|
|
5924
|
+
logInitialization(this.config, this.fieldHandlers, timezoneAutoDetected);
|
|
5925
|
+
}
|
|
5926
|
+
/**
|
|
5927
|
+
* Setup hook - create resources and register helpers
|
|
5928
|
+
*/
|
|
5929
|
+
async onSetup() {
|
|
5930
|
+
await onSetup(
|
|
5931
|
+
this.database,
|
|
5932
|
+
this.fieldHandlers,
|
|
5933
|
+
(handler) => completeFieldSetup(handler, this.database, this.config, this),
|
|
5934
|
+
(resourceName) => watchForResource(
|
|
5935
|
+
resourceName,
|
|
5936
|
+
this.database,
|
|
5937
|
+
this.fieldHandlers,
|
|
5938
|
+
(handler) => completeFieldSetup(handler, this.database, this.config, this)
|
|
5939
|
+
)
|
|
5940
|
+
);
|
|
5941
|
+
}
|
|
5942
|
+
/**
|
|
5943
|
+
* Start hook - begin timers and emit events
|
|
5944
|
+
*/
|
|
5945
|
+
async onStart() {
|
|
5946
|
+
await onStart(
|
|
5947
|
+
this.fieldHandlers,
|
|
5948
|
+
this.config,
|
|
5949
|
+
(handler, resourceName, fieldName) => this._runConsolidationForHandler(handler, resourceName, fieldName),
|
|
5950
|
+
(handler, resourceName, fieldName) => this._runGarbageCollectionForHandler(handler, resourceName, fieldName),
|
|
5951
|
+
(event, data) => this.emit(event, data)
|
|
5952
|
+
);
|
|
5953
|
+
}
|
|
5954
|
+
/**
|
|
5955
|
+
* Stop hook - stop timers and flush pending
|
|
5956
|
+
*/
|
|
5957
|
+
async onStop() {
|
|
5958
|
+
await onStop(
|
|
5959
|
+
this.fieldHandlers,
|
|
5960
|
+
(event, data) => this.emit(event, data)
|
|
5961
|
+
);
|
|
5962
|
+
}
|
|
5963
|
+
/**
|
|
5964
|
+
* Create partition configuration
|
|
5965
|
+
* @returns {Object} Partition configuration
|
|
5966
|
+
*/
|
|
5967
|
+
createPartitionConfig() {
|
|
5968
|
+
return createPartitionConfig();
|
|
5969
|
+
}
|
|
5970
|
+
/**
|
|
5971
|
+
* Get cohort information for a date
|
|
5972
|
+
* @param {Date} date - Date to get cohort info for
|
|
5973
|
+
* @returns {Object} Cohort information
|
|
5974
|
+
*/
|
|
5975
|
+
getCohortInfo(date) {
|
|
5976
|
+
return getCohortInfo(date, this.config.cohort.timezone, this.config.verbose);
|
|
5977
|
+
}
|
|
5978
|
+
/**
|
|
5979
|
+
* Create a transaction for a field handler
|
|
5980
|
+
* @param {Object} handler - Field handler
|
|
5981
|
+
* @param {Object} data - Transaction data
|
|
5982
|
+
* @returns {Promise<Object|null>} Created transaction
|
|
5983
|
+
*/
|
|
5984
|
+
async createTransaction(handler, data) {
|
|
5985
|
+
return await createTransaction(handler, data, this.config);
|
|
5435
5986
|
}
|
|
5436
5987
|
/**
|
|
5437
|
-
*
|
|
5988
|
+
* Consolidate a single record (internal method)
|
|
5989
|
+
* This is used internally by consolidation timers and helper methods
|
|
5990
|
+
* @private
|
|
5438
5991
|
*/
|
|
5439
|
-
|
|
5440
|
-
|
|
5441
|
-
|
|
5442
|
-
|
|
5443
|
-
|
|
5992
|
+
async consolidateRecord(originalId) {
|
|
5993
|
+
return await consolidateRecord(
|
|
5994
|
+
originalId,
|
|
5995
|
+
this.transactionResource,
|
|
5996
|
+
this.targetResource,
|
|
5997
|
+
this.lockResource,
|
|
5998
|
+
this.analyticsResource,
|
|
5999
|
+
(transactions) => this.updateAnalytics(transactions),
|
|
6000
|
+
this.config
|
|
6001
|
+
);
|
|
5444
6002
|
}
|
|
5445
|
-
|
|
5446
|
-
|
|
5447
|
-
|
|
5448
|
-
|
|
5449
|
-
|
|
6003
|
+
/**
|
|
6004
|
+
* Get consolidated value without applying (internal method)
|
|
6005
|
+
* @private
|
|
6006
|
+
*/
|
|
6007
|
+
async getConsolidatedValue(originalId, options = {}) {
|
|
6008
|
+
return await getConsolidatedValue(
|
|
6009
|
+
originalId,
|
|
6010
|
+
options,
|
|
6011
|
+
this.transactionResource,
|
|
6012
|
+
this.targetResource,
|
|
6013
|
+
this.config
|
|
6014
|
+
);
|
|
5450
6015
|
}
|
|
5451
|
-
|
|
5452
|
-
|
|
5453
|
-
|
|
5454
|
-
|
|
5455
|
-
|
|
5456
|
-
|
|
5457
|
-
this.
|
|
5458
|
-
this.config.field = fieldName;
|
|
5459
|
-
this.transactionResource = handler.transactionResource;
|
|
5460
|
-
this.targetResource = handler.targetResource;
|
|
5461
|
-
this.lockResource = handler.lockResource;
|
|
5462
|
-
try {
|
|
5463
|
-
await this.runGarbageCollection();
|
|
5464
|
-
} finally {
|
|
5465
|
-
this.config.resource = oldResource;
|
|
5466
|
-
this.config.field = oldField;
|
|
5467
|
-
this.transactionResource = oldTransactionResource;
|
|
5468
|
-
this.targetResource = oldTargetResource;
|
|
5469
|
-
this.lockResource = oldLockResource;
|
|
5470
|
-
}
|
|
6016
|
+
/**
|
|
6017
|
+
* Get cohort statistics
|
|
6018
|
+
* @param {string} cohortDate - Cohort date
|
|
6019
|
+
* @returns {Promise<Object|null>} Cohort statistics
|
|
6020
|
+
*/
|
|
6021
|
+
async getCohortStats(cohortDate) {
|
|
6022
|
+
return await getCohortStats(cohortDate, this.transactionResource);
|
|
5471
6023
|
}
|
|
5472
6024
|
/**
|
|
5473
|
-
*
|
|
5474
|
-
*
|
|
6025
|
+
* Recalculate from scratch (internal method)
|
|
6026
|
+
* @private
|
|
5475
6027
|
*/
|
|
5476
|
-
async
|
|
5477
|
-
|
|
5478
|
-
|
|
5479
|
-
|
|
5480
|
-
|
|
5481
|
-
|
|
5482
|
-
|
|
5483
|
-
|
|
6028
|
+
async recalculateRecord(originalId) {
|
|
6029
|
+
return await recalculateRecord(
|
|
6030
|
+
originalId,
|
|
6031
|
+
this.transactionResource,
|
|
6032
|
+
this.targetResource,
|
|
6033
|
+
this.lockResource,
|
|
6034
|
+
(id) => this.consolidateRecord(id),
|
|
6035
|
+
this.config
|
|
5484
6036
|
);
|
|
5485
|
-
if (!lockAcquired) {
|
|
5486
|
-
if (this.config.verbose) {
|
|
5487
|
-
console.log(`[EventualConsistency] GC already running in another container`);
|
|
5488
|
-
}
|
|
5489
|
-
return;
|
|
5490
|
-
}
|
|
5491
|
-
try {
|
|
5492
|
-
const now = Date.now();
|
|
5493
|
-
const retentionMs = this.config.transactionRetention * 24 * 60 * 60 * 1e3;
|
|
5494
|
-
const cutoffDate = new Date(now - retentionMs);
|
|
5495
|
-
const cutoffIso = cutoffDate.toISOString();
|
|
5496
|
-
if (this.config.verbose) {
|
|
5497
|
-
console.log(`[EventualConsistency] Running GC for transactions older than ${cutoffIso} (${this.config.transactionRetention} days)`);
|
|
5498
|
-
}
|
|
5499
|
-
const cutoffMonth = cutoffDate.toISOString().substring(0, 7);
|
|
5500
|
-
const [ok, err, oldTransactions] = await tryFn(
|
|
5501
|
-
() => this.transactionResource.query({
|
|
5502
|
-
applied: true,
|
|
5503
|
-
timestamp: { "<": cutoffIso }
|
|
5504
|
-
})
|
|
5505
|
-
);
|
|
5506
|
-
if (!ok) {
|
|
5507
|
-
if (this.config.verbose) {
|
|
5508
|
-
console.warn(`[EventualConsistency] GC failed to query transactions:`, err?.message);
|
|
5509
|
-
}
|
|
5510
|
-
return;
|
|
5511
|
-
}
|
|
5512
|
-
if (!oldTransactions || oldTransactions.length === 0) {
|
|
5513
|
-
if (this.config.verbose) {
|
|
5514
|
-
console.log(`[EventualConsistency] No old transactions to clean up`);
|
|
5515
|
-
}
|
|
5516
|
-
return;
|
|
5517
|
-
}
|
|
5518
|
-
if (this.config.verbose) {
|
|
5519
|
-
console.log(`[EventualConsistency] Deleting ${oldTransactions.length} old transactions`);
|
|
5520
|
-
}
|
|
5521
|
-
const { results, errors } = await PromisePool.for(oldTransactions).withConcurrency(10).process(async (txn) => {
|
|
5522
|
-
const [deleted] = await tryFn(() => this.transactionResource.delete(txn.id));
|
|
5523
|
-
return deleted;
|
|
5524
|
-
});
|
|
5525
|
-
if (this.config.verbose) {
|
|
5526
|
-
console.log(`[EventualConsistency] GC completed: ${results.length} deleted, ${errors.length} errors`);
|
|
5527
|
-
}
|
|
5528
|
-
this.emit("eventual-consistency.gc-completed", {
|
|
5529
|
-
resource: this.config.resource,
|
|
5530
|
-
field: this.config.field,
|
|
5531
|
-
deletedCount: results.length,
|
|
5532
|
-
errorCount: errors.length
|
|
5533
|
-
});
|
|
5534
|
-
} catch (error) {
|
|
5535
|
-
if (this.config.verbose) {
|
|
5536
|
-
console.warn(`[EventualConsistency] GC error:`, error.message);
|
|
5537
|
-
}
|
|
5538
|
-
this.emit("eventual-consistency.gc-error", error);
|
|
5539
|
-
} finally {
|
|
5540
|
-
await tryFn(() => this.lockResource.delete(gcLockId));
|
|
5541
|
-
}
|
|
5542
6037
|
}
|
|
5543
6038
|
/**
|
|
5544
|
-
* Update analytics
|
|
5545
|
-
* @param {Array} transactions - Array of transactions that were just consolidated
|
|
6039
|
+
* Update analytics
|
|
5546
6040
|
* @private
|
|
5547
6041
|
*/
|
|
5548
6042
|
async updateAnalytics(transactions) {
|
|
5549
|
-
|
|
5550
|
-
if (this.config.verbose) {
|
|
5551
|
-
console.log(
|
|
5552
|
-
`[EventualConsistency] ${this.config.resource}.${this.config.field} - Updating analytics for ${transactions.length} transactions...`
|
|
5553
|
-
);
|
|
5554
|
-
}
|
|
5555
|
-
try {
|
|
5556
|
-
const byHour = this._groupByCohort(transactions, "cohortHour");
|
|
5557
|
-
const cohortCount = Object.keys(byHour).length;
|
|
5558
|
-
if (this.config.verbose) {
|
|
5559
|
-
console.log(
|
|
5560
|
-
`[EventualConsistency] ${this.config.resource}.${this.config.field} - Updating ${cohortCount} hourly analytics cohorts...`
|
|
5561
|
-
);
|
|
5562
|
-
}
|
|
5563
|
-
for (const [cohort, txns] of Object.entries(byHour)) {
|
|
5564
|
-
await this._upsertAnalytics("hour", cohort, txns);
|
|
5565
|
-
}
|
|
5566
|
-
if (this.config.analyticsConfig.rollupStrategy === "incremental") {
|
|
5567
|
-
const uniqueHours = Object.keys(byHour);
|
|
5568
|
-
if (this.config.verbose) {
|
|
5569
|
-
console.log(
|
|
5570
|
-
`[EventualConsistency] ${this.config.resource}.${this.config.field} - Rolling up ${uniqueHours.length} hours to daily/monthly analytics...`
|
|
5571
|
-
);
|
|
5572
|
-
}
|
|
5573
|
-
for (const cohortHour of uniqueHours) {
|
|
5574
|
-
await this._rollupAnalytics(cohortHour);
|
|
5575
|
-
}
|
|
5576
|
-
}
|
|
5577
|
-
if (this.config.verbose) {
|
|
5578
|
-
console.log(
|
|
5579
|
-
`[EventualConsistency] ${this.config.resource}.${this.config.field} - Analytics update complete for ${cohortCount} cohorts`
|
|
5580
|
-
);
|
|
5581
|
-
}
|
|
5582
|
-
} catch (error) {
|
|
5583
|
-
console.warn(
|
|
5584
|
-
`[EventualConsistency] ${this.config.resource}.${this.config.field} - Analytics update error:`,
|
|
5585
|
-
error.message
|
|
5586
|
-
);
|
|
5587
|
-
}
|
|
6043
|
+
return await updateAnalytics(transactions, this.analyticsResource, this.config);
|
|
5588
6044
|
}
|
|
5589
6045
|
/**
|
|
5590
|
-
*
|
|
6046
|
+
* Helper method for sync mode consolidation
|
|
5591
6047
|
* @private
|
|
5592
6048
|
*/
|
|
5593
|
-
|
|
5594
|
-
const
|
|
5595
|
-
|
|
5596
|
-
|
|
5597
|
-
|
|
5598
|
-
|
|
5599
|
-
|
|
5600
|
-
|
|
5601
|
-
|
|
5602
|
-
|
|
5603
|
-
|
|
6049
|
+
async _syncModeConsolidate(handler, id, field) {
|
|
6050
|
+
const oldResource = this.config.resource;
|
|
6051
|
+
const oldField = this.config.field;
|
|
6052
|
+
const oldTransactionResource = this.transactionResource;
|
|
6053
|
+
const oldTargetResource = this.targetResource;
|
|
6054
|
+
const oldLockResource = this.lockResource;
|
|
6055
|
+
const oldAnalyticsResource = this.analyticsResource;
|
|
6056
|
+
this.config.resource = handler.resource;
|
|
6057
|
+
this.config.field = handler.field;
|
|
6058
|
+
this.transactionResource = handler.transactionResource;
|
|
6059
|
+
this.targetResource = handler.targetResource;
|
|
6060
|
+
this.lockResource = handler.lockResource;
|
|
6061
|
+
this.analyticsResource = handler.analyticsResource;
|
|
6062
|
+
const result = await this.consolidateRecord(id);
|
|
6063
|
+
this.config.resource = oldResource;
|
|
6064
|
+
this.config.field = oldField;
|
|
6065
|
+
this.transactionResource = oldTransactionResource;
|
|
6066
|
+
this.targetResource = oldTargetResource;
|
|
6067
|
+
this.lockResource = oldLockResource;
|
|
6068
|
+
this.analyticsResource = oldAnalyticsResource;
|
|
6069
|
+
return result;
|
|
5604
6070
|
}
|
|
5605
6071
|
/**
|
|
5606
|
-
*
|
|
6072
|
+
* Helper method for consolidate with handler
|
|
5607
6073
|
* @private
|
|
5608
6074
|
*/
|
|
5609
|
-
async
|
|
5610
|
-
const
|
|
5611
|
-
const
|
|
5612
|
-
const
|
|
5613
|
-
|
|
5614
|
-
|
|
5615
|
-
|
|
5616
|
-
|
|
5617
|
-
|
|
5618
|
-
|
|
5619
|
-
|
|
5620
|
-
|
|
5621
|
-
|
|
5622
|
-
const
|
|
5623
|
-
|
|
5624
|
-
|
|
5625
|
-
|
|
5626
|
-
|
|
5627
|
-
|
|
5628
|
-
|
|
5629
|
-
|
|
5630
|
-
const newMinValue = Math.min(existing.minValue, minValue);
|
|
5631
|
-
const newMaxValue = Math.max(existing.maxValue, maxValue);
|
|
5632
|
-
const newOperations = { ...existing.operations };
|
|
5633
|
-
for (const [op, stats] of Object.entries(operations)) {
|
|
5634
|
-
if (!newOperations[op]) {
|
|
5635
|
-
newOperations[op] = { count: 0, sum: 0 };
|
|
5636
|
-
}
|
|
5637
|
-
newOperations[op].count += stats.count;
|
|
5638
|
-
newOperations[op].sum += stats.sum;
|
|
5639
|
-
}
|
|
5640
|
-
const newRecordCount = Math.max(existing.recordCount, recordCount);
|
|
5641
|
-
await tryFn(
|
|
5642
|
-
() => this.analyticsResource.update(id, {
|
|
5643
|
-
transactionCount: newTransactionCount,
|
|
5644
|
-
totalValue: newTotalValue,
|
|
5645
|
-
avgValue: newAvgValue,
|
|
5646
|
-
minValue: newMinValue,
|
|
5647
|
-
maxValue: newMaxValue,
|
|
5648
|
-
operations: newOperations,
|
|
5649
|
-
recordCount: newRecordCount,
|
|
5650
|
-
updatedAt: now
|
|
5651
|
-
})
|
|
5652
|
-
);
|
|
5653
|
-
} else {
|
|
5654
|
-
await tryFn(
|
|
5655
|
-
() => this.analyticsResource.insert({
|
|
5656
|
-
id,
|
|
5657
|
-
period,
|
|
5658
|
-
cohort,
|
|
5659
|
-
transactionCount,
|
|
5660
|
-
totalValue,
|
|
5661
|
-
avgValue,
|
|
5662
|
-
minValue,
|
|
5663
|
-
maxValue,
|
|
5664
|
-
operations,
|
|
5665
|
-
recordCount,
|
|
5666
|
-
consolidatedAt: now,
|
|
5667
|
-
updatedAt: now
|
|
5668
|
-
})
|
|
5669
|
-
);
|
|
5670
|
-
}
|
|
6075
|
+
async _consolidateWithHandler(handler, id) {
|
|
6076
|
+
const oldResource = this.config.resource;
|
|
6077
|
+
const oldField = this.config.field;
|
|
6078
|
+
const oldTransactionResource = this.transactionResource;
|
|
6079
|
+
const oldTargetResource = this.targetResource;
|
|
6080
|
+
const oldLockResource = this.lockResource;
|
|
6081
|
+
const oldAnalyticsResource = this.analyticsResource;
|
|
6082
|
+
this.config.resource = handler.resource;
|
|
6083
|
+
this.config.field = handler.field;
|
|
6084
|
+
this.transactionResource = handler.transactionResource;
|
|
6085
|
+
this.targetResource = handler.targetResource;
|
|
6086
|
+
this.lockResource = handler.lockResource;
|
|
6087
|
+
this.analyticsResource = handler.analyticsResource;
|
|
6088
|
+
const result = await this.consolidateRecord(id);
|
|
6089
|
+
this.config.resource = oldResource;
|
|
6090
|
+
this.config.field = oldField;
|
|
6091
|
+
this.transactionResource = oldTransactionResource;
|
|
6092
|
+
this.targetResource = oldTargetResource;
|
|
6093
|
+
this.lockResource = oldLockResource;
|
|
6094
|
+
this.analyticsResource = oldAnalyticsResource;
|
|
6095
|
+
return result;
|
|
5671
6096
|
}
|
|
5672
6097
|
/**
|
|
5673
|
-
*
|
|
6098
|
+
* Helper method for getConsolidatedValue with handler
|
|
5674
6099
|
* @private
|
|
5675
6100
|
*/
|
|
5676
|
-
|
|
5677
|
-
const
|
|
5678
|
-
|
|
5679
|
-
|
|
5680
|
-
|
|
5681
|
-
|
|
5682
|
-
|
|
5683
|
-
|
|
5684
|
-
|
|
5685
|
-
|
|
5686
|
-
|
|
5687
|
-
|
|
6101
|
+
async _getConsolidatedValueWithHandler(handler, id, options) {
|
|
6102
|
+
const oldResource = this.config.resource;
|
|
6103
|
+
const oldField = this.config.field;
|
|
6104
|
+
const oldTransactionResource = this.transactionResource;
|
|
6105
|
+
const oldTargetResource = this.targetResource;
|
|
6106
|
+
this.config.resource = handler.resource;
|
|
6107
|
+
this.config.field = handler.field;
|
|
6108
|
+
this.transactionResource = handler.transactionResource;
|
|
6109
|
+
this.targetResource = handler.targetResource;
|
|
6110
|
+
const result = await this.getConsolidatedValue(id, options);
|
|
6111
|
+
this.config.resource = oldResource;
|
|
6112
|
+
this.config.field = oldField;
|
|
6113
|
+
this.transactionResource = oldTransactionResource;
|
|
6114
|
+
this.targetResource = oldTargetResource;
|
|
6115
|
+
return result;
|
|
5688
6116
|
}
|
|
5689
6117
|
/**
|
|
5690
|
-
*
|
|
6118
|
+
* Helper method for recalculate with handler
|
|
5691
6119
|
* @private
|
|
5692
6120
|
*/
|
|
5693
|
-
async
|
|
5694
|
-
const
|
|
5695
|
-
const
|
|
5696
|
-
|
|
5697
|
-
|
|
6121
|
+
async _recalculateWithHandler(handler, id) {
|
|
6122
|
+
const oldResource = this.config.resource;
|
|
6123
|
+
const oldField = this.config.field;
|
|
6124
|
+
const oldTransactionResource = this.transactionResource;
|
|
6125
|
+
const oldTargetResource = this.targetResource;
|
|
6126
|
+
const oldLockResource = this.lockResource;
|
|
6127
|
+
const oldAnalyticsResource = this.analyticsResource;
|
|
6128
|
+
this.config.resource = handler.resource;
|
|
6129
|
+
this.config.field = handler.field;
|
|
6130
|
+
this.transactionResource = handler.transactionResource;
|
|
6131
|
+
this.targetResource = handler.targetResource;
|
|
6132
|
+
this.lockResource = handler.lockResource;
|
|
6133
|
+
this.analyticsResource = handler.analyticsResource;
|
|
6134
|
+
const result = await this.recalculateRecord(id);
|
|
6135
|
+
this.config.resource = oldResource;
|
|
6136
|
+
this.config.field = oldField;
|
|
6137
|
+
this.transactionResource = oldTransactionResource;
|
|
6138
|
+
this.targetResource = oldTargetResource;
|
|
6139
|
+
this.lockResource = oldLockResource;
|
|
6140
|
+
this.analyticsResource = oldAnalyticsResource;
|
|
6141
|
+
return result;
|
|
5698
6142
|
}
|
|
5699
6143
|
/**
|
|
5700
|
-
*
|
|
6144
|
+
* Run consolidation for a handler
|
|
5701
6145
|
* @private
|
|
5702
6146
|
*/
|
|
5703
|
-
async
|
|
5704
|
-
const
|
|
5705
|
-
const
|
|
5706
|
-
|
|
5707
|
-
|
|
5708
|
-
|
|
5709
|
-
const
|
|
5710
|
-
|
|
5711
|
-
|
|
5712
|
-
|
|
5713
|
-
|
|
5714
|
-
|
|
5715
|
-
|
|
5716
|
-
|
|
5717
|
-
|
|
5718
|
-
|
|
5719
|
-
|
|
5720
|
-
|
|
5721
|
-
|
|
5722
|
-
operations[op] = { count: 0, sum: 0 };
|
|
5723
|
-
}
|
|
5724
|
-
operations[op].count += stats.count;
|
|
5725
|
-
operations[op].sum += stats.sum;
|
|
5726
|
-
}
|
|
5727
|
-
}
|
|
5728
|
-
const recordCount = Math.max(...sourceAnalytics.map((a) => a.recordCount));
|
|
5729
|
-
const id = `${period}-${cohort}`;
|
|
5730
|
-
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
5731
|
-
const [existingOk, existingErr, existing] = await tryFn(
|
|
5732
|
-
() => this.analyticsResource.get(id)
|
|
5733
|
-
);
|
|
5734
|
-
if (existingOk && existing) {
|
|
5735
|
-
await tryFn(
|
|
5736
|
-
() => this.analyticsResource.update(id, {
|
|
5737
|
-
transactionCount,
|
|
5738
|
-
totalValue,
|
|
5739
|
-
avgValue,
|
|
5740
|
-
minValue,
|
|
5741
|
-
maxValue,
|
|
5742
|
-
operations,
|
|
5743
|
-
recordCount,
|
|
5744
|
-
updatedAt: now
|
|
5745
|
-
})
|
|
6147
|
+
async _runConsolidationForHandler(handler, resourceName, fieldName) {
|
|
6148
|
+
const oldResource = this.config.resource;
|
|
6149
|
+
const oldField = this.config.field;
|
|
6150
|
+
const oldTransactionResource = this.transactionResource;
|
|
6151
|
+
const oldTargetResource = this.targetResource;
|
|
6152
|
+
const oldLockResource = this.lockResource;
|
|
6153
|
+
const oldAnalyticsResource = this.analyticsResource;
|
|
6154
|
+
this.config.resource = resourceName;
|
|
6155
|
+
this.config.field = fieldName;
|
|
6156
|
+
this.transactionResource = handler.transactionResource;
|
|
6157
|
+
this.targetResource = handler.targetResource;
|
|
6158
|
+
this.lockResource = handler.lockResource;
|
|
6159
|
+
this.analyticsResource = handler.analyticsResource;
|
|
6160
|
+
try {
|
|
6161
|
+
await runConsolidation(
|
|
6162
|
+
this.transactionResource,
|
|
6163
|
+
(id) => this.consolidateRecord(id),
|
|
6164
|
+
(event, data) => this.emit(event, data),
|
|
6165
|
+
this.config
|
|
5746
6166
|
);
|
|
5747
|
-
}
|
|
5748
|
-
|
|
5749
|
-
|
|
5750
|
-
|
|
5751
|
-
|
|
5752
|
-
|
|
5753
|
-
|
|
5754
|
-
|
|
5755
|
-
|
|
5756
|
-
|
|
5757
|
-
|
|
5758
|
-
|
|
5759
|
-
|
|
5760
|
-
|
|
5761
|
-
|
|
5762
|
-
|
|
6167
|
+
} finally {
|
|
6168
|
+
this.config.resource = oldResource;
|
|
6169
|
+
this.config.field = oldField;
|
|
6170
|
+
this.transactionResource = oldTransactionResource;
|
|
6171
|
+
this.targetResource = oldTargetResource;
|
|
6172
|
+
this.lockResource = oldLockResource;
|
|
6173
|
+
this.analyticsResource = oldAnalyticsResource;
|
|
6174
|
+
}
|
|
6175
|
+
}
|
|
6176
|
+
/**
|
|
6177
|
+
* Run garbage collection for a handler
|
|
6178
|
+
* @private
|
|
6179
|
+
*/
|
|
6180
|
+
async _runGarbageCollectionForHandler(handler, resourceName, fieldName) {
|
|
6181
|
+
const oldResource = this.config.resource;
|
|
6182
|
+
const oldField = this.config.field;
|
|
6183
|
+
const oldTransactionResource = this.transactionResource;
|
|
6184
|
+
const oldTargetResource = this.targetResource;
|
|
6185
|
+
const oldLockResource = this.lockResource;
|
|
6186
|
+
this.config.resource = resourceName;
|
|
6187
|
+
this.config.field = fieldName;
|
|
6188
|
+
this.transactionResource = handler.transactionResource;
|
|
6189
|
+
this.targetResource = handler.targetResource;
|
|
6190
|
+
this.lockResource = handler.lockResource;
|
|
6191
|
+
try {
|
|
6192
|
+
await runGarbageCollection(
|
|
6193
|
+
this.transactionResource,
|
|
6194
|
+
this.lockResource,
|
|
6195
|
+
this.config,
|
|
6196
|
+
(event, data) => this.emit(event, data)
|
|
5763
6197
|
);
|
|
6198
|
+
} finally {
|
|
6199
|
+
this.config.resource = oldResource;
|
|
6200
|
+
this.config.field = oldField;
|
|
6201
|
+
this.transactionResource = oldTransactionResource;
|
|
6202
|
+
this.targetResource = oldTargetResource;
|
|
6203
|
+
this.lockResource = oldLockResource;
|
|
5764
6204
|
}
|
|
5765
6205
|
}
|
|
6206
|
+
// Public Analytics API
|
|
5766
6207
|
/**
|
|
5767
6208
|
* Get analytics for a specific period
|
|
5768
6209
|
* @param {string} resourceName - Resource name
|
|
@@ -5771,111 +6212,7 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
5771
6212
|
* @returns {Promise<Array>} Analytics data
|
|
5772
6213
|
*/
|
|
5773
6214
|
async getAnalytics(resourceName, field, options = {}) {
|
|
5774
|
-
|
|
5775
|
-
if (!fieldHandlers) {
|
|
5776
|
-
throw new Error(`No eventual consistency configured for resource: ${resourceName}`);
|
|
5777
|
-
}
|
|
5778
|
-
const handler = fieldHandlers.get(field);
|
|
5779
|
-
if (!handler) {
|
|
5780
|
-
throw new Error(`No eventual consistency configured for field: ${resourceName}.${field}`);
|
|
5781
|
-
}
|
|
5782
|
-
if (!handler.analyticsResource) {
|
|
5783
|
-
throw new Error("Analytics not enabled for this plugin");
|
|
5784
|
-
}
|
|
5785
|
-
const { period = "day", date, startDate, endDate, month, year, breakdown = false } = options;
|
|
5786
|
-
const [ok, err, allAnalytics] = await tryFn(
|
|
5787
|
-
() => handler.analyticsResource.list()
|
|
5788
|
-
);
|
|
5789
|
-
if (!ok || !allAnalytics) {
|
|
5790
|
-
return [];
|
|
5791
|
-
}
|
|
5792
|
-
let filtered = allAnalytics.filter((a) => a.period === period);
|
|
5793
|
-
if (date) {
|
|
5794
|
-
if (period === "hour") {
|
|
5795
|
-
filtered = filtered.filter((a) => a.cohort.startsWith(date));
|
|
5796
|
-
} else {
|
|
5797
|
-
filtered = filtered.filter((a) => a.cohort === date);
|
|
5798
|
-
}
|
|
5799
|
-
} else if (startDate && endDate) {
|
|
5800
|
-
filtered = filtered.filter((a) => a.cohort >= startDate && a.cohort <= endDate);
|
|
5801
|
-
} else if (month) {
|
|
5802
|
-
filtered = filtered.filter((a) => a.cohort.startsWith(month));
|
|
5803
|
-
} else if (year) {
|
|
5804
|
-
filtered = filtered.filter((a) => a.cohort.startsWith(String(year)));
|
|
5805
|
-
}
|
|
5806
|
-
filtered.sort((a, b) => a.cohort.localeCompare(b.cohort));
|
|
5807
|
-
if (breakdown === "operations") {
|
|
5808
|
-
return filtered.map((a) => ({
|
|
5809
|
-
cohort: a.cohort,
|
|
5810
|
-
...a.operations
|
|
5811
|
-
}));
|
|
5812
|
-
}
|
|
5813
|
-
return filtered.map((a) => ({
|
|
5814
|
-
cohort: a.cohort,
|
|
5815
|
-
count: a.transactionCount,
|
|
5816
|
-
sum: a.totalValue,
|
|
5817
|
-
avg: a.avgValue,
|
|
5818
|
-
min: a.minValue,
|
|
5819
|
-
max: a.maxValue,
|
|
5820
|
-
operations: a.operations,
|
|
5821
|
-
recordCount: a.recordCount
|
|
5822
|
-
}));
|
|
5823
|
-
}
|
|
5824
|
-
/**
|
|
5825
|
-
* Fill gaps in analytics data with zeros for continuous time series
|
|
5826
|
-
* @private
|
|
5827
|
-
* @param {Array} data - Sparse analytics data
|
|
5828
|
-
* @param {string} period - Period type ('hour', 'day', 'month')
|
|
5829
|
-
* @param {string} startDate - Start date (ISO format)
|
|
5830
|
-
* @param {string} endDate - End date (ISO format)
|
|
5831
|
-
* @returns {Array} Complete time series with gaps filled
|
|
5832
|
-
*/
|
|
5833
|
-
_fillGaps(data, period, startDate, endDate) {
|
|
5834
|
-
if (!data || data.length === 0) {
|
|
5835
|
-
data = [];
|
|
5836
|
-
}
|
|
5837
|
-
const dataMap = /* @__PURE__ */ new Map();
|
|
5838
|
-
data.forEach((item) => {
|
|
5839
|
-
dataMap.set(item.cohort, item);
|
|
5840
|
-
});
|
|
5841
|
-
const result = [];
|
|
5842
|
-
const emptyRecord = {
|
|
5843
|
-
count: 0,
|
|
5844
|
-
sum: 0,
|
|
5845
|
-
avg: 0,
|
|
5846
|
-
min: 0,
|
|
5847
|
-
max: 0,
|
|
5848
|
-
recordCount: 0
|
|
5849
|
-
};
|
|
5850
|
-
if (period === "hour") {
|
|
5851
|
-
const start = /* @__PURE__ */ new Date(startDate + "T00:00:00Z");
|
|
5852
|
-
const end = /* @__PURE__ */ new Date(endDate + "T23:59:59Z");
|
|
5853
|
-
for (let dt = new Date(start); dt <= end; dt.setHours(dt.getHours() + 1)) {
|
|
5854
|
-
const cohort = dt.toISOString().substring(0, 13);
|
|
5855
|
-
result.push(dataMap.get(cohort) || { cohort, ...emptyRecord });
|
|
5856
|
-
}
|
|
5857
|
-
} else if (period === "day") {
|
|
5858
|
-
const start = new Date(startDate);
|
|
5859
|
-
const end = new Date(endDate);
|
|
5860
|
-
for (let dt = new Date(start); dt <= end; dt.setDate(dt.getDate() + 1)) {
|
|
5861
|
-
const cohort = dt.toISOString().substring(0, 10);
|
|
5862
|
-
result.push(dataMap.get(cohort) || { cohort, ...emptyRecord });
|
|
5863
|
-
}
|
|
5864
|
-
} else if (period === "month") {
|
|
5865
|
-
const startYear = parseInt(startDate.substring(0, 4));
|
|
5866
|
-
const startMonth = parseInt(startDate.substring(5, 7));
|
|
5867
|
-
const endYear = parseInt(endDate.substring(0, 4));
|
|
5868
|
-
const endMonth = parseInt(endDate.substring(5, 7));
|
|
5869
|
-
for (let year = startYear; year <= endYear; year++) {
|
|
5870
|
-
const firstMonth = year === startYear ? startMonth : 1;
|
|
5871
|
-
const lastMonth = year === endYear ? endMonth : 12;
|
|
5872
|
-
for (let month = firstMonth; month <= lastMonth; month++) {
|
|
5873
|
-
const cohort = `${year}-${month.toString().padStart(2, "0")}`;
|
|
5874
|
-
result.push(dataMap.get(cohort) || { cohort, ...emptyRecord });
|
|
5875
|
-
}
|
|
5876
|
-
}
|
|
5877
|
-
}
|
|
5878
|
-
return result;
|
|
6215
|
+
return await getAnalytics(resourceName, field, options, this.fieldHandlers);
|
|
5879
6216
|
}
|
|
5880
6217
|
/**
|
|
5881
6218
|
* Get analytics for entire month, broken down by days
|
|
@@ -5883,25 +6220,10 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
5883
6220
|
* @param {string} field - Field name
|
|
5884
6221
|
* @param {string} month - Month in YYYY-MM format
|
|
5885
6222
|
* @param {Object} options - Options
|
|
5886
|
-
* @param {boolean} options.fillGaps - Fill missing days with zeros (default: false)
|
|
5887
6223
|
* @returns {Promise<Array>} Daily analytics for the month
|
|
5888
6224
|
*/
|
|
5889
6225
|
async getMonthByDay(resourceName, field, month, options = {}) {
|
|
5890
|
-
|
|
5891
|
-
const monthNum = parseInt(month.substring(5, 7));
|
|
5892
|
-
const firstDay = new Date(year, monthNum - 1, 1);
|
|
5893
|
-
const lastDay = new Date(year, monthNum, 0);
|
|
5894
|
-
const startDate = firstDay.toISOString().substring(0, 10);
|
|
5895
|
-
const endDate = lastDay.toISOString().substring(0, 10);
|
|
5896
|
-
const data = await this.getAnalytics(resourceName, field, {
|
|
5897
|
-
period: "day",
|
|
5898
|
-
startDate,
|
|
5899
|
-
endDate
|
|
5900
|
-
});
|
|
5901
|
-
if (options.fillGaps) {
|
|
5902
|
-
return this._fillGaps(data, "day", startDate, endDate);
|
|
5903
|
-
}
|
|
5904
|
-
return data;
|
|
6226
|
+
return await getMonthByDay(resourceName, field, month, options, this.fieldHandlers);
|
|
5905
6227
|
}
|
|
5906
6228
|
/**
|
|
5907
6229
|
* Get analytics for entire day, broken down by hours
|
|
@@ -5909,18 +6231,10 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
5909
6231
|
* @param {string} field - Field name
|
|
5910
6232
|
* @param {string} date - Date in YYYY-MM-DD format
|
|
5911
6233
|
* @param {Object} options - Options
|
|
5912
|
-
* @param {boolean} options.fillGaps - Fill missing hours with zeros (default: false)
|
|
5913
6234
|
* @returns {Promise<Array>} Hourly analytics for the day
|
|
5914
6235
|
*/
|
|
5915
6236
|
async getDayByHour(resourceName, field, date, options = {}) {
|
|
5916
|
-
|
|
5917
|
-
period: "hour",
|
|
5918
|
-
date
|
|
5919
|
-
});
|
|
5920
|
-
if (options.fillGaps) {
|
|
5921
|
-
return this._fillGaps(data, "hour", date, date);
|
|
5922
|
-
}
|
|
5923
|
-
return data;
|
|
6237
|
+
return await getDayByHour(resourceName, field, date, options, this.fieldHandlers);
|
|
5924
6238
|
}
|
|
5925
6239
|
/**
|
|
5926
6240
|
* Get analytics for last N days, broken down by days
|
|
@@ -5928,24 +6242,10 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
5928
6242
|
* @param {string} field - Field name
|
|
5929
6243
|
* @param {number} days - Number of days to look back (default: 7)
|
|
5930
6244
|
* @param {Object} options - Options
|
|
5931
|
-
* @param {boolean} options.fillGaps - Fill missing days with zeros (default: false)
|
|
5932
6245
|
* @returns {Promise<Array>} Daily analytics
|
|
5933
6246
|
*/
|
|
5934
6247
|
async getLastNDays(resourceName, field, days = 7, options = {}) {
|
|
5935
|
-
|
|
5936
|
-
const date = /* @__PURE__ */ new Date();
|
|
5937
|
-
date.setDate(date.getDate() - i);
|
|
5938
|
-
return date.toISOString().substring(0, 10);
|
|
5939
|
-
}).reverse();
|
|
5940
|
-
const data = await this.getAnalytics(resourceName, field, {
|
|
5941
|
-
period: "day",
|
|
5942
|
-
startDate: dates[0],
|
|
5943
|
-
endDate: dates[dates.length - 1]
|
|
5944
|
-
});
|
|
5945
|
-
if (options.fillGaps) {
|
|
5946
|
-
return this._fillGaps(data, "day", dates[0], dates[dates.length - 1]);
|
|
5947
|
-
}
|
|
5948
|
-
return data;
|
|
6248
|
+
return await getLastNDays(resourceName, field, days, options, this.fieldHandlers);
|
|
5949
6249
|
}
|
|
5950
6250
|
/**
|
|
5951
6251
|
* Get analytics for entire year, broken down by months
|
|
@@ -5953,20 +6253,10 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
5953
6253
|
* @param {string} field - Field name
|
|
5954
6254
|
* @param {number} year - Year (e.g., 2025)
|
|
5955
6255
|
* @param {Object} options - Options
|
|
5956
|
-
* @param {boolean} options.fillGaps - Fill missing months with zeros (default: false)
|
|
5957
6256
|
* @returns {Promise<Array>} Monthly analytics for the year
|
|
5958
6257
|
*/
|
|
5959
6258
|
async getYearByMonth(resourceName, field, year, options = {}) {
|
|
5960
|
-
|
|
5961
|
-
period: "month",
|
|
5962
|
-
year
|
|
5963
|
-
});
|
|
5964
|
-
if (options.fillGaps) {
|
|
5965
|
-
const startDate = `${year}-01`;
|
|
5966
|
-
const endDate = `${year}-12`;
|
|
5967
|
-
return this._fillGaps(data, "month", startDate, endDate);
|
|
5968
|
-
}
|
|
5969
|
-
return data;
|
|
6259
|
+
return await getYearByMonth(resourceName, field, year, options, this.fieldHandlers);
|
|
5970
6260
|
}
|
|
5971
6261
|
/**
|
|
5972
6262
|
* Get analytics for entire month, broken down by hours
|
|
@@ -5974,33 +6264,10 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
5974
6264
|
* @param {string} field - Field name
|
|
5975
6265
|
* @param {string} month - Month in YYYY-MM format (or 'last' for previous month)
|
|
5976
6266
|
* @param {Object} options - Options
|
|
5977
|
-
* @
|
|
5978
|
-
* @returns {Promise<Array>} Hourly analytics for the month (up to 24*31=744 records)
|
|
6267
|
+
* @returns {Promise<Array>} Hourly analytics for the month
|
|
5979
6268
|
*/
|
|
5980
6269
|
async getMonthByHour(resourceName, field, month, options = {}) {
|
|
5981
|
-
|
|
5982
|
-
if (month === "last") {
|
|
5983
|
-
const now = /* @__PURE__ */ new Date();
|
|
5984
|
-
now.setMonth(now.getMonth() - 1);
|
|
5985
|
-
year = now.getFullYear();
|
|
5986
|
-
monthNum = now.getMonth() + 1;
|
|
5987
|
-
} else {
|
|
5988
|
-
year = parseInt(month.substring(0, 4));
|
|
5989
|
-
monthNum = parseInt(month.substring(5, 7));
|
|
5990
|
-
}
|
|
5991
|
-
const firstDay = new Date(year, monthNum - 1, 1);
|
|
5992
|
-
const lastDay = new Date(year, monthNum, 0);
|
|
5993
|
-
const startDate = firstDay.toISOString().substring(0, 10);
|
|
5994
|
-
const endDate = lastDay.toISOString().substring(0, 10);
|
|
5995
|
-
const data = await this.getAnalytics(resourceName, field, {
|
|
5996
|
-
period: "hour",
|
|
5997
|
-
startDate,
|
|
5998
|
-
endDate
|
|
5999
|
-
});
|
|
6000
|
-
if (options.fillGaps) {
|
|
6001
|
-
return this._fillGaps(data, "hour", startDate, endDate);
|
|
6002
|
-
}
|
|
6003
|
-
return data;
|
|
6270
|
+
return await getMonthByHour(resourceName, field, month, options, this.fieldHandlers);
|
|
6004
6271
|
}
|
|
6005
6272
|
/**
|
|
6006
6273
|
* Get top records by volume
|
|
@@ -6010,57 +6277,7 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
6010
6277
|
* @returns {Promise<Array>} Top records
|
|
6011
6278
|
*/
|
|
6012
6279
|
async getTopRecords(resourceName, field, options = {}) {
|
|
6013
|
-
|
|
6014
|
-
if (!fieldHandlers) {
|
|
6015
|
-
throw new Error(`No eventual consistency configured for resource: ${resourceName}`);
|
|
6016
|
-
}
|
|
6017
|
-
const handler = fieldHandlers.get(field);
|
|
6018
|
-
if (!handler) {
|
|
6019
|
-
throw new Error(`No eventual consistency configured for field: ${resourceName}.${field}`);
|
|
6020
|
-
}
|
|
6021
|
-
if (!handler.transactionResource) {
|
|
6022
|
-
throw new Error("Transaction resource not initialized");
|
|
6023
|
-
}
|
|
6024
|
-
const { period = "day", date, metric = "transactionCount", limit = 10 } = options;
|
|
6025
|
-
const [ok, err, transactions] = await tryFn(
|
|
6026
|
-
() => handler.transactionResource.list()
|
|
6027
|
-
);
|
|
6028
|
-
if (!ok || !transactions) {
|
|
6029
|
-
return [];
|
|
6030
|
-
}
|
|
6031
|
-
let filtered = transactions;
|
|
6032
|
-
if (date) {
|
|
6033
|
-
if (period === "hour") {
|
|
6034
|
-
filtered = transactions.filter((t) => t.cohortHour && t.cohortHour.startsWith(date));
|
|
6035
|
-
} else if (period === "day") {
|
|
6036
|
-
filtered = transactions.filter((t) => t.cohortDate === date);
|
|
6037
|
-
} else if (period === "month") {
|
|
6038
|
-
filtered = transactions.filter((t) => t.cohortMonth && t.cohortMonth.startsWith(date));
|
|
6039
|
-
}
|
|
6040
|
-
}
|
|
6041
|
-
const byRecord = {};
|
|
6042
|
-
for (const txn of filtered) {
|
|
6043
|
-
const recordId = txn.originalId;
|
|
6044
|
-
if (!byRecord[recordId]) {
|
|
6045
|
-
byRecord[recordId] = { count: 0, sum: 0 };
|
|
6046
|
-
}
|
|
6047
|
-
byRecord[recordId].count++;
|
|
6048
|
-
byRecord[recordId].sum += txn.value;
|
|
6049
|
-
}
|
|
6050
|
-
const records = Object.entries(byRecord).map(([recordId, stats]) => ({
|
|
6051
|
-
recordId,
|
|
6052
|
-
count: stats.count,
|
|
6053
|
-
sum: stats.sum
|
|
6054
|
-
}));
|
|
6055
|
-
records.sort((a, b) => {
|
|
6056
|
-
if (metric === "transactionCount") {
|
|
6057
|
-
return b.count - a.count;
|
|
6058
|
-
} else if (metric === "totalValue") {
|
|
6059
|
-
return b.sum - a.sum;
|
|
6060
|
-
}
|
|
6061
|
-
return 0;
|
|
6062
|
-
});
|
|
6063
|
-
return records.slice(0, limit);
|
|
6280
|
+
return await getTopRecords(resourceName, field, options, this.fieldHandlers);
|
|
6064
6281
|
}
|
|
6065
6282
|
}
|
|
6066
6283
|
|
|
@@ -12131,7 +12348,7 @@ class Database extends EventEmitter {
|
|
|
12131
12348
|
this.id = idGenerator(7);
|
|
12132
12349
|
this.version = "1";
|
|
12133
12350
|
this.s3dbVersion = (() => {
|
|
12134
|
-
const [ok, err, version] = tryFn(() => true ? "10.0.
|
|
12351
|
+
const [ok, err, version] = tryFn(() => true ? "10.0.18" : "latest");
|
|
12135
12352
|
return ok ? version : "latest";
|
|
12136
12353
|
})();
|
|
12137
12354
|
this.resources = {};
|