s3db.js 10.0.15 → 10.0.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/s3db.cjs.js +1758 -1574
- package/dist/s3db.cjs.js.map +1 -1
- package/dist/s3db.es.js +1758 -1574
- package/dist/s3db.es.js.map +1 -1
- package/package.json +1 -1
- package/src/plugins/eventual-consistency/analytics.js +668 -0
- package/src/plugins/eventual-consistency/config.js +120 -0
- package/src/plugins/eventual-consistency/consolidation.js +770 -0
- package/src/plugins/eventual-consistency/garbage-collection.js +126 -0
- package/src/plugins/eventual-consistency/helpers.js +179 -0
- package/src/plugins/eventual-consistency/index.js +455 -0
- package/src/plugins/eventual-consistency/locks.js +77 -0
- package/src/plugins/eventual-consistency/partitions.js +45 -0
- package/src/plugins/eventual-consistency/setup.js +298 -0
- package/src/plugins/eventual-consistency/transactions.js +119 -0
- package/src/plugins/eventual-consistency/utils.js +182 -0
- package/src/plugins/eventual-consistency.plugin.js +216 -52
- package/src/plugins/index.js +1 -1
package/dist/s3db.cjs.js
CHANGED
|
@@ -4307,1491 +4307,1899 @@ const CostsPlugin = {
|
|
|
4307
4307
|
}
|
|
4308
4308
|
};
|
|
4309
4309
|
|
|
4310
|
-
|
|
4311
|
-
|
|
4312
|
-
|
|
4313
|
-
|
|
4310
|
+
function createConfig(options, detectedTimezone) {
|
|
4311
|
+
return {
|
|
4312
|
+
cohort: {
|
|
4313
|
+
timezone: options.cohort?.timezone || detectedTimezone
|
|
4314
|
+
},
|
|
4315
|
+
reducer: options.reducer || ((transactions) => {
|
|
4316
|
+
let baseValue = 0;
|
|
4317
|
+
for (const t of transactions) {
|
|
4318
|
+
if (t.operation === "set") {
|
|
4319
|
+
baseValue = t.value;
|
|
4320
|
+
} else if (t.operation === "add") {
|
|
4321
|
+
baseValue += t.value;
|
|
4322
|
+
} else if (t.operation === "sub") {
|
|
4323
|
+
baseValue -= t.value;
|
|
4324
|
+
}
|
|
4325
|
+
}
|
|
4326
|
+
return baseValue;
|
|
4327
|
+
}),
|
|
4328
|
+
consolidationInterval: options.consolidationInterval ?? 300,
|
|
4329
|
+
consolidationConcurrency: options.consolidationConcurrency || 5,
|
|
4330
|
+
consolidationWindow: options.consolidationWindow || 24,
|
|
4331
|
+
autoConsolidate: options.autoConsolidate !== false,
|
|
4332
|
+
lateArrivalStrategy: options.lateArrivalStrategy || "warn",
|
|
4333
|
+
batchTransactions: options.batchTransactions || false,
|
|
4334
|
+
batchSize: options.batchSize || 100,
|
|
4335
|
+
mode: options.mode || "async",
|
|
4336
|
+
lockTimeout: options.lockTimeout || 300,
|
|
4337
|
+
transactionRetention: options.transactionRetention || 30,
|
|
4338
|
+
gcInterval: options.gcInterval || 86400,
|
|
4339
|
+
verbose: options.verbose || false,
|
|
4340
|
+
enableAnalytics: options.enableAnalytics || false,
|
|
4341
|
+
analyticsConfig: {
|
|
4342
|
+
periods: options.analyticsConfig?.periods || ["hour", "day", "month"],
|
|
4343
|
+
metrics: options.analyticsConfig?.metrics || ["count", "sum", "avg", "min", "max"],
|
|
4344
|
+
rollupStrategy: options.analyticsConfig?.rollupStrategy || "incremental",
|
|
4345
|
+
retentionDays: options.analyticsConfig?.retentionDays || 365
|
|
4346
|
+
},
|
|
4347
|
+
// Checkpoint configuration for high-volume scenarios
|
|
4348
|
+
enableCheckpoints: options.enableCheckpoints !== false,
|
|
4349
|
+
// Default: true
|
|
4350
|
+
checkpointStrategy: options.checkpointStrategy || "hourly",
|
|
4351
|
+
// 'hourly', 'daily', 'manual', 'disabled'
|
|
4352
|
+
checkpointRetention: options.checkpointRetention || 90,
|
|
4353
|
+
// Days to keep checkpoints
|
|
4354
|
+
checkpointThreshold: options.checkpointThreshold || 1e3,
|
|
4355
|
+
// Min transactions before creating checkpoint
|
|
4356
|
+
deleteConsolidatedTransactions: options.deleteConsolidatedTransactions !== false,
|
|
4357
|
+
// Delete transactions after checkpoint
|
|
4358
|
+
autoCheckpoint: options.autoCheckpoint !== false
|
|
4359
|
+
// Auto-create checkpoints for old cohorts
|
|
4360
|
+
};
|
|
4361
|
+
}
|
|
4362
|
+
function validateResourcesConfig(resources) {
|
|
4363
|
+
if (!resources || typeof resources !== "object") {
|
|
4364
|
+
throw new Error(
|
|
4365
|
+
"EventualConsistencyPlugin requires 'resources' option.\nExample: { resources: { urls: ['clicks', 'views'], posts: ['likes'] } }"
|
|
4366
|
+
);
|
|
4367
|
+
}
|
|
4368
|
+
for (const [resourceName, fields] of Object.entries(resources)) {
|
|
4369
|
+
if (!Array.isArray(fields)) {
|
|
4314
4370
|
throw new Error(
|
|
4315
|
-
|
|
4371
|
+
`EventualConsistencyPlugin resources.${resourceName} must be an array of field names`
|
|
4316
4372
|
);
|
|
4317
4373
|
}
|
|
4318
|
-
|
|
4319
|
-
|
|
4320
|
-
|
|
4321
|
-
|
|
4322
|
-
|
|
4323
|
-
|
|
4324
|
-
|
|
4325
|
-
|
|
4326
|
-
|
|
4327
|
-
|
|
4328
|
-
|
|
4329
|
-
|
|
4330
|
-
|
|
4331
|
-
|
|
4332
|
-
|
|
4333
|
-
|
|
4334
|
-
|
|
4335
|
-
})
|
|
4336
|
-
|
|
4337
|
-
|
|
4338
|
-
|
|
4339
|
-
|
|
4340
|
-
|
|
4341
|
-
|
|
4342
|
-
|
|
4343
|
-
|
|
4344
|
-
|
|
4345
|
-
|
|
4346
|
-
|
|
4347
|
-
|
|
4348
|
-
|
|
4349
|
-
|
|
4350
|
-
|
|
4351
|
-
|
|
4352
|
-
|
|
4353
|
-
|
|
4354
|
-
|
|
4374
|
+
}
|
|
4375
|
+
}
|
|
4376
|
+
function logConfigWarnings(config) {
|
|
4377
|
+
if (config.batchTransactions && !config.verbose) {
|
|
4378
|
+
console.warn(
|
|
4379
|
+
`[EventualConsistency] WARNING: batchTransactions is enabled. This stores transactions in memory and will lose data if container crashes. Not recommended for distributed/production environments.`
|
|
4380
|
+
);
|
|
4381
|
+
}
|
|
4382
|
+
}
|
|
4383
|
+
function logInitialization(config, fieldHandlers, timezoneAutoDetected) {
|
|
4384
|
+
if (!config.verbose) return;
|
|
4385
|
+
const totalFields = Array.from(fieldHandlers.values()).reduce((sum, handlers) => sum + handlers.size, 0);
|
|
4386
|
+
console.log(
|
|
4387
|
+
`[EventualConsistency] Initialized with ${fieldHandlers.size} resource(s), ${totalFields} field(s) total`
|
|
4388
|
+
);
|
|
4389
|
+
if (timezoneAutoDetected) {
|
|
4390
|
+
console.log(
|
|
4391
|
+
`[EventualConsistency] Auto-detected timezone: ${config.cohort.timezone} (from ${process.env.TZ ? "TZ env var" : "system Intl API"})`
|
|
4392
|
+
);
|
|
4393
|
+
}
|
|
4394
|
+
}
|
|
4395
|
+
|
|
4396
|
+
function detectTimezone() {
|
|
4397
|
+
if (process.env.TZ) {
|
|
4398
|
+
return process.env.TZ;
|
|
4399
|
+
}
|
|
4400
|
+
try {
|
|
4401
|
+
const systemTimezone = Intl.DateTimeFormat().resolvedOptions().timeZone;
|
|
4402
|
+
if (systemTimezone) {
|
|
4403
|
+
return systemTimezone;
|
|
4404
|
+
}
|
|
4405
|
+
} catch (err) {
|
|
4406
|
+
}
|
|
4407
|
+
return "UTC";
|
|
4408
|
+
}
|
|
4409
|
+
function getTimezoneOffset(timezone, verbose = false) {
|
|
4410
|
+
try {
|
|
4411
|
+
const now = /* @__PURE__ */ new Date();
|
|
4412
|
+
const utcDate = new Date(now.toLocaleString("en-US", { timeZone: "UTC" }));
|
|
4413
|
+
const tzDate = new Date(now.toLocaleString("en-US", { timeZone: timezone }));
|
|
4414
|
+
return tzDate.getTime() - utcDate.getTime();
|
|
4415
|
+
} catch (err) {
|
|
4416
|
+
const offsets = {
|
|
4417
|
+
"UTC": 0,
|
|
4418
|
+
"America/New_York": -5 * 36e5,
|
|
4419
|
+
"America/Chicago": -6 * 36e5,
|
|
4420
|
+
"America/Denver": -7 * 36e5,
|
|
4421
|
+
"America/Los_Angeles": -8 * 36e5,
|
|
4422
|
+
"America/Sao_Paulo": -3 * 36e5,
|
|
4423
|
+
"Europe/London": 0,
|
|
4424
|
+
"Europe/Paris": 1 * 36e5,
|
|
4425
|
+
"Europe/Berlin": 1 * 36e5,
|
|
4426
|
+
"Asia/Tokyo": 9 * 36e5,
|
|
4427
|
+
"Asia/Shanghai": 8 * 36e5,
|
|
4428
|
+
"Australia/Sydney": 10 * 36e5
|
|
4355
4429
|
};
|
|
4356
|
-
|
|
4357
|
-
|
|
4358
|
-
|
|
4359
|
-
|
|
4360
|
-
|
|
4361
|
-
|
|
4430
|
+
if (verbose && !offsets[timezone]) {
|
|
4431
|
+
console.warn(
|
|
4432
|
+
`[EventualConsistency] Unknown timezone '${timezone}', using UTC. Consider using a valid IANA timezone (e.g., 'America/New_York')`
|
|
4433
|
+
);
|
|
4434
|
+
}
|
|
4435
|
+
return offsets[timezone] || 0;
|
|
4436
|
+
}
|
|
4437
|
+
}
|
|
4438
|
+
function getCohortInfo(date, timezone, verbose = false) {
|
|
4439
|
+
const offset = getTimezoneOffset(timezone, verbose);
|
|
4440
|
+
const localDate = new Date(date.getTime() + offset);
|
|
4441
|
+
const year = localDate.getFullYear();
|
|
4442
|
+
const month = String(localDate.getMonth() + 1).padStart(2, "0");
|
|
4443
|
+
const day = String(localDate.getDate()).padStart(2, "0");
|
|
4444
|
+
const hour = String(localDate.getHours()).padStart(2, "0");
|
|
4445
|
+
return {
|
|
4446
|
+
date: `${year}-${month}-${day}`,
|
|
4447
|
+
hour: `${year}-${month}-${day}T${hour}`,
|
|
4448
|
+
// ISO-like format for hour partition
|
|
4449
|
+
month: `${year}-${month}`
|
|
4450
|
+
};
|
|
4451
|
+
}
|
|
4452
|
+
function createSyntheticSetTransaction(currentValue) {
|
|
4453
|
+
return {
|
|
4454
|
+
id: "__synthetic__",
|
|
4455
|
+
operation: "set",
|
|
4456
|
+
value: currentValue,
|
|
4457
|
+
timestamp: (/* @__PURE__ */ new Date(0)).toISOString(),
|
|
4458
|
+
synthetic: true
|
|
4459
|
+
};
|
|
4460
|
+
}
|
|
4461
|
+
function createFieldHandler(resourceName, fieldName) {
|
|
4462
|
+
return {
|
|
4463
|
+
resource: resourceName,
|
|
4464
|
+
field: fieldName,
|
|
4465
|
+
transactionResource: null,
|
|
4466
|
+
targetResource: null,
|
|
4467
|
+
analyticsResource: null,
|
|
4468
|
+
lockResource: null,
|
|
4469
|
+
checkpointResource: null,
|
|
4470
|
+
consolidationTimer: null,
|
|
4471
|
+
gcTimer: null,
|
|
4472
|
+
pendingTransactions: /* @__PURE__ */ new Map(),
|
|
4473
|
+
deferredSetup: false
|
|
4474
|
+
};
|
|
4475
|
+
}
|
|
4476
|
+
function resolveFieldAndPlugin(resource, field, value) {
|
|
4477
|
+
if (!resource._eventualConsistencyPlugins) {
|
|
4478
|
+
throw new Error(`No eventual consistency plugins configured for this resource`);
|
|
4479
|
+
}
|
|
4480
|
+
const fieldPlugin = resource._eventualConsistencyPlugins[field];
|
|
4481
|
+
if (!fieldPlugin) {
|
|
4482
|
+
const availableFields = Object.keys(resource._eventualConsistencyPlugins).join(", ");
|
|
4483
|
+
throw new Error(
|
|
4484
|
+
`No eventual consistency plugin found for field "${field}". Available fields: ${availableFields}`
|
|
4485
|
+
);
|
|
4486
|
+
}
|
|
4487
|
+
return { field, value, plugin: fieldPlugin };
|
|
4488
|
+
}
|
|
4489
|
+
function groupByCohort(transactions, cohortField) {
|
|
4490
|
+
const groups = {};
|
|
4491
|
+
for (const txn of transactions) {
|
|
4492
|
+
const cohort = txn[cohortField];
|
|
4493
|
+
if (!cohort) continue;
|
|
4494
|
+
if (!groups[cohort]) {
|
|
4495
|
+
groups[cohort] = [];
|
|
4496
|
+
}
|
|
4497
|
+
groups[cohort].push(txn);
|
|
4498
|
+
}
|
|
4499
|
+
return groups;
|
|
4500
|
+
}
|
|
4501
|
+
|
|
4502
|
+
function createPartitionConfig() {
|
|
4503
|
+
const partitions = {
|
|
4504
|
+
// Composite partition by originalId + applied status
|
|
4505
|
+
// This is THE MOST CRITICAL optimization for consolidation!
|
|
4506
|
+
// Why: Consolidation always queries { originalId, applied: false }
|
|
4507
|
+
// Without this: Reads ALL transactions (applied + pending) and filters manually
|
|
4508
|
+
// With this: Reads ONLY pending transactions - can be 1000x faster!
|
|
4509
|
+
byOriginalIdAndApplied: {
|
|
4510
|
+
fields: {
|
|
4511
|
+
originalId: "string",
|
|
4512
|
+
applied: "boolean"
|
|
4362
4513
|
}
|
|
4363
|
-
|
|
4364
|
-
|
|
4365
|
-
|
|
4514
|
+
},
|
|
4515
|
+
// Partition by time cohorts for batch consolidation across many records
|
|
4516
|
+
byHour: {
|
|
4517
|
+
fields: {
|
|
4518
|
+
cohortHour: "string"
|
|
4519
|
+
}
|
|
4520
|
+
},
|
|
4521
|
+
byDay: {
|
|
4522
|
+
fields: {
|
|
4523
|
+
cohortDate: "string"
|
|
4524
|
+
}
|
|
4525
|
+
},
|
|
4526
|
+
byMonth: {
|
|
4527
|
+
fields: {
|
|
4528
|
+
cohortMonth: "string"
|
|
4366
4529
|
}
|
|
4367
|
-
this.fieldHandlers.set(resourceName, resourceHandlers);
|
|
4368
4530
|
}
|
|
4369
|
-
|
|
4531
|
+
};
|
|
4532
|
+
return partitions;
|
|
4533
|
+
}
|
|
4534
|
+
|
|
4535
|
+
async function createTransaction(handler, data, config) {
|
|
4536
|
+
const now = /* @__PURE__ */ new Date();
|
|
4537
|
+
const cohortInfo = getCohortInfo(now, config.cohort.timezone, config.verbose);
|
|
4538
|
+
const watermarkMs = config.consolidationWindow * 60 * 60 * 1e3;
|
|
4539
|
+
const watermarkTime = now.getTime() - watermarkMs;
|
|
4540
|
+
const cohortHourDate = /* @__PURE__ */ new Date(cohortInfo.hour + ":00:00Z");
|
|
4541
|
+
if (cohortHourDate.getTime() < watermarkTime) {
|
|
4542
|
+
const hoursLate = Math.floor((now.getTime() - cohortHourDate.getTime()) / (60 * 60 * 1e3));
|
|
4543
|
+
if (config.lateArrivalStrategy === "ignore") {
|
|
4544
|
+
if (config.verbose) {
|
|
4545
|
+
console.warn(
|
|
4546
|
+
`[EventualConsistency] Late arrival ignored: transaction for ${cohortInfo.hour} is ${hoursLate}h late (watermark: ${config.consolidationWindow}h)`
|
|
4547
|
+
);
|
|
4548
|
+
}
|
|
4549
|
+
return null;
|
|
4550
|
+
} else if (config.lateArrivalStrategy === "warn") {
|
|
4370
4551
|
console.warn(
|
|
4371
|
-
`[EventualConsistency]
|
|
4552
|
+
`[EventualConsistency] Late arrival detected: transaction for ${cohortInfo.hour} is ${hoursLate}h late (watermark: ${config.consolidationWindow}h). Processing anyway, but consolidation may not pick it up.`
|
|
4372
4553
|
);
|
|
4373
4554
|
}
|
|
4374
|
-
|
|
4375
|
-
|
|
4555
|
+
}
|
|
4556
|
+
const transaction = {
|
|
4557
|
+
id: idGenerator(),
|
|
4558
|
+
originalId: data.originalId,
|
|
4559
|
+
field: handler.field,
|
|
4560
|
+
value: data.value || 0,
|
|
4561
|
+
operation: data.operation || "set",
|
|
4562
|
+
timestamp: now.toISOString(),
|
|
4563
|
+
cohortDate: cohortInfo.date,
|
|
4564
|
+
cohortHour: cohortInfo.hour,
|
|
4565
|
+
cohortMonth: cohortInfo.month,
|
|
4566
|
+
source: data.source || "unknown",
|
|
4567
|
+
applied: false
|
|
4568
|
+
};
|
|
4569
|
+
if (config.batchTransactions) {
|
|
4570
|
+
handler.pendingTransactions.set(transaction.id, transaction);
|
|
4571
|
+
if (config.verbose) {
|
|
4376
4572
|
console.log(
|
|
4377
|
-
`[EventualConsistency]
|
|
4573
|
+
`[EventualConsistency] ${handler.resource}.${handler.field} - Transaction batched: ${data.operation} ${data.value} for ${data.originalId} (batch: ${handler.pendingTransactions.size}/${config.batchSize})`
|
|
4574
|
+
);
|
|
4575
|
+
}
|
|
4576
|
+
if (handler.pendingTransactions.size >= config.batchSize) {
|
|
4577
|
+
await flushPendingTransactions(handler);
|
|
4578
|
+
}
|
|
4579
|
+
} else {
|
|
4580
|
+
await handler.transactionResource.insert(transaction);
|
|
4581
|
+
if (config.verbose) {
|
|
4582
|
+
console.log(
|
|
4583
|
+
`[EventualConsistency] ${handler.resource}.${handler.field} - Transaction created: ${data.operation} ${data.value} for ${data.originalId} (cohort: ${cohortInfo.hour}, applied: false)`
|
|
4378
4584
|
);
|
|
4379
|
-
if (!options.cohort?.timezone) {
|
|
4380
|
-
console.log(
|
|
4381
|
-
`[EventualConsistency] Auto-detected timezone: ${this.config.cohort.timezone} (from ${process.env.TZ ? "TZ env var" : "system Intl API"})`
|
|
4382
|
-
);
|
|
4383
|
-
}
|
|
4384
4585
|
}
|
|
4385
4586
|
}
|
|
4386
|
-
|
|
4387
|
-
|
|
4388
|
-
|
|
4389
|
-
|
|
4390
|
-
|
|
4391
|
-
|
|
4392
|
-
|
|
4393
|
-
|
|
4394
|
-
|
|
4395
|
-
|
|
4396
|
-
|
|
4397
|
-
|
|
4398
|
-
|
|
4399
|
-
|
|
4400
|
-
|
|
4401
|
-
deferredSetup: false
|
|
4402
|
-
};
|
|
4587
|
+
return transaction;
|
|
4588
|
+
}
|
|
4589
|
+
async function flushPendingTransactions(handler) {
|
|
4590
|
+
if (handler.pendingTransactions.size === 0) return;
|
|
4591
|
+
const transactions = Array.from(handler.pendingTransactions.values());
|
|
4592
|
+
try {
|
|
4593
|
+
await Promise.all(
|
|
4594
|
+
transactions.map(
|
|
4595
|
+
(transaction) => handler.transactionResource.insert(transaction)
|
|
4596
|
+
)
|
|
4597
|
+
);
|
|
4598
|
+
handler.pendingTransactions.clear();
|
|
4599
|
+
} catch (error) {
|
|
4600
|
+
console.error("Failed to flush pending transactions:", error);
|
|
4601
|
+
throw error;
|
|
4403
4602
|
}
|
|
4404
|
-
|
|
4405
|
-
|
|
4406
|
-
|
|
4407
|
-
|
|
4408
|
-
|
|
4409
|
-
|
|
4410
|
-
|
|
4411
|
-
|
|
4412
|
-
|
|
4413
|
-
|
|
4414
|
-
|
|
4415
|
-
|
|
4416
|
-
|
|
4417
|
-
|
|
4603
|
+
}
|
|
4604
|
+
|
|
4605
|
+
async function cleanupStaleLocks(lockResource, config) {
|
|
4606
|
+
const now = Date.now();
|
|
4607
|
+
const lockTimeoutMs = config.lockTimeout * 1e3;
|
|
4608
|
+
const cutoffTime = now - lockTimeoutMs;
|
|
4609
|
+
const cleanupLockId = `lock-cleanup-${config.resource}-${config.field}`;
|
|
4610
|
+
const [lockAcquired] = await tryFn(
|
|
4611
|
+
() => lockResource.insert({
|
|
4612
|
+
id: cleanupLockId,
|
|
4613
|
+
lockedAt: Date.now(),
|
|
4614
|
+
workerId: process.pid ? String(process.pid) : "unknown"
|
|
4615
|
+
})
|
|
4616
|
+
);
|
|
4617
|
+
if (!lockAcquired) {
|
|
4618
|
+
if (config.verbose) {
|
|
4619
|
+
console.log(`[EventualConsistency] Lock cleanup already running in another container`);
|
|
4418
4620
|
}
|
|
4621
|
+
return;
|
|
4419
4622
|
}
|
|
4420
|
-
|
|
4421
|
-
|
|
4422
|
-
|
|
4423
|
-
|
|
4424
|
-
|
|
4425
|
-
|
|
4426
|
-
|
|
4427
|
-
|
|
4428
|
-
|
|
4429
|
-
|
|
4430
|
-
|
|
4431
|
-
|
|
4432
|
-
|
|
4433
|
-
|
|
4434
|
-
|
|
4435
|
-
|
|
4436
|
-
|
|
4437
|
-
|
|
4438
|
-
|
|
4623
|
+
try {
|
|
4624
|
+
const [ok, err, locks] = await tryFn(() => lockResource.list());
|
|
4625
|
+
if (!ok || !locks || locks.length === 0) return;
|
|
4626
|
+
const staleLocks = locks.filter(
|
|
4627
|
+
(lock) => lock.id !== cleanupLockId && lock.lockedAt < cutoffTime
|
|
4628
|
+
);
|
|
4629
|
+
if (staleLocks.length === 0) return;
|
|
4630
|
+
if (config.verbose) {
|
|
4631
|
+
console.log(`[EventualConsistency] Cleaning up ${staleLocks.length} stale locks`);
|
|
4632
|
+
}
|
|
4633
|
+
const { results, errors } = await promisePool.PromisePool.for(staleLocks).withConcurrency(5).process(async (lock) => {
|
|
4634
|
+
const [deleted] = await tryFn(() => lockResource.delete(lock.id));
|
|
4635
|
+
return deleted;
|
|
4636
|
+
});
|
|
4637
|
+
if (errors && errors.length > 0 && config.verbose) {
|
|
4638
|
+
console.warn(`[EventualConsistency] ${errors.length} stale locks failed to delete`);
|
|
4639
|
+
}
|
|
4640
|
+
} catch (error) {
|
|
4641
|
+
if (config.verbose) {
|
|
4642
|
+
console.warn(`[EventualConsistency] Error cleaning up stale locks:`, error.message);
|
|
4643
|
+
}
|
|
4644
|
+
} finally {
|
|
4645
|
+
await tryFn(() => lockResource.delete(cleanupLockId));
|
|
4439
4646
|
}
|
|
4440
|
-
|
|
4441
|
-
|
|
4442
|
-
|
|
4443
|
-
|
|
4444
|
-
|
|
4445
|
-
|
|
4446
|
-
|
|
4447
|
-
|
|
4448
|
-
|
|
4449
|
-
|
|
4450
|
-
|
|
4451
|
-
|
|
4452
|
-
|
|
4453
|
-
|
|
4454
|
-
|
|
4455
|
-
|
|
4456
|
-
|
|
4457
|
-
|
|
4458
|
-
|
|
4459
|
-
|
|
4460
|
-
timestamp: "string|required",
|
|
4461
|
-
cohortDate: "string|required",
|
|
4462
|
-
cohortHour: "string|required",
|
|
4463
|
-
cohortMonth: "string|optional",
|
|
4464
|
-
source: "string|optional",
|
|
4465
|
-
applied: "boolean|optional"
|
|
4466
|
-
},
|
|
4467
|
-
behavior: "body-overflow",
|
|
4468
|
-
timestamps: true,
|
|
4469
|
-
partitions: partitionConfig,
|
|
4470
|
-
asyncPartitions: true,
|
|
4471
|
-
createdBy: "EventualConsistencyPlugin"
|
|
4472
|
-
})
|
|
4647
|
+
}
|
|
4648
|
+
|
|
4649
|
+
function startConsolidationTimer(handler, resourceName, fieldName, runConsolidationCallback, config) {
|
|
4650
|
+
const intervalMs = config.consolidationInterval * 1e3;
|
|
4651
|
+
if (config.verbose) {
|
|
4652
|
+
const nextRun = new Date(Date.now() + intervalMs);
|
|
4653
|
+
console.log(
|
|
4654
|
+
`[EventualConsistency] ${resourceName}.${fieldName} - Consolidation timer started. Next run at ${nextRun.toISOString()} (every ${config.consolidationInterval}s)`
|
|
4655
|
+
);
|
|
4656
|
+
}
|
|
4657
|
+
handler.consolidationTimer = setInterval(async () => {
|
|
4658
|
+
await runConsolidationCallback(handler, resourceName, fieldName);
|
|
4659
|
+
}, intervalMs);
|
|
4660
|
+
return handler.consolidationTimer;
|
|
4661
|
+
}
|
|
4662
|
+
async function runConsolidation(transactionResource, consolidateRecordFn, emitFn, config) {
|
|
4663
|
+
const startTime = Date.now();
|
|
4664
|
+
if (config.verbose) {
|
|
4665
|
+
console.log(
|
|
4666
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Starting consolidation run at ${(/* @__PURE__ */ new Date()).toISOString()}`
|
|
4473
4667
|
);
|
|
4474
|
-
|
|
4475
|
-
|
|
4668
|
+
}
|
|
4669
|
+
try {
|
|
4670
|
+
const now = /* @__PURE__ */ new Date();
|
|
4671
|
+
const hoursToCheck = config.consolidationWindow || 24;
|
|
4672
|
+
const cohortHours = [];
|
|
4673
|
+
for (let i = 0; i < hoursToCheck; i++) {
|
|
4674
|
+
const date = new Date(now.getTime() - i * 60 * 60 * 1e3);
|
|
4675
|
+
const cohortInfo = getCohortInfo(date, config.cohort.timezone, config.verbose);
|
|
4676
|
+
cohortHours.push(cohortInfo.hour);
|
|
4476
4677
|
}
|
|
4477
|
-
|
|
4478
|
-
|
|
4479
|
-
|
|
4480
|
-
|
|
4481
|
-
|
|
4482
|
-
|
|
4483
|
-
|
|
4484
|
-
|
|
4485
|
-
|
|
4486
|
-
|
|
4487
|
-
|
|
4488
|
-
|
|
4489
|
-
|
|
4678
|
+
if (config.verbose) {
|
|
4679
|
+
console.log(
|
|
4680
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Querying ${hoursToCheck} hour partitions for pending transactions...`
|
|
4681
|
+
);
|
|
4682
|
+
}
|
|
4683
|
+
const transactionsByHour = await Promise.all(
|
|
4684
|
+
cohortHours.map(async (cohortHour) => {
|
|
4685
|
+
const [ok, err, txns] = await tryFn(
|
|
4686
|
+
() => transactionResource.query({
|
|
4687
|
+
cohortHour,
|
|
4688
|
+
applied: false
|
|
4689
|
+
})
|
|
4690
|
+
);
|
|
4691
|
+
return ok ? txns : [];
|
|
4490
4692
|
})
|
|
4491
4693
|
);
|
|
4492
|
-
|
|
4493
|
-
|
|
4694
|
+
const transactions = transactionsByHour.flat();
|
|
4695
|
+
if (transactions.length === 0) {
|
|
4696
|
+
if (config.verbose) {
|
|
4697
|
+
console.log(
|
|
4698
|
+
`[EventualConsistency] ${config.resource}.${config.field} - No pending transactions found. Next run in ${config.consolidationInterval}s`
|
|
4699
|
+
);
|
|
4700
|
+
}
|
|
4701
|
+
return;
|
|
4702
|
+
}
|
|
4703
|
+
const uniqueIds = [...new Set(transactions.map((t) => t.originalId))];
|
|
4704
|
+
if (config.verbose) {
|
|
4705
|
+
console.log(
|
|
4706
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Found ${transactions.length} pending transactions for ${uniqueIds.length} records. Consolidating with concurrency=${config.consolidationConcurrency}...`
|
|
4707
|
+
);
|
|
4494
4708
|
}
|
|
4495
|
-
|
|
4496
|
-
|
|
4497
|
-
|
|
4709
|
+
const { results, errors } = await promisePool.PromisePool.for(uniqueIds).withConcurrency(config.consolidationConcurrency).process(async (id) => {
|
|
4710
|
+
return await consolidateRecordFn(id);
|
|
4711
|
+
});
|
|
4712
|
+
const duration = Date.now() - startTime;
|
|
4713
|
+
if (errors && errors.length > 0) {
|
|
4714
|
+
console.error(
|
|
4715
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Consolidation completed with ${errors.length} errors in ${duration}ms:`,
|
|
4716
|
+
errors
|
|
4717
|
+
);
|
|
4498
4718
|
}
|
|
4499
|
-
this._addHelperMethodsForHandler(handler);
|
|
4500
4719
|
if (config.verbose) {
|
|
4501
4720
|
console.log(
|
|
4502
|
-
`[EventualConsistency] ${
|
|
4721
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Consolidation complete: ${results.length} records consolidated in ${duration}ms (${errors.length} errors). Next run in ${config.consolidationInterval}s`
|
|
4503
4722
|
);
|
|
4504
4723
|
}
|
|
4505
|
-
|
|
4506
|
-
|
|
4507
|
-
|
|
4508
|
-
|
|
4509
|
-
|
|
4510
|
-
|
|
4511
|
-
|
|
4512
|
-
|
|
4513
|
-
|
|
4514
|
-
|
|
4515
|
-
|
|
4516
|
-
|
|
4517
|
-
|
|
4518
|
-
|
|
4519
|
-
|
|
4520
|
-
cohort: "string|required",
|
|
4521
|
-
transactionCount: "number|required",
|
|
4522
|
-
totalValue: "number|required",
|
|
4523
|
-
avgValue: "number|required",
|
|
4524
|
-
minValue: "number|required",
|
|
4525
|
-
maxValue: "number|required",
|
|
4526
|
-
operations: "object|optional",
|
|
4527
|
-
recordCount: "number|required",
|
|
4528
|
-
consolidatedAt: "string|required",
|
|
4529
|
-
updatedAt: "string|required"
|
|
4530
|
-
},
|
|
4531
|
-
behavior: "body-overflow",
|
|
4532
|
-
timestamps: false,
|
|
4533
|
-
createdBy: "EventualConsistencyPlugin"
|
|
4534
|
-
})
|
|
4724
|
+
if (emitFn) {
|
|
4725
|
+
emitFn("eventual-consistency.consolidated", {
|
|
4726
|
+
resource: config.resource,
|
|
4727
|
+
field: config.field,
|
|
4728
|
+
recordCount: uniqueIds.length,
|
|
4729
|
+
successCount: results.length,
|
|
4730
|
+
errorCount: errors.length,
|
|
4731
|
+
duration
|
|
4732
|
+
});
|
|
4733
|
+
}
|
|
4734
|
+
} catch (error) {
|
|
4735
|
+
const duration = Date.now() - startTime;
|
|
4736
|
+
console.error(
|
|
4737
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Consolidation error after ${duration}ms:`,
|
|
4738
|
+
error
|
|
4535
4739
|
);
|
|
4536
|
-
if (
|
|
4537
|
-
|
|
4740
|
+
if (emitFn) {
|
|
4741
|
+
emitFn("eventual-consistency.consolidation-error", error);
|
|
4538
4742
|
}
|
|
4539
|
-
handler.analyticsResource = ok ? analyticsResource : this.database.resources[analyticsResourceName];
|
|
4540
4743
|
}
|
|
4541
|
-
|
|
4542
|
-
|
|
4543
|
-
|
|
4544
|
-
|
|
4545
|
-
|
|
4546
|
-
|
|
4547
|
-
|
|
4548
|
-
|
|
4549
|
-
|
|
4550
|
-
}
|
|
4551
|
-
|
|
4552
|
-
|
|
4553
|
-
|
|
4744
|
+
}
|
|
4745
|
+
async function consolidateRecord(originalId, transactionResource, targetResource, lockResource, analyticsResource, updateAnalyticsFn, config) {
|
|
4746
|
+
await cleanupStaleLocks(lockResource, config);
|
|
4747
|
+
const lockId = `lock-${originalId}`;
|
|
4748
|
+
const [lockAcquired, lockErr, lock] = await tryFn(
|
|
4749
|
+
() => lockResource.insert({
|
|
4750
|
+
id: lockId,
|
|
4751
|
+
lockedAt: Date.now(),
|
|
4752
|
+
workerId: process.pid ? String(process.pid) : "unknown"
|
|
4753
|
+
})
|
|
4754
|
+
);
|
|
4755
|
+
if (!lockAcquired) {
|
|
4756
|
+
if (config.verbose) {
|
|
4757
|
+
console.log(`[EventualConsistency] Lock for ${originalId} already held, skipping`);
|
|
4554
4758
|
}
|
|
4759
|
+
const [recordOk, recordErr, record] = await tryFn(
|
|
4760
|
+
() => targetResource.get(originalId)
|
|
4761
|
+
);
|
|
4762
|
+
return recordOk && record ? record[config.field] || 0 : 0;
|
|
4555
4763
|
}
|
|
4556
|
-
|
|
4557
|
-
|
|
4558
|
-
|
|
4559
|
-
|
|
4560
|
-
|
|
4561
|
-
|
|
4562
|
-
|
|
4563
|
-
|
|
4564
|
-
|
|
4565
|
-
|
|
4566
|
-
|
|
4567
|
-
|
|
4568
|
-
|
|
4569
|
-
|
|
4570
|
-
}
|
|
4571
|
-
|
|
4764
|
+
try {
|
|
4765
|
+
const [ok, err, transactions] = await tryFn(
|
|
4766
|
+
() => transactionResource.query({
|
|
4767
|
+
originalId,
|
|
4768
|
+
applied: false
|
|
4769
|
+
})
|
|
4770
|
+
);
|
|
4771
|
+
if (!ok || !transactions || transactions.length === 0) {
|
|
4772
|
+
const [recordOk, recordErr, record] = await tryFn(
|
|
4773
|
+
() => targetResource.get(originalId)
|
|
4774
|
+
);
|
|
4775
|
+
const currentValue2 = recordOk && record ? record[config.field] || 0 : 0;
|
|
4776
|
+
if (config.verbose) {
|
|
4777
|
+
console.log(
|
|
4778
|
+
`[EventualConsistency] ${config.resource}.${config.field} - No pending transactions for ${originalId}, skipping`
|
|
4779
|
+
);
|
|
4572
4780
|
}
|
|
4781
|
+
return currentValue2;
|
|
4573
4782
|
}
|
|
4574
|
-
|
|
4575
|
-
|
|
4576
|
-
|
|
4577
|
-
|
|
4578
|
-
|
|
4579
|
-
|
|
4580
|
-
|
|
4783
|
+
const [appliedOk, appliedErr, appliedTransactions] = await tryFn(
|
|
4784
|
+
() => transactionResource.query({
|
|
4785
|
+
originalId,
|
|
4786
|
+
applied: true
|
|
4787
|
+
})
|
|
4788
|
+
);
|
|
4789
|
+
let currentValue = 0;
|
|
4790
|
+
if (appliedOk && appliedTransactions && appliedTransactions.length > 0) {
|
|
4791
|
+
const [recordExistsOk, recordExistsErr, recordExists] = await tryFn(
|
|
4792
|
+
() => targetResource.get(originalId)
|
|
4793
|
+
);
|
|
4794
|
+
if (!recordExistsOk || !recordExists) {
|
|
4795
|
+
if (config.verbose) {
|
|
4796
|
+
console.log(
|
|
4797
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Record ${originalId} doesn't exist, deleting ${appliedTransactions.length} old applied transactions`
|
|
4798
|
+
);
|
|
4581
4799
|
}
|
|
4582
|
-
|
|
4583
|
-
|
|
4584
|
-
|
|
4800
|
+
const { results, errors } = await promisePool.PromisePool.for(appliedTransactions).withConcurrency(10).process(async (txn) => {
|
|
4801
|
+
const [deleted] = await tryFn(() => transactionResource.delete(txn.id));
|
|
4802
|
+
return deleted;
|
|
4803
|
+
});
|
|
4804
|
+
if (config.verbose && errors && errors.length > 0) {
|
|
4805
|
+
console.warn(
|
|
4806
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Failed to delete ${errors.length} old applied transactions`
|
|
4807
|
+
);
|
|
4585
4808
|
}
|
|
4586
|
-
|
|
4587
|
-
|
|
4809
|
+
currentValue = 0;
|
|
4810
|
+
} else {
|
|
4811
|
+
appliedTransactions.sort(
|
|
4812
|
+
(a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()
|
|
4813
|
+
);
|
|
4814
|
+
const hasSetInApplied = appliedTransactions.some((t) => t.operation === "set");
|
|
4815
|
+
if (!hasSetInApplied) {
|
|
4816
|
+
const recordValue = recordExists[config.field] || 0;
|
|
4817
|
+
let appliedDelta = 0;
|
|
4818
|
+
for (const t of appliedTransactions) {
|
|
4819
|
+
if (t.operation === "add") appliedDelta += t.value;
|
|
4820
|
+
else if (t.operation === "sub") appliedDelta -= t.value;
|
|
4821
|
+
}
|
|
4822
|
+
const baseValue = recordValue - appliedDelta;
|
|
4823
|
+
const hasExistingAnchor = appliedTransactions.some((t) => t.source === "anchor");
|
|
4824
|
+
if (baseValue !== 0 && !hasExistingAnchor) {
|
|
4825
|
+
const firstTransactionDate = new Date(appliedTransactions[0].timestamp);
|
|
4826
|
+
const cohortInfo = getCohortInfo(firstTransactionDate, config.cohort.timezone, config.verbose);
|
|
4827
|
+
const anchorTransaction = {
|
|
4828
|
+
id: idGenerator(),
|
|
4829
|
+
originalId,
|
|
4830
|
+
field: config.field,
|
|
4831
|
+
value: baseValue,
|
|
4832
|
+
operation: "set",
|
|
4833
|
+
timestamp: new Date(firstTransactionDate.getTime() - 1).toISOString(),
|
|
4834
|
+
// 1ms before first txn to ensure it's first
|
|
4835
|
+
cohortDate: cohortInfo.date,
|
|
4836
|
+
cohortHour: cohortInfo.hour,
|
|
4837
|
+
cohortMonth: cohortInfo.month,
|
|
4838
|
+
source: "anchor",
|
|
4839
|
+
applied: true
|
|
4840
|
+
};
|
|
4841
|
+
await transactionResource.insert(anchorTransaction);
|
|
4842
|
+
appliedTransactions.unshift(anchorTransaction);
|
|
4843
|
+
}
|
|
4588
4844
|
}
|
|
4589
|
-
|
|
4590
|
-
resource: resourceName,
|
|
4591
|
-
field: fieldName
|
|
4592
|
-
});
|
|
4845
|
+
currentValue = config.reducer(appliedTransactions);
|
|
4593
4846
|
}
|
|
4594
|
-
}
|
|
4595
|
-
|
|
4596
|
-
|
|
4597
|
-
|
|
4598
|
-
|
|
4599
|
-
|
|
4600
|
-
|
|
4601
|
-
|
|
4602
|
-
|
|
4603
|
-
|
|
4604
|
-
|
|
4605
|
-
|
|
4847
|
+
} else {
|
|
4848
|
+
const [recordOk, recordErr, record] = await tryFn(
|
|
4849
|
+
() => targetResource.get(originalId)
|
|
4850
|
+
);
|
|
4851
|
+
currentValue = recordOk && record ? record[config.field] || 0 : 0;
|
|
4852
|
+
if (currentValue !== 0) {
|
|
4853
|
+
let anchorTimestamp;
|
|
4854
|
+
if (transactions && transactions.length > 0) {
|
|
4855
|
+
const firstPendingDate = new Date(transactions[0].timestamp);
|
|
4856
|
+
anchorTimestamp = new Date(firstPendingDate.getTime() - 1).toISOString();
|
|
4857
|
+
} else {
|
|
4858
|
+
anchorTimestamp = (/* @__PURE__ */ new Date()).toISOString();
|
|
4606
4859
|
}
|
|
4607
|
-
|
|
4608
|
-
|
|
4609
|
-
|
|
4610
|
-
|
|
4860
|
+
const cohortInfo = getCohortInfo(new Date(anchorTimestamp), config.cohort.timezone, config.verbose);
|
|
4861
|
+
const anchorTransaction = {
|
|
4862
|
+
id: idGenerator(),
|
|
4863
|
+
originalId,
|
|
4864
|
+
field: config.field,
|
|
4865
|
+
value: currentValue,
|
|
4866
|
+
operation: "set",
|
|
4867
|
+
timestamp: anchorTimestamp,
|
|
4868
|
+
cohortDate: cohortInfo.date,
|
|
4869
|
+
cohortHour: cohortInfo.hour,
|
|
4870
|
+
cohortMonth: cohortInfo.month,
|
|
4871
|
+
source: "anchor",
|
|
4872
|
+
applied: true
|
|
4873
|
+
};
|
|
4874
|
+
await transactionResource.insert(anchorTransaction);
|
|
4875
|
+
if (config.verbose) {
|
|
4876
|
+
console.log(
|
|
4877
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Created anchor transaction for ${originalId} with base value ${currentValue}`
|
|
4878
|
+
);
|
|
4611
4879
|
}
|
|
4612
4880
|
}
|
|
4613
|
-
};
|
|
4614
|
-
return partitions;
|
|
4615
|
-
}
|
|
4616
|
-
/**
|
|
4617
|
-
* Auto-detect timezone from environment or system
|
|
4618
|
-
* @private
|
|
4619
|
-
*/
|
|
4620
|
-
_detectTimezone() {
|
|
4621
|
-
if (process.env.TZ) {
|
|
4622
|
-
return process.env.TZ;
|
|
4623
|
-
}
|
|
4624
|
-
try {
|
|
4625
|
-
const systemTimezone = Intl.DateTimeFormat().resolvedOptions().timeZone;
|
|
4626
|
-
if (systemTimezone) {
|
|
4627
|
-
return systemTimezone;
|
|
4628
|
-
}
|
|
4629
|
-
} catch (err) {
|
|
4630
4881
|
}
|
|
4631
|
-
|
|
4632
|
-
|
|
4633
|
-
|
|
4634
|
-
|
|
4635
|
-
* @private
|
|
4636
|
-
*/
|
|
4637
|
-
_resolveFieldAndPlugin(resource, field, value) {
|
|
4638
|
-
if (!resource._eventualConsistencyPlugins) {
|
|
4639
|
-
throw new Error(`No eventual consistency plugins configured for this resource`);
|
|
4882
|
+
if (config.verbose) {
|
|
4883
|
+
console.log(
|
|
4884
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Consolidating ${originalId}: ${transactions.length} pending transactions (current: ${currentValue} from ${appliedOk && appliedTransactions?.length > 0 ? "applied transactions" : "record"})`
|
|
4885
|
+
);
|
|
4640
4886
|
}
|
|
4641
|
-
|
|
4642
|
-
|
|
4643
|
-
|
|
4644
|
-
|
|
4645
|
-
|
|
4887
|
+
transactions.sort(
|
|
4888
|
+
(a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()
|
|
4889
|
+
);
|
|
4890
|
+
const hasSetOperation = transactions.some((t) => t.operation === "set");
|
|
4891
|
+
if (currentValue !== 0 && !hasSetOperation) {
|
|
4892
|
+
transactions.unshift(createSyntheticSetTransaction(currentValue));
|
|
4893
|
+
}
|
|
4894
|
+
const consolidatedValue = config.reducer(transactions);
|
|
4895
|
+
if (config.verbose) {
|
|
4896
|
+
console.log(
|
|
4897
|
+
`[EventualConsistency] ${config.resource}.${config.field} - ${originalId}: ${currentValue} \u2192 ${consolidatedValue} (${consolidatedValue > currentValue ? "+" : ""}${consolidatedValue - currentValue})`
|
|
4646
4898
|
);
|
|
4647
4899
|
}
|
|
4648
|
-
|
|
4649
|
-
|
|
4650
|
-
|
|
4651
|
-
|
|
4652
|
-
|
|
4653
|
-
|
|
4654
|
-
|
|
4655
|
-
|
|
4656
|
-
return consolidatedValue;
|
|
4657
|
-
}
|
|
4658
|
-
/**
|
|
4659
|
-
* Create synthetic 'set' transaction from current value
|
|
4660
|
-
* @private
|
|
4661
|
-
*/
|
|
4662
|
-
_createSyntheticSetTransaction(currentValue) {
|
|
4663
|
-
return {
|
|
4664
|
-
id: "__synthetic__",
|
|
4665
|
-
operation: "set",
|
|
4666
|
-
value: currentValue,
|
|
4667
|
-
timestamp: (/* @__PURE__ */ new Date(0)).toISOString(),
|
|
4668
|
-
synthetic: true
|
|
4669
|
-
};
|
|
4670
|
-
}
|
|
4671
|
-
addHelperMethods() {
|
|
4672
|
-
const firstResource = this.fieldHandlers.values().next().value;
|
|
4673
|
-
if (!firstResource) return;
|
|
4674
|
-
const firstHandler = firstResource.values().next().value;
|
|
4675
|
-
if (!firstHandler || !firstHandler.targetResource) return;
|
|
4676
|
-
const resource = firstHandler.targetResource;
|
|
4677
|
-
const plugin = this;
|
|
4678
|
-
resource.set = async (id, field, value) => {
|
|
4679
|
-
const { plugin: handler } = plugin._resolveFieldAndPlugin(resource, field, value);
|
|
4680
|
-
const now = /* @__PURE__ */ new Date();
|
|
4681
|
-
const cohortInfo = plugin.getCohortInfo(now);
|
|
4682
|
-
const transaction = {
|
|
4683
|
-
id: idGenerator(),
|
|
4684
|
-
originalId: id,
|
|
4685
|
-
field: handler.field,
|
|
4686
|
-
value,
|
|
4687
|
-
operation: "set",
|
|
4688
|
-
timestamp: now.toISOString(),
|
|
4689
|
-
cohortDate: cohortInfo.date,
|
|
4690
|
-
cohortHour: cohortInfo.hour,
|
|
4691
|
-
cohortMonth: cohortInfo.month,
|
|
4692
|
-
source: "set",
|
|
4693
|
-
applied: false
|
|
4694
|
-
};
|
|
4695
|
-
await handler.transactionResource.insert(transaction);
|
|
4696
|
-
if (plugin.config.mode === "sync") {
|
|
4697
|
-
const oldResource = plugin.config.resource;
|
|
4698
|
-
const oldField = plugin.config.field;
|
|
4699
|
-
const oldTransactionResource = plugin.transactionResource;
|
|
4700
|
-
const oldTargetResource = plugin.targetResource;
|
|
4701
|
-
const oldLockResource = plugin.lockResource;
|
|
4702
|
-
const oldAnalyticsResource = plugin.analyticsResource;
|
|
4703
|
-
plugin.config.resource = handler.resource;
|
|
4704
|
-
plugin.config.field = handler.field;
|
|
4705
|
-
plugin.transactionResource = handler.transactionResource;
|
|
4706
|
-
plugin.targetResource = handler.targetResource;
|
|
4707
|
-
plugin.lockResource = handler.lockResource;
|
|
4708
|
-
plugin.analyticsResource = handler.analyticsResource;
|
|
4709
|
-
const result = await plugin._syncModeConsolidate(id, field);
|
|
4710
|
-
plugin.config.resource = oldResource;
|
|
4711
|
-
plugin.config.field = oldField;
|
|
4712
|
-
plugin.transactionResource = oldTransactionResource;
|
|
4713
|
-
plugin.targetResource = oldTargetResource;
|
|
4714
|
-
plugin.lockResource = oldLockResource;
|
|
4715
|
-
plugin.analyticsResource = oldAnalyticsResource;
|
|
4716
|
-
return result;
|
|
4717
|
-
}
|
|
4718
|
-
return value;
|
|
4719
|
-
};
|
|
4720
|
-
resource.add = async (id, field, amount) => {
|
|
4721
|
-
const { plugin: handler } = plugin._resolveFieldAndPlugin(resource, field, amount);
|
|
4722
|
-
const now = /* @__PURE__ */ new Date();
|
|
4723
|
-
const cohortInfo = plugin.getCohortInfo(now);
|
|
4724
|
-
const transaction = {
|
|
4725
|
-
id: idGenerator(),
|
|
4726
|
-
originalId: id,
|
|
4727
|
-
field: handler.field,
|
|
4728
|
-
value: amount,
|
|
4729
|
-
operation: "add",
|
|
4730
|
-
timestamp: now.toISOString(),
|
|
4731
|
-
cohortDate: cohortInfo.date,
|
|
4732
|
-
cohortHour: cohortInfo.hour,
|
|
4733
|
-
cohortMonth: cohortInfo.month,
|
|
4734
|
-
source: "add",
|
|
4735
|
-
applied: false
|
|
4736
|
-
};
|
|
4737
|
-
await handler.transactionResource.insert(transaction);
|
|
4738
|
-
if (plugin.config.mode === "sync") {
|
|
4739
|
-
const oldResource = plugin.config.resource;
|
|
4740
|
-
const oldField = plugin.config.field;
|
|
4741
|
-
const oldTransactionResource = plugin.transactionResource;
|
|
4742
|
-
const oldTargetResource = plugin.targetResource;
|
|
4743
|
-
const oldLockResource = plugin.lockResource;
|
|
4744
|
-
const oldAnalyticsResource = plugin.analyticsResource;
|
|
4745
|
-
plugin.config.resource = handler.resource;
|
|
4746
|
-
plugin.config.field = handler.field;
|
|
4747
|
-
plugin.transactionResource = handler.transactionResource;
|
|
4748
|
-
plugin.targetResource = handler.targetResource;
|
|
4749
|
-
plugin.lockResource = handler.lockResource;
|
|
4750
|
-
plugin.analyticsResource = handler.analyticsResource;
|
|
4751
|
-
const result = await plugin._syncModeConsolidate(id, field);
|
|
4752
|
-
plugin.config.resource = oldResource;
|
|
4753
|
-
plugin.config.field = oldField;
|
|
4754
|
-
plugin.transactionResource = oldTransactionResource;
|
|
4755
|
-
plugin.targetResource = oldTargetResource;
|
|
4756
|
-
plugin.lockResource = oldLockResource;
|
|
4757
|
-
plugin.analyticsResource = oldAnalyticsResource;
|
|
4758
|
-
return result;
|
|
4759
|
-
}
|
|
4760
|
-
const [ok, err, record] = await tryFn(() => handler.targetResource.get(id));
|
|
4761
|
-
const currentValue = ok && record ? record[field] || 0 : 0;
|
|
4762
|
-
return currentValue + amount;
|
|
4763
|
-
};
|
|
4764
|
-
resource.sub = async (id, field, amount) => {
|
|
4765
|
-
const { plugin: handler } = plugin._resolveFieldAndPlugin(resource, field, amount);
|
|
4766
|
-
const now = /* @__PURE__ */ new Date();
|
|
4767
|
-
const cohortInfo = plugin.getCohortInfo(now);
|
|
4768
|
-
const transaction = {
|
|
4769
|
-
id: idGenerator(),
|
|
4770
|
-
originalId: id,
|
|
4771
|
-
field: handler.field,
|
|
4772
|
-
value: amount,
|
|
4773
|
-
operation: "sub",
|
|
4774
|
-
timestamp: now.toISOString(),
|
|
4775
|
-
cohortDate: cohortInfo.date,
|
|
4776
|
-
cohortHour: cohortInfo.hour,
|
|
4777
|
-
cohortMonth: cohortInfo.month,
|
|
4778
|
-
source: "sub",
|
|
4779
|
-
applied: false
|
|
4780
|
-
};
|
|
4781
|
-
await handler.transactionResource.insert(transaction);
|
|
4782
|
-
if (plugin.config.mode === "sync") {
|
|
4783
|
-
const oldResource = plugin.config.resource;
|
|
4784
|
-
const oldField = plugin.config.field;
|
|
4785
|
-
const oldTransactionResource = plugin.transactionResource;
|
|
4786
|
-
const oldTargetResource = plugin.targetResource;
|
|
4787
|
-
const oldLockResource = plugin.lockResource;
|
|
4788
|
-
const oldAnalyticsResource = plugin.analyticsResource;
|
|
4789
|
-
plugin.config.resource = handler.resource;
|
|
4790
|
-
plugin.config.field = handler.field;
|
|
4791
|
-
plugin.transactionResource = handler.transactionResource;
|
|
4792
|
-
plugin.targetResource = handler.targetResource;
|
|
4793
|
-
plugin.lockResource = handler.lockResource;
|
|
4794
|
-
plugin.analyticsResource = handler.analyticsResource;
|
|
4795
|
-
const result = await plugin._syncModeConsolidate(id, field);
|
|
4796
|
-
plugin.config.resource = oldResource;
|
|
4797
|
-
plugin.config.field = oldField;
|
|
4798
|
-
plugin.transactionResource = oldTransactionResource;
|
|
4799
|
-
plugin.targetResource = oldTargetResource;
|
|
4800
|
-
plugin.lockResource = oldLockResource;
|
|
4801
|
-
plugin.analyticsResource = oldAnalyticsResource;
|
|
4802
|
-
return result;
|
|
4803
|
-
}
|
|
4804
|
-
const [ok, err, record] = await tryFn(() => handler.targetResource.get(id));
|
|
4805
|
-
const currentValue = ok && record ? record[field] || 0 : 0;
|
|
4806
|
-
return currentValue - amount;
|
|
4807
|
-
};
|
|
4808
|
-
resource.consolidate = async (id, field) => {
|
|
4809
|
-
if (!field) {
|
|
4810
|
-
throw new Error(`Field parameter is required: consolidate(id, field)`);
|
|
4811
|
-
}
|
|
4812
|
-
const handler = resource._eventualConsistencyPlugins[field];
|
|
4813
|
-
if (!handler) {
|
|
4814
|
-
const availableFields = Object.keys(resource._eventualConsistencyPlugins).join(", ");
|
|
4815
|
-
throw new Error(
|
|
4816
|
-
`No eventual consistency plugin found for field "${field}". Available fields: ${availableFields}`
|
|
4817
|
-
);
|
|
4818
|
-
}
|
|
4819
|
-
const oldResource = plugin.config.resource;
|
|
4820
|
-
const oldField = plugin.config.field;
|
|
4821
|
-
const oldTransactionResource = plugin.transactionResource;
|
|
4822
|
-
const oldTargetResource = plugin.targetResource;
|
|
4823
|
-
const oldLockResource = plugin.lockResource;
|
|
4824
|
-
const oldAnalyticsResource = plugin.analyticsResource;
|
|
4825
|
-
plugin.config.resource = handler.resource;
|
|
4826
|
-
plugin.config.field = handler.field;
|
|
4827
|
-
plugin.transactionResource = handler.transactionResource;
|
|
4828
|
-
plugin.targetResource = handler.targetResource;
|
|
4829
|
-
plugin.lockResource = handler.lockResource;
|
|
4830
|
-
plugin.analyticsResource = handler.analyticsResource;
|
|
4831
|
-
const result = await plugin.consolidateRecord(id);
|
|
4832
|
-
plugin.config.resource = oldResource;
|
|
4833
|
-
plugin.config.field = oldField;
|
|
4834
|
-
plugin.transactionResource = oldTransactionResource;
|
|
4835
|
-
plugin.targetResource = oldTargetResource;
|
|
4836
|
-
plugin.lockResource = oldLockResource;
|
|
4837
|
-
plugin.analyticsResource = oldAnalyticsResource;
|
|
4838
|
-
return result;
|
|
4839
|
-
};
|
|
4840
|
-
resource.getConsolidatedValue = async (id, field, options = {}) => {
|
|
4841
|
-
const handler = resource._eventualConsistencyPlugins[field];
|
|
4842
|
-
if (!handler) {
|
|
4843
|
-
const availableFields = Object.keys(resource._eventualConsistencyPlugins).join(", ");
|
|
4844
|
-
throw new Error(
|
|
4845
|
-
`No eventual consistency plugin found for field "${field}". Available fields: ${availableFields}`
|
|
4846
|
-
);
|
|
4847
|
-
}
|
|
4848
|
-
const oldResource = plugin.config.resource;
|
|
4849
|
-
const oldField = plugin.config.field;
|
|
4850
|
-
const oldTransactionResource = plugin.transactionResource;
|
|
4851
|
-
const oldTargetResource = plugin.targetResource;
|
|
4852
|
-
plugin.config.resource = handler.resource;
|
|
4853
|
-
plugin.config.field = handler.field;
|
|
4854
|
-
plugin.transactionResource = handler.transactionResource;
|
|
4855
|
-
plugin.targetResource = handler.targetResource;
|
|
4856
|
-
const result = await plugin.getConsolidatedValue(id, options);
|
|
4857
|
-
plugin.config.resource = oldResource;
|
|
4858
|
-
plugin.config.field = oldField;
|
|
4859
|
-
plugin.transactionResource = oldTransactionResource;
|
|
4860
|
-
plugin.targetResource = oldTargetResource;
|
|
4861
|
-
return result;
|
|
4862
|
-
};
|
|
4863
|
-
}
|
|
4864
|
-
async createTransaction(handler, data) {
|
|
4865
|
-
const now = /* @__PURE__ */ new Date();
|
|
4866
|
-
const cohortInfo = this.getCohortInfo(now);
|
|
4867
|
-
const watermarkMs = this.config.consolidationWindow * 60 * 60 * 1e3;
|
|
4868
|
-
const watermarkTime = now.getTime() - watermarkMs;
|
|
4869
|
-
const cohortHourDate = /* @__PURE__ */ new Date(cohortInfo.hour + ":00:00Z");
|
|
4870
|
-
if (cohortHourDate.getTime() < watermarkTime) {
|
|
4871
|
-
const hoursLate = Math.floor((now.getTime() - cohortHourDate.getTime()) / (60 * 60 * 1e3));
|
|
4872
|
-
if (this.config.lateArrivalStrategy === "ignore") {
|
|
4873
|
-
if (this.config.verbose) {
|
|
4900
|
+
const [updateOk, updateErr] = await tryFn(
|
|
4901
|
+
() => targetResource.update(originalId, {
|
|
4902
|
+
[config.field]: consolidatedValue
|
|
4903
|
+
})
|
|
4904
|
+
);
|
|
4905
|
+
if (!updateOk) {
|
|
4906
|
+
if (updateErr?.message?.includes("does not exist")) {
|
|
4907
|
+
if (config.verbose) {
|
|
4874
4908
|
console.warn(
|
|
4875
|
-
`[EventualConsistency]
|
|
4909
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Record ${originalId} doesn't exist. Skipping consolidation. ${transactions.length} transactions will remain pending until record is created.`
|
|
4876
4910
|
);
|
|
4877
4911
|
}
|
|
4878
|
-
return
|
|
4879
|
-
} else if (this.config.lateArrivalStrategy === "warn") {
|
|
4880
|
-
console.warn(
|
|
4881
|
-
`[EventualConsistency] Late arrival detected: transaction for ${cohortInfo.hour} is ${hoursLate}h late (watermark: ${this.config.consolidationWindow}h). Processing anyway, but consolidation may not pick it up.`
|
|
4882
|
-
);
|
|
4912
|
+
return consolidatedValue;
|
|
4883
4913
|
}
|
|
4914
|
+
console.error(
|
|
4915
|
+
`[EventualConsistency] ${config.resource}.${config.field} - FAILED to update ${originalId}: ${updateErr?.message || updateErr}`,
|
|
4916
|
+
{ error: updateErr, consolidatedValue, currentValue }
|
|
4917
|
+
);
|
|
4918
|
+
throw updateErr;
|
|
4884
4919
|
}
|
|
4885
|
-
|
|
4886
|
-
|
|
4887
|
-
|
|
4888
|
-
|
|
4889
|
-
|
|
4890
|
-
operation: data.operation || "set",
|
|
4891
|
-
timestamp: now.toISOString(),
|
|
4892
|
-
cohortDate: cohortInfo.date,
|
|
4893
|
-
cohortHour: cohortInfo.hour,
|
|
4894
|
-
cohortMonth: cohortInfo.month,
|
|
4895
|
-
source: data.source || "unknown",
|
|
4896
|
-
applied: false
|
|
4897
|
-
};
|
|
4898
|
-
if (this.config.batchTransactions) {
|
|
4899
|
-
handler.pendingTransactions.set(transaction.id, transaction);
|
|
4900
|
-
if (this.config.verbose) {
|
|
4901
|
-
console.log(
|
|
4902
|
-
`[EventualConsistency] ${handler.resource}.${handler.field} - Transaction batched: ${data.operation} ${data.value} for ${data.originalId} (batch: ${handler.pendingTransactions.size}/${this.config.batchSize})`
|
|
4920
|
+
if (updateOk) {
|
|
4921
|
+
const transactionsToUpdate = transactions.filter((txn) => txn.id !== "__synthetic__");
|
|
4922
|
+
const { results, errors } = await promisePool.PromisePool.for(transactionsToUpdate).withConcurrency(10).process(async (txn) => {
|
|
4923
|
+
const [ok2, err2] = await tryFn(
|
|
4924
|
+
() => transactionResource.update(txn.id, { applied: true })
|
|
4903
4925
|
);
|
|
4926
|
+
if (!ok2 && config.verbose) {
|
|
4927
|
+
console.warn(`[EventualConsistency] Failed to mark transaction ${txn.id} as applied:`, err2?.message);
|
|
4928
|
+
}
|
|
4929
|
+
return ok2;
|
|
4930
|
+
});
|
|
4931
|
+
if (errors && errors.length > 0 && config.verbose) {
|
|
4932
|
+
console.warn(`[EventualConsistency] ${errors.length} transactions failed to mark as applied`);
|
|
4904
4933
|
}
|
|
4905
|
-
if (
|
|
4906
|
-
await
|
|
4934
|
+
if (config.enableAnalytics && transactionsToUpdate.length > 0 && updateAnalyticsFn) {
|
|
4935
|
+
await updateAnalyticsFn(transactionsToUpdate);
|
|
4907
4936
|
}
|
|
4908
|
-
|
|
4909
|
-
|
|
4910
|
-
|
|
4911
|
-
|
|
4912
|
-
|
|
4913
|
-
|
|
4937
|
+
if (targetResource && targetResource.cache && typeof targetResource.cache.delete === "function") {
|
|
4938
|
+
try {
|
|
4939
|
+
const cacheKey = await targetResource.cacheKeyFor({ id: originalId });
|
|
4940
|
+
await targetResource.cache.delete(cacheKey);
|
|
4941
|
+
if (config.verbose) {
|
|
4942
|
+
console.log(
|
|
4943
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Cache invalidated for ${originalId}`
|
|
4944
|
+
);
|
|
4945
|
+
}
|
|
4946
|
+
} catch (cacheErr) {
|
|
4947
|
+
if (config.verbose) {
|
|
4948
|
+
console.warn(
|
|
4949
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Failed to invalidate cache for ${originalId}: ${cacheErr?.message}`
|
|
4950
|
+
);
|
|
4951
|
+
}
|
|
4952
|
+
}
|
|
4914
4953
|
}
|
|
4915
4954
|
}
|
|
4916
|
-
return
|
|
4955
|
+
return consolidatedValue;
|
|
4956
|
+
} finally {
|
|
4957
|
+
const [lockReleased, lockReleaseErr] = await tryFn(() => lockResource.delete(lockId));
|
|
4958
|
+
if (!lockReleased && config.verbose) {
|
|
4959
|
+
console.warn(`[EventualConsistency] Failed to release lock ${lockId}:`, lockReleaseErr?.message);
|
|
4960
|
+
}
|
|
4917
4961
|
}
|
|
4918
|
-
|
|
4919
|
-
|
|
4920
|
-
|
|
4921
|
-
|
|
4922
|
-
|
|
4923
|
-
|
|
4924
|
-
|
|
4925
|
-
|
|
4926
|
-
|
|
4927
|
-
|
|
4928
|
-
|
|
4929
|
-
|
|
4930
|
-
|
|
4962
|
+
}
|
|
4963
|
+
async function getConsolidatedValue(originalId, options, transactionResource, targetResource, config) {
|
|
4964
|
+
const includeApplied = options.includeApplied || false;
|
|
4965
|
+
const startDate = options.startDate;
|
|
4966
|
+
const endDate = options.endDate;
|
|
4967
|
+
const query = { originalId };
|
|
4968
|
+
if (!includeApplied) {
|
|
4969
|
+
query.applied = false;
|
|
4970
|
+
}
|
|
4971
|
+
const [ok, err, transactions] = await tryFn(
|
|
4972
|
+
() => transactionResource.query(query)
|
|
4973
|
+
);
|
|
4974
|
+
if (!ok || !transactions || transactions.length === 0) {
|
|
4975
|
+
const [recordOk2, recordErr2, record2] = await tryFn(
|
|
4976
|
+
() => targetResource.get(originalId)
|
|
4977
|
+
);
|
|
4978
|
+
if (recordOk2 && record2) {
|
|
4979
|
+
return record2[config.field] || 0;
|
|
4931
4980
|
}
|
|
4981
|
+
return 0;
|
|
4932
4982
|
}
|
|
4933
|
-
|
|
4934
|
-
|
|
4935
|
-
|
|
4936
|
-
|
|
4937
|
-
|
|
4938
|
-
|
|
4939
|
-
|
|
4940
|
-
|
|
4941
|
-
return {
|
|
4942
|
-
date: `${year}-${month}-${day}`,
|
|
4943
|
-
hour: `${year}-${month}-${day}T${hour}`,
|
|
4944
|
-
// ISO-like format for hour partition
|
|
4945
|
-
month: `${year}-${month}`
|
|
4946
|
-
};
|
|
4983
|
+
let filtered = transactions;
|
|
4984
|
+
if (startDate || endDate) {
|
|
4985
|
+
filtered = transactions.filter((t) => {
|
|
4986
|
+
const timestamp = new Date(t.timestamp);
|
|
4987
|
+
if (startDate && timestamp < new Date(startDate)) return false;
|
|
4988
|
+
if (endDate && timestamp > new Date(endDate)) return false;
|
|
4989
|
+
return true;
|
|
4990
|
+
});
|
|
4947
4991
|
}
|
|
4948
|
-
|
|
4949
|
-
|
|
4950
|
-
|
|
4951
|
-
|
|
4952
|
-
|
|
4953
|
-
|
|
4954
|
-
|
|
4955
|
-
|
|
4956
|
-
|
|
4957
|
-
|
|
4958
|
-
|
|
4959
|
-
|
|
4960
|
-
|
|
4961
|
-
|
|
4962
|
-
|
|
4963
|
-
|
|
4964
|
-
|
|
4965
|
-
|
|
4966
|
-
|
|
4967
|
-
|
|
4992
|
+
const [recordOk, recordErr, record] = await tryFn(
|
|
4993
|
+
() => targetResource.get(originalId)
|
|
4994
|
+
);
|
|
4995
|
+
const currentValue = recordOk && record ? record[config.field] || 0 : 0;
|
|
4996
|
+
const hasSetOperation = filtered.some((t) => t.operation === "set");
|
|
4997
|
+
if (currentValue !== 0 && !hasSetOperation) {
|
|
4998
|
+
filtered.unshift(createSyntheticSetTransaction(currentValue));
|
|
4999
|
+
}
|
|
5000
|
+
filtered.sort(
|
|
5001
|
+
(a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()
|
|
5002
|
+
);
|
|
5003
|
+
return config.reducer(filtered);
|
|
5004
|
+
}
|
|
5005
|
+
async function getCohortStats(cohortDate, transactionResource) {
|
|
5006
|
+
const [ok, err, transactions] = await tryFn(
|
|
5007
|
+
() => transactionResource.query({
|
|
5008
|
+
cohortDate
|
|
5009
|
+
})
|
|
5010
|
+
);
|
|
5011
|
+
if (!ok) return null;
|
|
5012
|
+
const stats = {
|
|
5013
|
+
date: cohortDate,
|
|
5014
|
+
transactionCount: transactions.length,
|
|
5015
|
+
totalValue: 0,
|
|
5016
|
+
byOperation: { set: 0, add: 0, sub: 0 },
|
|
5017
|
+
byOriginalId: {}
|
|
5018
|
+
};
|
|
5019
|
+
for (const txn of transactions) {
|
|
5020
|
+
stats.totalValue += txn.value || 0;
|
|
5021
|
+
stats.byOperation[txn.operation] = (stats.byOperation[txn.operation] || 0) + 1;
|
|
5022
|
+
if (!stats.byOriginalId[txn.originalId]) {
|
|
5023
|
+
stats.byOriginalId[txn.originalId] = {
|
|
5024
|
+
count: 0,
|
|
5025
|
+
value: 0
|
|
4968
5026
|
};
|
|
4969
|
-
|
|
4970
|
-
|
|
4971
|
-
|
|
5027
|
+
}
|
|
5028
|
+
stats.byOriginalId[txn.originalId].count++;
|
|
5029
|
+
stats.byOriginalId[txn.originalId].value += txn.value || 0;
|
|
5030
|
+
}
|
|
5031
|
+
return stats;
|
|
5032
|
+
}
|
|
5033
|
+
async function recalculateRecord(originalId, transactionResource, targetResource, lockResource, consolidateRecordFn, config) {
|
|
5034
|
+
await cleanupStaleLocks(lockResource, config);
|
|
5035
|
+
const lockId = `lock-recalculate-${originalId}`;
|
|
5036
|
+
const [lockAcquired, lockErr, lock] = await tryFn(
|
|
5037
|
+
() => lockResource.insert({
|
|
5038
|
+
id: lockId,
|
|
5039
|
+
lockedAt: Date.now(),
|
|
5040
|
+
workerId: process.pid ? String(process.pid) : "unknown"
|
|
5041
|
+
})
|
|
5042
|
+
);
|
|
5043
|
+
if (!lockAcquired) {
|
|
5044
|
+
if (config.verbose) {
|
|
5045
|
+
console.log(`[EventualConsistency] Recalculate lock for ${originalId} already held, skipping`);
|
|
5046
|
+
}
|
|
5047
|
+
throw new Error(`Cannot recalculate ${originalId}: lock already held by another worker`);
|
|
5048
|
+
}
|
|
5049
|
+
try {
|
|
5050
|
+
if (config.verbose) {
|
|
5051
|
+
console.log(
|
|
5052
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Starting recalculation for ${originalId} (resetting all transactions to pending)`
|
|
5053
|
+
);
|
|
5054
|
+
}
|
|
5055
|
+
const [allOk, allErr, allTransactions] = await tryFn(
|
|
5056
|
+
() => transactionResource.query({
|
|
5057
|
+
originalId
|
|
5058
|
+
})
|
|
5059
|
+
);
|
|
5060
|
+
if (!allOk || !allTransactions || allTransactions.length === 0) {
|
|
5061
|
+
if (config.verbose) {
|
|
5062
|
+
console.log(
|
|
5063
|
+
`[EventualConsistency] ${config.resource}.${config.field} - No transactions found for ${originalId}, nothing to recalculate`
|
|
4972
5064
|
);
|
|
4973
5065
|
}
|
|
4974
|
-
return
|
|
5066
|
+
return 0;
|
|
4975
5067
|
}
|
|
4976
|
-
|
|
4977
|
-
startConsolidationTimer() {
|
|
4978
|
-
const intervalMs = this.config.consolidationInterval * 1e3;
|
|
4979
|
-
if (this.config.verbose) {
|
|
4980
|
-
const nextRun = new Date(Date.now() + intervalMs);
|
|
5068
|
+
if (config.verbose) {
|
|
4981
5069
|
console.log(
|
|
4982
|
-
`[EventualConsistency] ${
|
|
5070
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Found ${allTransactions.length} total transactions for ${originalId}, marking all as pending...`
|
|
4983
5071
|
);
|
|
4984
5072
|
}
|
|
4985
|
-
|
|
4986
|
-
|
|
4987
|
-
|
|
4988
|
-
|
|
4989
|
-
|
|
4990
|
-
|
|
4991
|
-
|
|
4992
|
-
|
|
5073
|
+
const transactionsToReset = allTransactions.filter((txn) => txn.source !== "anchor");
|
|
5074
|
+
const { results, errors } = await promisePool.PromisePool.for(transactionsToReset).withConcurrency(10).process(async (txn) => {
|
|
5075
|
+
const [ok, err] = await tryFn(
|
|
5076
|
+
() => transactionResource.update(txn.id, { applied: false })
|
|
5077
|
+
);
|
|
5078
|
+
if (!ok && config.verbose) {
|
|
5079
|
+
console.warn(`[EventualConsistency] Failed to reset transaction ${txn.id}:`, err?.message);
|
|
5080
|
+
}
|
|
5081
|
+
return ok;
|
|
5082
|
+
});
|
|
5083
|
+
if (errors && errors.length > 0) {
|
|
5084
|
+
console.warn(
|
|
5085
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Failed to reset ${errors.length} transactions during recalculation`
|
|
5086
|
+
);
|
|
5087
|
+
}
|
|
5088
|
+
if (config.verbose) {
|
|
4993
5089
|
console.log(
|
|
4994
|
-
`[EventualConsistency] ${
|
|
5090
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Reset ${results.length} transactions to pending, now resetting record value and running consolidation...`
|
|
4995
5091
|
);
|
|
4996
5092
|
}
|
|
4997
|
-
|
|
4998
|
-
|
|
4999
|
-
|
|
5000
|
-
|
|
5001
|
-
|
|
5002
|
-
|
|
5003
|
-
|
|
5004
|
-
|
|
5005
|
-
|
|
5006
|
-
const oldLockResource = this.lockResource;
|
|
5007
|
-
const oldAnalyticsResource = this.analyticsResource;
|
|
5008
|
-
this.config.resource = resourceName;
|
|
5009
|
-
this.config.field = fieldName;
|
|
5010
|
-
this.transactionResource = handler.transactionResource;
|
|
5011
|
-
this.targetResource = handler.targetResource;
|
|
5012
|
-
this.lockResource = handler.lockResource;
|
|
5013
|
-
this.analyticsResource = handler.analyticsResource;
|
|
5014
|
-
try {
|
|
5015
|
-
await this.runConsolidation();
|
|
5016
|
-
} finally {
|
|
5017
|
-
this.config.resource = oldResource;
|
|
5018
|
-
this.config.field = oldField;
|
|
5019
|
-
this.transactionResource = oldTransactionResource;
|
|
5020
|
-
this.targetResource = oldTargetResource;
|
|
5021
|
-
this.lockResource = oldLockResource;
|
|
5022
|
-
this.analyticsResource = oldAnalyticsResource;
|
|
5093
|
+
const [resetOk, resetErr] = await tryFn(
|
|
5094
|
+
() => targetResource.update(originalId, {
|
|
5095
|
+
[config.field]: 0
|
|
5096
|
+
})
|
|
5097
|
+
);
|
|
5098
|
+
if (!resetOk && config.verbose) {
|
|
5099
|
+
console.warn(
|
|
5100
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Failed to reset record value for ${originalId}: ${resetErr?.message}`
|
|
5101
|
+
);
|
|
5023
5102
|
}
|
|
5024
|
-
|
|
5025
|
-
|
|
5026
|
-
const startTime = Date.now();
|
|
5027
|
-
if (this.config.verbose) {
|
|
5103
|
+
const consolidatedValue = await consolidateRecordFn(originalId);
|
|
5104
|
+
if (config.verbose) {
|
|
5028
5105
|
console.log(
|
|
5029
|
-
`[EventualConsistency] ${
|
|
5106
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Recalculation complete for ${originalId}: final value = ${consolidatedValue}`
|
|
5030
5107
|
);
|
|
5031
5108
|
}
|
|
5032
|
-
|
|
5033
|
-
|
|
5034
|
-
|
|
5035
|
-
|
|
5036
|
-
|
|
5037
|
-
|
|
5038
|
-
|
|
5039
|
-
|
|
5109
|
+
return consolidatedValue;
|
|
5110
|
+
} finally {
|
|
5111
|
+
const [lockReleased, lockReleaseErr] = await tryFn(() => lockResource.delete(lockId));
|
|
5112
|
+
if (!lockReleased && config.verbose) {
|
|
5113
|
+
console.warn(`[EventualConsistency] Failed to release recalculate lock ${lockId}:`, lockReleaseErr?.message);
|
|
5114
|
+
}
|
|
5115
|
+
}
|
|
5116
|
+
}
|
|
5117
|
+
|
|
5118
|
+
function startGarbageCollectionTimer(handler, resourceName, fieldName, runGCCallback, config) {
|
|
5119
|
+
const gcIntervalMs = config.gcInterval * 1e3;
|
|
5120
|
+
handler.gcTimer = setInterval(async () => {
|
|
5121
|
+
await runGCCallback(handler, resourceName, fieldName);
|
|
5122
|
+
}, gcIntervalMs);
|
|
5123
|
+
return handler.gcTimer;
|
|
5124
|
+
}
|
|
5125
|
+
async function runGarbageCollection(transactionResource, lockResource, config, emitFn) {
|
|
5126
|
+
const gcLockId = `lock-gc-${config.resource}-${config.field}`;
|
|
5127
|
+
const [lockAcquired] = await tryFn(
|
|
5128
|
+
() => lockResource.insert({
|
|
5129
|
+
id: gcLockId,
|
|
5130
|
+
lockedAt: Date.now(),
|
|
5131
|
+
workerId: process.pid ? String(process.pid) : "unknown"
|
|
5132
|
+
})
|
|
5133
|
+
);
|
|
5134
|
+
if (!lockAcquired) {
|
|
5135
|
+
if (config.verbose) {
|
|
5136
|
+
console.log(`[EventualConsistency] GC already running in another container`);
|
|
5137
|
+
}
|
|
5138
|
+
return;
|
|
5139
|
+
}
|
|
5140
|
+
try {
|
|
5141
|
+
const now = Date.now();
|
|
5142
|
+
const retentionMs = config.transactionRetention * 24 * 60 * 60 * 1e3;
|
|
5143
|
+
const cutoffDate = new Date(now - retentionMs);
|
|
5144
|
+
const cutoffIso = cutoffDate.toISOString();
|
|
5145
|
+
if (config.verbose) {
|
|
5146
|
+
console.log(`[EventualConsistency] Running GC for transactions older than ${cutoffIso} (${config.transactionRetention} days)`);
|
|
5147
|
+
}
|
|
5148
|
+
const [ok, err, oldTransactions] = await tryFn(
|
|
5149
|
+
() => transactionResource.query({
|
|
5150
|
+
applied: true,
|
|
5151
|
+
timestamp: { "<": cutoffIso }
|
|
5152
|
+
})
|
|
5153
|
+
);
|
|
5154
|
+
if (!ok) {
|
|
5155
|
+
if (config.verbose) {
|
|
5156
|
+
console.warn(`[EventualConsistency] GC failed to query transactions:`, err?.message);
|
|
5040
5157
|
}
|
|
5041
|
-
|
|
5042
|
-
|
|
5043
|
-
|
|
5044
|
-
|
|
5158
|
+
return;
|
|
5159
|
+
}
|
|
5160
|
+
if (!oldTransactions || oldTransactions.length === 0) {
|
|
5161
|
+
if (config.verbose) {
|
|
5162
|
+
console.log(`[EventualConsistency] No old transactions to clean up`);
|
|
5045
5163
|
}
|
|
5046
|
-
|
|
5047
|
-
|
|
5048
|
-
|
|
5049
|
-
|
|
5050
|
-
|
|
5051
|
-
|
|
5052
|
-
|
|
5053
|
-
|
|
5054
|
-
|
|
5055
|
-
|
|
5164
|
+
return;
|
|
5165
|
+
}
|
|
5166
|
+
if (config.verbose) {
|
|
5167
|
+
console.log(`[EventualConsistency] Deleting ${oldTransactions.length} old transactions`);
|
|
5168
|
+
}
|
|
5169
|
+
const { results, errors } = await promisePool.PromisePool.for(oldTransactions).withConcurrency(10).process(async (txn) => {
|
|
5170
|
+
const [deleted] = await tryFn(() => transactionResource.delete(txn.id));
|
|
5171
|
+
return deleted;
|
|
5172
|
+
});
|
|
5173
|
+
if (config.verbose) {
|
|
5174
|
+
console.log(`[EventualConsistency] GC completed: ${results.length} deleted, ${errors.length} errors`);
|
|
5175
|
+
}
|
|
5176
|
+
if (emitFn) {
|
|
5177
|
+
emitFn("eventual-consistency.gc-completed", {
|
|
5178
|
+
resource: config.resource,
|
|
5179
|
+
field: config.field,
|
|
5180
|
+
deletedCount: results.length,
|
|
5181
|
+
errorCount: errors.length
|
|
5182
|
+
});
|
|
5183
|
+
}
|
|
5184
|
+
} catch (error) {
|
|
5185
|
+
if (config.verbose) {
|
|
5186
|
+
console.warn(`[EventualConsistency] GC error:`, error.message);
|
|
5187
|
+
}
|
|
5188
|
+
if (emitFn) {
|
|
5189
|
+
emitFn("eventual-consistency.gc-error", error);
|
|
5190
|
+
}
|
|
5191
|
+
} finally {
|
|
5192
|
+
await tryFn(() => lockResource.delete(gcLockId));
|
|
5193
|
+
}
|
|
5194
|
+
}
|
|
5195
|
+
|
|
5196
|
+
async function updateAnalytics(transactions, analyticsResource, config) {
|
|
5197
|
+
if (!analyticsResource || transactions.length === 0) return;
|
|
5198
|
+
if (config.verbose) {
|
|
5199
|
+
console.log(
|
|
5200
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Updating analytics for ${transactions.length} transactions...`
|
|
5201
|
+
);
|
|
5202
|
+
}
|
|
5203
|
+
try {
|
|
5204
|
+
const byHour = groupByCohort(transactions, "cohortHour");
|
|
5205
|
+
const cohortCount = Object.keys(byHour).length;
|
|
5206
|
+
if (config.verbose) {
|
|
5207
|
+
console.log(
|
|
5208
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Updating ${cohortCount} hourly analytics cohorts...`
|
|
5056
5209
|
);
|
|
5057
|
-
|
|
5058
|
-
|
|
5059
|
-
|
|
5060
|
-
|
|
5061
|
-
|
|
5062
|
-
|
|
5063
|
-
|
|
5064
|
-
return;
|
|
5065
|
-
}
|
|
5066
|
-
const uniqueIds = [...new Set(transactions.map((t) => t.originalId))];
|
|
5067
|
-
if (this.config.verbose) {
|
|
5210
|
+
}
|
|
5211
|
+
for (const [cohort, txns] of Object.entries(byHour)) {
|
|
5212
|
+
await upsertAnalytics("hour", cohort, txns, analyticsResource, config);
|
|
5213
|
+
}
|
|
5214
|
+
if (config.analyticsConfig.rollupStrategy === "incremental") {
|
|
5215
|
+
const uniqueHours = Object.keys(byHour);
|
|
5216
|
+
if (config.verbose) {
|
|
5068
5217
|
console.log(
|
|
5069
|
-
`[EventualConsistency] ${
|
|
5070
|
-
);
|
|
5071
|
-
}
|
|
5072
|
-
const { results, errors } = await promisePool.PromisePool.for(uniqueIds).withConcurrency(this.config.consolidationConcurrency).process(async (id) => {
|
|
5073
|
-
return await this.consolidateRecord(id);
|
|
5074
|
-
});
|
|
5075
|
-
const duration = Date.now() - startTime;
|
|
5076
|
-
if (errors && errors.length > 0) {
|
|
5077
|
-
console.error(
|
|
5078
|
-
`[EventualConsistency] ${this.config.resource}.${this.config.field} - Consolidation completed with ${errors.length} errors in ${duration}ms:`,
|
|
5079
|
-
errors
|
|
5218
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Rolling up ${uniqueHours.length} hours to daily/monthly analytics...`
|
|
5080
5219
|
);
|
|
5081
5220
|
}
|
|
5082
|
-
|
|
5083
|
-
|
|
5084
|
-
`[EventualConsistency] ${this.config.resource}.${this.config.field} - Consolidation complete: ${results.length} records consolidated in ${duration}ms (${errors.length} errors). Next run in ${this.config.consolidationInterval}s`
|
|
5085
|
-
);
|
|
5221
|
+
for (const cohortHour of uniqueHours) {
|
|
5222
|
+
await rollupAnalytics(cohortHour, analyticsResource, config);
|
|
5086
5223
|
}
|
|
5087
|
-
|
|
5088
|
-
|
|
5089
|
-
|
|
5090
|
-
|
|
5091
|
-
successCount: results.length,
|
|
5092
|
-
errorCount: errors.length,
|
|
5093
|
-
duration
|
|
5094
|
-
});
|
|
5095
|
-
} catch (error) {
|
|
5096
|
-
const duration = Date.now() - startTime;
|
|
5097
|
-
console.error(
|
|
5098
|
-
`[EventualConsistency] ${this.config.resource}.${this.config.field} - Consolidation error after ${duration}ms:`,
|
|
5099
|
-
error
|
|
5224
|
+
}
|
|
5225
|
+
if (config.verbose) {
|
|
5226
|
+
console.log(
|
|
5227
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Analytics update complete for ${cohortCount} cohorts`
|
|
5100
5228
|
);
|
|
5101
|
-
this.emit("eventual-consistency.consolidation-error", error);
|
|
5102
5229
|
}
|
|
5230
|
+
} catch (error) {
|
|
5231
|
+
console.warn(
|
|
5232
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Analytics update error:`,
|
|
5233
|
+
error.message
|
|
5234
|
+
);
|
|
5103
5235
|
}
|
|
5104
|
-
|
|
5105
|
-
|
|
5106
|
-
|
|
5107
|
-
|
|
5108
|
-
|
|
5109
|
-
|
|
5110
|
-
|
|
5111
|
-
|
|
5236
|
+
}
|
|
5237
|
+
async function upsertAnalytics(period, cohort, transactions, analyticsResource, config) {
|
|
5238
|
+
const id = `${period}-${cohort}`;
|
|
5239
|
+
const transactionCount = transactions.length;
|
|
5240
|
+
const signedValues = transactions.map((t) => {
|
|
5241
|
+
if (t.operation === "sub") return -t.value;
|
|
5242
|
+
return t.value;
|
|
5243
|
+
});
|
|
5244
|
+
const totalValue = signedValues.reduce((sum, v) => sum + v, 0);
|
|
5245
|
+
const avgValue = totalValue / transactionCount;
|
|
5246
|
+
const minValue = Math.min(...signedValues);
|
|
5247
|
+
const maxValue = Math.max(...signedValues);
|
|
5248
|
+
const operations = calculateOperationBreakdown(transactions);
|
|
5249
|
+
const recordCount = new Set(transactions.map((t) => t.originalId)).size;
|
|
5250
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
5251
|
+
const [existingOk, existingErr, existing] = await tryFn(
|
|
5252
|
+
() => analyticsResource.get(id)
|
|
5253
|
+
);
|
|
5254
|
+
if (existingOk && existing) {
|
|
5255
|
+
const newTransactionCount = existing.transactionCount + transactionCount;
|
|
5256
|
+
const newTotalValue = existing.totalValue + totalValue;
|
|
5257
|
+
const newAvgValue = newTotalValue / newTransactionCount;
|
|
5258
|
+
const newMinValue = Math.min(existing.minValue, minValue);
|
|
5259
|
+
const newMaxValue = Math.max(existing.maxValue, maxValue);
|
|
5260
|
+
const newOperations = { ...existing.operations };
|
|
5261
|
+
for (const [op, stats] of Object.entries(operations)) {
|
|
5262
|
+
if (!newOperations[op]) {
|
|
5263
|
+
newOperations[op] = { count: 0, sum: 0 };
|
|
5264
|
+
}
|
|
5265
|
+
newOperations[op].count += stats.count;
|
|
5266
|
+
newOperations[op].sum += stats.sum;
|
|
5267
|
+
}
|
|
5268
|
+
const newRecordCount = Math.max(existing.recordCount, recordCount);
|
|
5269
|
+
await tryFn(
|
|
5270
|
+
() => analyticsResource.update(id, {
|
|
5271
|
+
transactionCount: newTransactionCount,
|
|
5272
|
+
totalValue: newTotalValue,
|
|
5273
|
+
avgValue: newAvgValue,
|
|
5274
|
+
minValue: newMinValue,
|
|
5275
|
+
maxValue: newMaxValue,
|
|
5276
|
+
operations: newOperations,
|
|
5277
|
+
recordCount: newRecordCount,
|
|
5278
|
+
updatedAt: now
|
|
5112
5279
|
})
|
|
5113
5280
|
);
|
|
5114
|
-
|
|
5115
|
-
|
|
5116
|
-
|
|
5117
|
-
|
|
5118
|
-
|
|
5119
|
-
|
|
5120
|
-
|
|
5121
|
-
|
|
5122
|
-
|
|
5123
|
-
|
|
5124
|
-
|
|
5125
|
-
|
|
5126
|
-
|
|
5127
|
-
|
|
5128
|
-
|
|
5129
|
-
)
|
|
5130
|
-
|
|
5131
|
-
|
|
5132
|
-
|
|
5133
|
-
|
|
5134
|
-
|
|
5135
|
-
|
|
5136
|
-
|
|
5137
|
-
|
|
5138
|
-
|
|
5139
|
-
|
|
5140
|
-
|
|
5281
|
+
} else {
|
|
5282
|
+
await tryFn(
|
|
5283
|
+
() => analyticsResource.insert({
|
|
5284
|
+
id,
|
|
5285
|
+
period,
|
|
5286
|
+
cohort,
|
|
5287
|
+
transactionCount,
|
|
5288
|
+
totalValue,
|
|
5289
|
+
avgValue,
|
|
5290
|
+
minValue,
|
|
5291
|
+
maxValue,
|
|
5292
|
+
operations,
|
|
5293
|
+
recordCount,
|
|
5294
|
+
consolidatedAt: now,
|
|
5295
|
+
updatedAt: now
|
|
5296
|
+
})
|
|
5297
|
+
);
|
|
5298
|
+
}
|
|
5299
|
+
}
|
|
5300
|
+
function calculateOperationBreakdown(transactions) {
|
|
5301
|
+
const breakdown = {};
|
|
5302
|
+
for (const txn of transactions) {
|
|
5303
|
+
const op = txn.operation;
|
|
5304
|
+
if (!breakdown[op]) {
|
|
5305
|
+
breakdown[op] = { count: 0, sum: 0 };
|
|
5306
|
+
}
|
|
5307
|
+
breakdown[op].count++;
|
|
5308
|
+
const signedValue = op === "sub" ? -txn.value : txn.value;
|
|
5309
|
+
breakdown[op].sum += signedValue;
|
|
5310
|
+
}
|
|
5311
|
+
return breakdown;
|
|
5312
|
+
}
|
|
5313
|
+
async function rollupAnalytics(cohortHour, analyticsResource, config) {
|
|
5314
|
+
const cohortDate = cohortHour.substring(0, 10);
|
|
5315
|
+
const cohortMonth = cohortHour.substring(0, 7);
|
|
5316
|
+
await rollupPeriod("day", cohortDate, cohortDate, analyticsResource);
|
|
5317
|
+
await rollupPeriod("month", cohortMonth, cohortMonth, analyticsResource);
|
|
5318
|
+
}
|
|
5319
|
+
async function rollupPeriod(period, cohort, sourcePrefix, analyticsResource, config) {
|
|
5320
|
+
const sourcePeriod = period === "day" ? "hour" : "day";
|
|
5321
|
+
const [ok, err, allAnalytics] = await tryFn(
|
|
5322
|
+
() => analyticsResource.list()
|
|
5323
|
+
);
|
|
5324
|
+
if (!ok || !allAnalytics) return;
|
|
5325
|
+
const sourceAnalytics = allAnalytics.filter(
|
|
5326
|
+
(a) => a.period === sourcePeriod && a.cohort.startsWith(sourcePrefix)
|
|
5327
|
+
);
|
|
5328
|
+
if (sourceAnalytics.length === 0) return;
|
|
5329
|
+
const transactionCount = sourceAnalytics.reduce((sum, a) => sum + a.transactionCount, 0);
|
|
5330
|
+
const totalValue = sourceAnalytics.reduce((sum, a) => sum + a.totalValue, 0);
|
|
5331
|
+
const avgValue = totalValue / transactionCount;
|
|
5332
|
+
const minValue = Math.min(...sourceAnalytics.map((a) => a.minValue));
|
|
5333
|
+
const maxValue = Math.max(...sourceAnalytics.map((a) => a.maxValue));
|
|
5334
|
+
const operations = {};
|
|
5335
|
+
for (const analytics of sourceAnalytics) {
|
|
5336
|
+
for (const [op, stats] of Object.entries(analytics.operations || {})) {
|
|
5337
|
+
if (!operations[op]) {
|
|
5338
|
+
operations[op] = { count: 0, sum: 0 };
|
|
5339
|
+
}
|
|
5340
|
+
operations[op].count += stats.count;
|
|
5341
|
+
operations[op].sum += stats.sum;
|
|
5342
|
+
}
|
|
5343
|
+
}
|
|
5344
|
+
const recordCount = Math.max(...sourceAnalytics.map((a) => a.recordCount));
|
|
5345
|
+
const id = `${period}-${cohort}`;
|
|
5346
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
5347
|
+
const [existingOk, existingErr, existing] = await tryFn(
|
|
5348
|
+
() => analyticsResource.get(id)
|
|
5349
|
+
);
|
|
5350
|
+
if (existingOk && existing) {
|
|
5351
|
+
await tryFn(
|
|
5352
|
+
() => analyticsResource.update(id, {
|
|
5353
|
+
transactionCount,
|
|
5354
|
+
totalValue,
|
|
5355
|
+
avgValue,
|
|
5356
|
+
minValue,
|
|
5357
|
+
maxValue,
|
|
5358
|
+
operations,
|
|
5359
|
+
recordCount,
|
|
5360
|
+
updatedAt: now
|
|
5361
|
+
})
|
|
5362
|
+
);
|
|
5363
|
+
} else {
|
|
5364
|
+
await tryFn(
|
|
5365
|
+
() => analyticsResource.insert({
|
|
5366
|
+
id,
|
|
5367
|
+
period,
|
|
5368
|
+
cohort,
|
|
5369
|
+
transactionCount,
|
|
5370
|
+
totalValue,
|
|
5371
|
+
avgValue,
|
|
5372
|
+
minValue,
|
|
5373
|
+
maxValue,
|
|
5374
|
+
operations,
|
|
5375
|
+
recordCount,
|
|
5376
|
+
consolidatedAt: now,
|
|
5377
|
+
updatedAt: now
|
|
5378
|
+
})
|
|
5379
|
+
);
|
|
5380
|
+
}
|
|
5381
|
+
}
|
|
5382
|
+
function fillGaps(data, period, startDate, endDate) {
|
|
5383
|
+
if (!data || data.length === 0) {
|
|
5384
|
+
data = [];
|
|
5385
|
+
}
|
|
5386
|
+
const dataMap = /* @__PURE__ */ new Map();
|
|
5387
|
+
data.forEach((item) => {
|
|
5388
|
+
dataMap.set(item.cohort, item);
|
|
5389
|
+
});
|
|
5390
|
+
const result = [];
|
|
5391
|
+
const emptyRecord = {
|
|
5392
|
+
count: 0,
|
|
5393
|
+
sum: 0,
|
|
5394
|
+
avg: 0,
|
|
5395
|
+
min: 0,
|
|
5396
|
+
max: 0,
|
|
5397
|
+
recordCount: 0
|
|
5398
|
+
};
|
|
5399
|
+
if (period === "hour") {
|
|
5400
|
+
const start = /* @__PURE__ */ new Date(startDate + "T00:00:00Z");
|
|
5401
|
+
const end = /* @__PURE__ */ new Date(endDate + "T23:59:59Z");
|
|
5402
|
+
for (let dt = new Date(start); dt <= end; dt.setHours(dt.getHours() + 1)) {
|
|
5403
|
+
const cohort = dt.toISOString().substring(0, 13);
|
|
5404
|
+
result.push(dataMap.get(cohort) || { cohort, ...emptyRecord });
|
|
5405
|
+
}
|
|
5406
|
+
} else if (period === "day") {
|
|
5407
|
+
const start = new Date(startDate);
|
|
5408
|
+
const end = new Date(endDate);
|
|
5409
|
+
for (let dt = new Date(start); dt <= end; dt.setDate(dt.getDate() + 1)) {
|
|
5410
|
+
const cohort = dt.toISOString().substring(0, 10);
|
|
5411
|
+
result.push(dataMap.get(cohort) || { cohort, ...emptyRecord });
|
|
5412
|
+
}
|
|
5413
|
+
} else if (period === "month") {
|
|
5414
|
+
const startYear = parseInt(startDate.substring(0, 4));
|
|
5415
|
+
const startMonth = parseInt(startDate.substring(5, 7));
|
|
5416
|
+
const endYear = parseInt(endDate.substring(0, 4));
|
|
5417
|
+
const endMonth = parseInt(endDate.substring(5, 7));
|
|
5418
|
+
for (let year = startYear; year <= endYear; year++) {
|
|
5419
|
+
const firstMonth = year === startYear ? startMonth : 1;
|
|
5420
|
+
const lastMonth = year === endYear ? endMonth : 12;
|
|
5421
|
+
for (let month = firstMonth; month <= lastMonth; month++) {
|
|
5422
|
+
const cohort = `${year}-${month.toString().padStart(2, "0")}`;
|
|
5423
|
+
result.push(dataMap.get(cohort) || { cohort, ...emptyRecord });
|
|
5141
5424
|
}
|
|
5142
|
-
|
|
5143
|
-
|
|
5144
|
-
|
|
5145
|
-
|
|
5146
|
-
|
|
5425
|
+
}
|
|
5426
|
+
}
|
|
5427
|
+
return result;
|
|
5428
|
+
}
|
|
5429
|
+
async function getAnalytics(resourceName, field, options, fieldHandlers) {
|
|
5430
|
+
const resourceHandlers = fieldHandlers.get(resourceName);
|
|
5431
|
+
if (!resourceHandlers) {
|
|
5432
|
+
throw new Error(`No eventual consistency configured for resource: ${resourceName}`);
|
|
5433
|
+
}
|
|
5434
|
+
const handler = resourceHandlers.get(field);
|
|
5435
|
+
if (!handler) {
|
|
5436
|
+
throw new Error(`No eventual consistency configured for field: ${resourceName}.${field}`);
|
|
5437
|
+
}
|
|
5438
|
+
if (!handler.analyticsResource) {
|
|
5439
|
+
throw new Error("Analytics not enabled for this plugin");
|
|
5440
|
+
}
|
|
5441
|
+
const { period = "day", date, startDate, endDate, month, year, breakdown = false } = options;
|
|
5442
|
+
const [ok, err, allAnalytics] = await tryFn(
|
|
5443
|
+
() => handler.analyticsResource.list()
|
|
5444
|
+
);
|
|
5445
|
+
if (!ok || !allAnalytics) {
|
|
5446
|
+
return [];
|
|
5447
|
+
}
|
|
5448
|
+
let filtered = allAnalytics.filter((a) => a.period === period);
|
|
5449
|
+
if (date) {
|
|
5450
|
+
if (period === "hour") {
|
|
5451
|
+
filtered = filtered.filter((a) => a.cohort.startsWith(date));
|
|
5452
|
+
} else {
|
|
5453
|
+
filtered = filtered.filter((a) => a.cohort === date);
|
|
5454
|
+
}
|
|
5455
|
+
} else if (startDate && endDate) {
|
|
5456
|
+
filtered = filtered.filter((a) => a.cohort >= startDate && a.cohort <= endDate);
|
|
5457
|
+
} else if (month) {
|
|
5458
|
+
filtered = filtered.filter((a) => a.cohort.startsWith(month));
|
|
5459
|
+
} else if (year) {
|
|
5460
|
+
filtered = filtered.filter((a) => a.cohort.startsWith(String(year)));
|
|
5461
|
+
}
|
|
5462
|
+
filtered.sort((a, b) => a.cohort.localeCompare(b.cohort));
|
|
5463
|
+
if (breakdown === "operations") {
|
|
5464
|
+
return filtered.map((a) => ({
|
|
5465
|
+
cohort: a.cohort,
|
|
5466
|
+
...a.operations
|
|
5467
|
+
}));
|
|
5468
|
+
}
|
|
5469
|
+
return filtered.map((a) => ({
|
|
5470
|
+
cohort: a.cohort,
|
|
5471
|
+
count: a.transactionCount,
|
|
5472
|
+
sum: a.totalValue,
|
|
5473
|
+
avg: a.avgValue,
|
|
5474
|
+
min: a.minValue,
|
|
5475
|
+
max: a.maxValue,
|
|
5476
|
+
operations: a.operations,
|
|
5477
|
+
recordCount: a.recordCount
|
|
5478
|
+
}));
|
|
5479
|
+
}
|
|
5480
|
+
async function getMonthByDay(resourceName, field, month, options, fieldHandlers) {
|
|
5481
|
+
const year = parseInt(month.substring(0, 4));
|
|
5482
|
+
const monthNum = parseInt(month.substring(5, 7));
|
|
5483
|
+
const firstDay = new Date(year, monthNum - 1, 1);
|
|
5484
|
+
const lastDay = new Date(year, monthNum, 0);
|
|
5485
|
+
const startDate = firstDay.toISOString().substring(0, 10);
|
|
5486
|
+
const endDate = lastDay.toISOString().substring(0, 10);
|
|
5487
|
+
const data = await getAnalytics(resourceName, field, {
|
|
5488
|
+
period: "day",
|
|
5489
|
+
startDate,
|
|
5490
|
+
endDate
|
|
5491
|
+
}, fieldHandlers);
|
|
5492
|
+
if (options.fillGaps) {
|
|
5493
|
+
return fillGaps(data, "day", startDate, endDate);
|
|
5494
|
+
}
|
|
5495
|
+
return data;
|
|
5496
|
+
}
|
|
5497
|
+
async function getDayByHour(resourceName, field, date, options, fieldHandlers) {
|
|
5498
|
+
const data = await getAnalytics(resourceName, field, {
|
|
5499
|
+
period: "hour",
|
|
5500
|
+
date
|
|
5501
|
+
}, fieldHandlers);
|
|
5502
|
+
if (options.fillGaps) {
|
|
5503
|
+
return fillGaps(data, "hour", date, date);
|
|
5504
|
+
}
|
|
5505
|
+
return data;
|
|
5506
|
+
}
|
|
5507
|
+
async function getLastNDays(resourceName, field, days, options, fieldHandlers) {
|
|
5508
|
+
const dates = Array.from({ length: days }, (_, i) => {
|
|
5509
|
+
const date = /* @__PURE__ */ new Date();
|
|
5510
|
+
date.setDate(date.getDate() - i);
|
|
5511
|
+
return date.toISOString().substring(0, 10);
|
|
5512
|
+
}).reverse();
|
|
5513
|
+
const data = await getAnalytics(resourceName, field, {
|
|
5514
|
+
period: "day",
|
|
5515
|
+
startDate: dates[0],
|
|
5516
|
+
endDate: dates[dates.length - 1]
|
|
5517
|
+
}, fieldHandlers);
|
|
5518
|
+
if (options.fillGaps) {
|
|
5519
|
+
return fillGaps(data, "day", dates[0], dates[dates.length - 1]);
|
|
5520
|
+
}
|
|
5521
|
+
return data;
|
|
5522
|
+
}
|
|
5523
|
+
async function getYearByMonth(resourceName, field, year, options, fieldHandlers) {
|
|
5524
|
+
const data = await getAnalytics(resourceName, field, {
|
|
5525
|
+
period: "month",
|
|
5526
|
+
year
|
|
5527
|
+
}, fieldHandlers);
|
|
5528
|
+
if (options.fillGaps) {
|
|
5529
|
+
const startDate = `${year}-01`;
|
|
5530
|
+
const endDate = `${year}-12`;
|
|
5531
|
+
return fillGaps(data, "month", startDate, endDate);
|
|
5532
|
+
}
|
|
5533
|
+
return data;
|
|
5534
|
+
}
|
|
5535
|
+
async function getMonthByHour(resourceName, field, month, options, fieldHandlers) {
|
|
5536
|
+
let year, monthNum;
|
|
5537
|
+
if (month === "last") {
|
|
5538
|
+
const now = /* @__PURE__ */ new Date();
|
|
5539
|
+
now.setMonth(now.getMonth() - 1);
|
|
5540
|
+
year = now.getFullYear();
|
|
5541
|
+
monthNum = now.getMonth() + 1;
|
|
5542
|
+
} else {
|
|
5543
|
+
year = parseInt(month.substring(0, 4));
|
|
5544
|
+
monthNum = parseInt(month.substring(5, 7));
|
|
5545
|
+
}
|
|
5546
|
+
const firstDay = new Date(year, monthNum - 1, 1);
|
|
5547
|
+
const lastDay = new Date(year, monthNum, 0);
|
|
5548
|
+
const startDate = firstDay.toISOString().substring(0, 10);
|
|
5549
|
+
const endDate = lastDay.toISOString().substring(0, 10);
|
|
5550
|
+
const data = await getAnalytics(resourceName, field, {
|
|
5551
|
+
period: "hour",
|
|
5552
|
+
startDate,
|
|
5553
|
+
endDate
|
|
5554
|
+
}, fieldHandlers);
|
|
5555
|
+
if (options.fillGaps) {
|
|
5556
|
+
return fillGaps(data, "hour", startDate, endDate);
|
|
5557
|
+
}
|
|
5558
|
+
return data;
|
|
5559
|
+
}
|
|
5560
|
+
async function getTopRecords(resourceName, field, options, fieldHandlers) {
|
|
5561
|
+
const resourceHandlers = fieldHandlers.get(resourceName);
|
|
5562
|
+
if (!resourceHandlers) {
|
|
5563
|
+
throw new Error(`No eventual consistency configured for resource: ${resourceName}`);
|
|
5564
|
+
}
|
|
5565
|
+
const handler = resourceHandlers.get(field);
|
|
5566
|
+
if (!handler) {
|
|
5567
|
+
throw new Error(`No eventual consistency configured for field: ${resourceName}.${field}`);
|
|
5568
|
+
}
|
|
5569
|
+
if (!handler.transactionResource) {
|
|
5570
|
+
throw new Error("Transaction resource not initialized");
|
|
5571
|
+
}
|
|
5572
|
+
const { period = "day", date, metric = "transactionCount", limit = 10 } = options;
|
|
5573
|
+
const [ok, err, transactions] = await tryFn(
|
|
5574
|
+
() => handler.transactionResource.list()
|
|
5575
|
+
);
|
|
5576
|
+
if (!ok || !transactions) {
|
|
5577
|
+
return [];
|
|
5578
|
+
}
|
|
5579
|
+
let filtered = transactions;
|
|
5580
|
+
if (date) {
|
|
5581
|
+
if (period === "hour") {
|
|
5582
|
+
filtered = transactions.filter((t) => t.cohortHour && t.cohortHour.startsWith(date));
|
|
5583
|
+
} else if (period === "day") {
|
|
5584
|
+
filtered = transactions.filter((t) => t.cohortDate === date);
|
|
5585
|
+
} else if (period === "month") {
|
|
5586
|
+
filtered = transactions.filter((t) => t.cohortMonth && t.cohortMonth.startsWith(date));
|
|
5587
|
+
}
|
|
5588
|
+
}
|
|
5589
|
+
const byRecord = {};
|
|
5590
|
+
for (const txn of filtered) {
|
|
5591
|
+
const recordId = txn.originalId;
|
|
5592
|
+
if (!byRecord[recordId]) {
|
|
5593
|
+
byRecord[recordId] = { count: 0, sum: 0 };
|
|
5594
|
+
}
|
|
5595
|
+
byRecord[recordId].count++;
|
|
5596
|
+
byRecord[recordId].sum += txn.value;
|
|
5597
|
+
}
|
|
5598
|
+
const records = Object.entries(byRecord).map(([recordId, stats]) => ({
|
|
5599
|
+
recordId,
|
|
5600
|
+
count: stats.count,
|
|
5601
|
+
sum: stats.sum
|
|
5602
|
+
}));
|
|
5603
|
+
records.sort((a, b) => {
|
|
5604
|
+
if (metric === "transactionCount") {
|
|
5605
|
+
return b.count - a.count;
|
|
5606
|
+
} else if (metric === "totalValue") {
|
|
5607
|
+
return b.sum - a.sum;
|
|
5608
|
+
}
|
|
5609
|
+
return 0;
|
|
5610
|
+
});
|
|
5611
|
+
return records.slice(0, limit);
|
|
5612
|
+
}
|
|
5613
|
+
|
|
5614
|
+
function addHelperMethods(resource, plugin, config) {
|
|
5615
|
+
resource.set = async (id, field, value) => {
|
|
5616
|
+
const { plugin: handler } = resolveFieldAndPlugin(resource, field, value);
|
|
5617
|
+
const now = /* @__PURE__ */ new Date();
|
|
5618
|
+
const cohortInfo = getCohortInfo(now, config.cohort.timezone, config.verbose);
|
|
5619
|
+
const transaction = {
|
|
5620
|
+
id: idGenerator(),
|
|
5621
|
+
originalId: id,
|
|
5622
|
+
field: handler.field,
|
|
5623
|
+
value,
|
|
5624
|
+
operation: "set",
|
|
5625
|
+
timestamp: now.toISOString(),
|
|
5626
|
+
cohortDate: cohortInfo.date,
|
|
5627
|
+
cohortHour: cohortInfo.hour,
|
|
5628
|
+
cohortMonth: cohortInfo.month,
|
|
5629
|
+
source: "set",
|
|
5630
|
+
applied: false
|
|
5631
|
+
};
|
|
5632
|
+
await handler.transactionResource.insert(transaction);
|
|
5633
|
+
if (config.mode === "sync") {
|
|
5634
|
+
return await plugin._syncModeConsolidate(handler, id, field);
|
|
5635
|
+
}
|
|
5636
|
+
return value;
|
|
5637
|
+
};
|
|
5638
|
+
resource.add = async (id, field, amount) => {
|
|
5639
|
+
const { plugin: handler } = resolveFieldAndPlugin(resource, field, amount);
|
|
5640
|
+
const now = /* @__PURE__ */ new Date();
|
|
5641
|
+
const cohortInfo = getCohortInfo(now, config.cohort.timezone, config.verbose);
|
|
5642
|
+
const transaction = {
|
|
5643
|
+
id: idGenerator(),
|
|
5644
|
+
originalId: id,
|
|
5645
|
+
field: handler.field,
|
|
5646
|
+
value: amount,
|
|
5647
|
+
operation: "add",
|
|
5648
|
+
timestamp: now.toISOString(),
|
|
5649
|
+
cohortDate: cohortInfo.date,
|
|
5650
|
+
cohortHour: cohortInfo.hour,
|
|
5651
|
+
cohortMonth: cohortInfo.month,
|
|
5652
|
+
source: "add",
|
|
5653
|
+
applied: false
|
|
5654
|
+
};
|
|
5655
|
+
await handler.transactionResource.insert(transaction);
|
|
5656
|
+
if (config.mode === "sync") {
|
|
5657
|
+
return await plugin._syncModeConsolidate(handler, id, field);
|
|
5658
|
+
}
|
|
5659
|
+
const [ok, err, record] = await tryFn(() => handler.targetResource.get(id));
|
|
5660
|
+
const currentValue = ok && record ? record[field] || 0 : 0;
|
|
5661
|
+
return currentValue + amount;
|
|
5662
|
+
};
|
|
5663
|
+
resource.sub = async (id, field, amount) => {
|
|
5664
|
+
const { plugin: handler } = resolveFieldAndPlugin(resource, field, amount);
|
|
5665
|
+
const now = /* @__PURE__ */ new Date();
|
|
5666
|
+
const cohortInfo = getCohortInfo(now, config.cohort.timezone, config.verbose);
|
|
5667
|
+
const transaction = {
|
|
5668
|
+
id: idGenerator(),
|
|
5669
|
+
originalId: id,
|
|
5670
|
+
field: handler.field,
|
|
5671
|
+
value: amount,
|
|
5672
|
+
operation: "sub",
|
|
5673
|
+
timestamp: now.toISOString(),
|
|
5674
|
+
cohortDate: cohortInfo.date,
|
|
5675
|
+
cohortHour: cohortInfo.hour,
|
|
5676
|
+
cohortMonth: cohortInfo.month,
|
|
5677
|
+
source: "sub",
|
|
5678
|
+
applied: false
|
|
5679
|
+
};
|
|
5680
|
+
await handler.transactionResource.insert(transaction);
|
|
5681
|
+
if (config.mode === "sync") {
|
|
5682
|
+
return await plugin._syncModeConsolidate(handler, id, field);
|
|
5683
|
+
}
|
|
5684
|
+
const [ok, err, record] = await tryFn(() => handler.targetResource.get(id));
|
|
5685
|
+
const currentValue = ok && record ? record[field] || 0 : 0;
|
|
5686
|
+
return currentValue - amount;
|
|
5687
|
+
};
|
|
5688
|
+
resource.consolidate = async (id, field) => {
|
|
5689
|
+
if (!field) {
|
|
5690
|
+
throw new Error(`Field parameter is required: consolidate(id, field)`);
|
|
5691
|
+
}
|
|
5692
|
+
const handler = resource._eventualConsistencyPlugins[field];
|
|
5693
|
+
if (!handler) {
|
|
5694
|
+
const availableFields = Object.keys(resource._eventualConsistencyPlugins).join(", ");
|
|
5695
|
+
throw new Error(
|
|
5696
|
+
`No eventual consistency plugin found for field "${field}". Available fields: ${availableFields}`
|
|
5147
5697
|
);
|
|
5148
|
-
|
|
5149
|
-
|
|
5150
|
-
|
|
5151
|
-
|
|
5152
|
-
|
|
5153
|
-
|
|
5154
|
-
|
|
5155
|
-
|
|
5156
|
-
|
|
5157
|
-
);
|
|
5158
|
-
}
|
|
5159
|
-
const { results, errors } = await promisePool.PromisePool.for(appliedTransactions).withConcurrency(10).process(async (txn) => {
|
|
5160
|
-
const [deleted] = await tryFn(() => this.transactionResource.delete(txn.id));
|
|
5161
|
-
return deleted;
|
|
5162
|
-
});
|
|
5163
|
-
if (this.config.verbose && errors && errors.length > 0) {
|
|
5164
|
-
console.warn(
|
|
5165
|
-
`[EventualConsistency] ${this.config.resource}.${this.config.field} - Failed to delete ${errors.length} old applied transactions`
|
|
5166
|
-
);
|
|
5167
|
-
}
|
|
5168
|
-
currentValue = 0;
|
|
5169
|
-
} else {
|
|
5170
|
-
appliedTransactions.sort(
|
|
5171
|
-
(a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()
|
|
5172
|
-
);
|
|
5173
|
-
const hasSetInApplied = appliedTransactions.some((t) => t.operation === "set");
|
|
5174
|
-
if (!hasSetInApplied) {
|
|
5175
|
-
const recordValue = recordExists[this.config.field] || 0;
|
|
5176
|
-
let appliedDelta = 0;
|
|
5177
|
-
for (const t of appliedTransactions) {
|
|
5178
|
-
if (t.operation === "add") appliedDelta += t.value;
|
|
5179
|
-
else if (t.operation === "sub") appliedDelta -= t.value;
|
|
5180
|
-
}
|
|
5181
|
-
const baseValue = recordValue - appliedDelta;
|
|
5182
|
-
const hasExistingAnchor = appliedTransactions.some((t) => t.source === "anchor");
|
|
5183
|
-
if (baseValue !== 0 && !hasExistingAnchor) {
|
|
5184
|
-
const firstTransactionDate = new Date(appliedTransactions[0].timestamp);
|
|
5185
|
-
const cohortInfo = this.getCohortInfo(firstTransactionDate);
|
|
5186
|
-
const anchorTransaction = {
|
|
5187
|
-
id: idGenerator(),
|
|
5188
|
-
originalId,
|
|
5189
|
-
field: this.config.field,
|
|
5190
|
-
value: baseValue,
|
|
5191
|
-
operation: "set",
|
|
5192
|
-
timestamp: new Date(firstTransactionDate.getTime() - 1).toISOString(),
|
|
5193
|
-
// 1ms before first txn to ensure it's first
|
|
5194
|
-
cohortDate: cohortInfo.date,
|
|
5195
|
-
cohortHour: cohortInfo.hour,
|
|
5196
|
-
cohortMonth: cohortInfo.month,
|
|
5197
|
-
source: "anchor",
|
|
5198
|
-
applied: true
|
|
5199
|
-
};
|
|
5200
|
-
await this.transactionResource.insert(anchorTransaction);
|
|
5201
|
-
appliedTransactions.unshift(anchorTransaction);
|
|
5202
|
-
}
|
|
5203
|
-
}
|
|
5204
|
-
currentValue = this.config.reducer(appliedTransactions);
|
|
5205
|
-
}
|
|
5206
|
-
} else {
|
|
5207
|
-
const [recordOk, recordErr, record] = await tryFn(
|
|
5208
|
-
() => this.targetResource.get(originalId)
|
|
5209
|
-
);
|
|
5210
|
-
currentValue = recordOk && record ? record[this.config.field] || 0 : 0;
|
|
5211
|
-
if (currentValue !== 0) {
|
|
5212
|
-
let anchorTimestamp;
|
|
5213
|
-
if (transactions && transactions.length > 0) {
|
|
5214
|
-
const firstPendingDate = new Date(transactions[0].timestamp);
|
|
5215
|
-
anchorTimestamp = new Date(firstPendingDate.getTime() - 1).toISOString();
|
|
5216
|
-
} else {
|
|
5217
|
-
anchorTimestamp = (/* @__PURE__ */ new Date()).toISOString();
|
|
5218
|
-
}
|
|
5219
|
-
const cohortInfo = this.getCohortInfo(new Date(anchorTimestamp));
|
|
5220
|
-
const anchorTransaction = {
|
|
5221
|
-
id: idGenerator(),
|
|
5222
|
-
originalId,
|
|
5223
|
-
field: this.config.field,
|
|
5224
|
-
value: currentValue,
|
|
5225
|
-
operation: "set",
|
|
5226
|
-
timestamp: anchorTimestamp,
|
|
5227
|
-
cohortDate: cohortInfo.date,
|
|
5228
|
-
cohortHour: cohortInfo.hour,
|
|
5229
|
-
cohortMonth: cohortInfo.month,
|
|
5230
|
-
source: "anchor",
|
|
5231
|
-
applied: true
|
|
5232
|
-
};
|
|
5233
|
-
await this.transactionResource.insert(anchorTransaction);
|
|
5234
|
-
if (this.config.verbose) {
|
|
5235
|
-
console.log(
|
|
5236
|
-
`[EventualConsistency] ${this.config.resource}.${this.config.field} - Created anchor transaction for ${originalId} with base value ${currentValue}`
|
|
5237
|
-
);
|
|
5238
|
-
}
|
|
5239
|
-
}
|
|
5240
|
-
}
|
|
5241
|
-
if (this.config.verbose) {
|
|
5242
|
-
console.log(
|
|
5243
|
-
`[EventualConsistency] ${this.config.resource}.${this.config.field} - Consolidating ${originalId}: ${transactions.length} pending transactions (current: ${currentValue} from ${appliedOk && appliedTransactions?.length > 0 ? "applied transactions" : "record"})`
|
|
5244
|
-
);
|
|
5245
|
-
}
|
|
5246
|
-
transactions.sort(
|
|
5247
|
-
(a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()
|
|
5698
|
+
}
|
|
5699
|
+
return await plugin._consolidateWithHandler(handler, id);
|
|
5700
|
+
};
|
|
5701
|
+
resource.getConsolidatedValue = async (id, field, options = {}) => {
|
|
5702
|
+
const handler = resource._eventualConsistencyPlugins[field];
|
|
5703
|
+
if (!handler) {
|
|
5704
|
+
const availableFields = Object.keys(resource._eventualConsistencyPlugins).join(", ");
|
|
5705
|
+
throw new Error(
|
|
5706
|
+
`No eventual consistency plugin found for field "${field}". Available fields: ${availableFields}`
|
|
5248
5707
|
);
|
|
5249
|
-
|
|
5250
|
-
|
|
5251
|
-
|
|
5252
|
-
|
|
5253
|
-
|
|
5254
|
-
|
|
5255
|
-
|
|
5256
|
-
|
|
5257
|
-
|
|
5708
|
+
}
|
|
5709
|
+
return await plugin._getConsolidatedValueWithHandler(handler, id, options);
|
|
5710
|
+
};
|
|
5711
|
+
resource.recalculate = async (id, field) => {
|
|
5712
|
+
if (!field) {
|
|
5713
|
+
throw new Error(`Field parameter is required: recalculate(id, field)`);
|
|
5714
|
+
}
|
|
5715
|
+
const handler = resource._eventualConsistencyPlugins[field];
|
|
5716
|
+
if (!handler) {
|
|
5717
|
+
const availableFields = Object.keys(resource._eventualConsistencyPlugins).join(", ");
|
|
5718
|
+
throw new Error(
|
|
5719
|
+
`No eventual consistency plugin found for field "${field}". Available fields: ${availableFields}`
|
|
5720
|
+
);
|
|
5721
|
+
}
|
|
5722
|
+
return await plugin._recalculateWithHandler(handler, id);
|
|
5723
|
+
};
|
|
5724
|
+
}
|
|
5725
|
+
|
|
5726
|
+
async function onSetup(database, fieldHandlers, completeFieldSetupFn, watchForResourceFn) {
|
|
5727
|
+
for (const [resourceName, resourceHandlers] of fieldHandlers) {
|
|
5728
|
+
const targetResource = database.resources[resourceName];
|
|
5729
|
+
if (!targetResource) {
|
|
5730
|
+
for (const handler of resourceHandlers.values()) {
|
|
5731
|
+
handler.deferredSetup = true;
|
|
5258
5732
|
}
|
|
5259
|
-
|
|
5260
|
-
|
|
5261
|
-
|
|
5262
|
-
|
|
5263
|
-
|
|
5264
|
-
|
|
5265
|
-
|
|
5266
|
-
|
|
5267
|
-
|
|
5268
|
-
|
|
5269
|
-
|
|
5270
|
-
|
|
5271
|
-
|
|
5272
|
-
|
|
5273
|
-
|
|
5274
|
-
|
|
5275
|
-
|
|
5276
|
-
|
|
5277
|
-
|
|
5278
|
-
})
|
|
5279
|
-
);
|
|
5280
|
-
if (insertOk) {
|
|
5281
|
-
return insertOk;
|
|
5282
|
-
}
|
|
5283
|
-
if (insertErr?.message?.includes("already exists")) {
|
|
5284
|
-
if (this.config.verbose) {
|
|
5285
|
-
console.log(
|
|
5286
|
-
`[EventualConsistency] ${this.config.resource}.${this.config.field} - Record ${originalId} was created by another consolidation during retry, updating instead`
|
|
5287
|
-
);
|
|
5288
|
-
}
|
|
5289
|
-
return await this.targetResource.update(originalId, {
|
|
5290
|
-
[this.config.field]: consolidatedValue
|
|
5291
|
-
});
|
|
5292
|
-
}
|
|
5293
|
-
throw insertErr;
|
|
5733
|
+
watchForResourceFn(resourceName);
|
|
5734
|
+
continue;
|
|
5735
|
+
}
|
|
5736
|
+
for (const [fieldName, handler] of resourceHandlers) {
|
|
5737
|
+
handler.targetResource = targetResource;
|
|
5738
|
+
await completeFieldSetupFn(handler);
|
|
5739
|
+
}
|
|
5740
|
+
}
|
|
5741
|
+
}
|
|
5742
|
+
function watchForResource(resourceName, database, fieldHandlers, completeFieldSetupFn) {
|
|
5743
|
+
const hookCallback = async ({ resource, config }) => {
|
|
5744
|
+
if (config.name === resourceName) {
|
|
5745
|
+
const resourceHandlers = fieldHandlers.get(resourceName);
|
|
5746
|
+
if (!resourceHandlers) return;
|
|
5747
|
+
for (const [fieldName, handler] of resourceHandlers) {
|
|
5748
|
+
if (handler.deferredSetup) {
|
|
5749
|
+
handler.targetResource = resource;
|
|
5750
|
+
handler.deferredSetup = false;
|
|
5751
|
+
await completeFieldSetupFn(handler);
|
|
5294
5752
|
}
|
|
5295
|
-
throw err2;
|
|
5296
|
-
});
|
|
5297
|
-
if (!updateOk) {
|
|
5298
|
-
console.error(
|
|
5299
|
-
`[EventualConsistency] ${this.config.resource}.${this.config.field} - FAILED to update ${originalId}: ${updateErr?.message || updateErr}`,
|
|
5300
|
-
{ error: updateErr, consolidatedValue, currentValue }
|
|
5301
|
-
);
|
|
5302
|
-
throw updateErr;
|
|
5303
5753
|
}
|
|
5304
|
-
|
|
5305
|
-
|
|
5306
|
-
|
|
5307
|
-
|
|
5308
|
-
|
|
5309
|
-
|
|
5310
|
-
|
|
5311
|
-
|
|
5312
|
-
|
|
5313
|
-
|
|
5314
|
-
|
|
5315
|
-
|
|
5316
|
-
|
|
5317
|
-
|
|
5318
|
-
|
|
5319
|
-
|
|
5320
|
-
|
|
5321
|
-
|
|
5322
|
-
|
|
5323
|
-
|
|
5324
|
-
|
|
5325
|
-
|
|
5326
|
-
|
|
5327
|
-
|
|
5328
|
-
|
|
5329
|
-
|
|
5330
|
-
|
|
5331
|
-
|
|
5332
|
-
|
|
5333
|
-
|
|
5334
|
-
|
|
5335
|
-
|
|
5336
|
-
|
|
5754
|
+
}
|
|
5755
|
+
};
|
|
5756
|
+
database.addHook("afterCreateResource", hookCallback);
|
|
5757
|
+
}
|
|
5758
|
+
async function completeFieldSetup(handler, database, config, plugin) {
|
|
5759
|
+
if (!handler.targetResource) return;
|
|
5760
|
+
const resourceName = handler.resource;
|
|
5761
|
+
const fieldName = handler.field;
|
|
5762
|
+
const transactionResourceName = `${resourceName}_transactions_${fieldName}`;
|
|
5763
|
+
const partitionConfig = createPartitionConfig();
|
|
5764
|
+
const [ok, err, transactionResource] = await tryFn(
|
|
5765
|
+
() => database.createResource({
|
|
5766
|
+
name: transactionResourceName,
|
|
5767
|
+
attributes: {
|
|
5768
|
+
id: "string|required",
|
|
5769
|
+
originalId: "string|required",
|
|
5770
|
+
field: "string|required",
|
|
5771
|
+
value: "number|required",
|
|
5772
|
+
operation: "string|required",
|
|
5773
|
+
timestamp: "string|required",
|
|
5774
|
+
cohortDate: "string|required",
|
|
5775
|
+
cohortHour: "string|required",
|
|
5776
|
+
cohortMonth: "string|optional",
|
|
5777
|
+
source: "string|optional",
|
|
5778
|
+
applied: "boolean|optional"
|
|
5779
|
+
},
|
|
5780
|
+
behavior: "body-overflow",
|
|
5781
|
+
timestamps: true,
|
|
5782
|
+
partitions: partitionConfig,
|
|
5783
|
+
asyncPartitions: true,
|
|
5784
|
+
createdBy: "EventualConsistencyPlugin"
|
|
5785
|
+
})
|
|
5786
|
+
);
|
|
5787
|
+
if (!ok && !database.resources[transactionResourceName]) {
|
|
5788
|
+
throw new Error(`Failed to create transaction resource for ${resourceName}.${fieldName}: ${err?.message}`);
|
|
5789
|
+
}
|
|
5790
|
+
handler.transactionResource = ok ? transactionResource : database.resources[transactionResourceName];
|
|
5791
|
+
const lockResourceName = `${resourceName}_consolidation_locks_${fieldName}`;
|
|
5792
|
+
const [lockOk, lockErr, lockResource] = await tryFn(
|
|
5793
|
+
() => database.createResource({
|
|
5794
|
+
name: lockResourceName,
|
|
5795
|
+
attributes: {
|
|
5796
|
+
id: "string|required",
|
|
5797
|
+
lockedAt: "number|required",
|
|
5798
|
+
workerId: "string|optional"
|
|
5799
|
+
},
|
|
5800
|
+
behavior: "body-only",
|
|
5801
|
+
timestamps: false,
|
|
5802
|
+
createdBy: "EventualConsistencyPlugin"
|
|
5803
|
+
})
|
|
5804
|
+
);
|
|
5805
|
+
if (!lockOk && !database.resources[lockResourceName]) {
|
|
5806
|
+
throw new Error(`Failed to create lock resource for ${resourceName}.${fieldName}: ${lockErr?.message}`);
|
|
5807
|
+
}
|
|
5808
|
+
handler.lockResource = lockOk ? lockResource : database.resources[lockResourceName];
|
|
5809
|
+
if (config.enableAnalytics) {
|
|
5810
|
+
await createAnalyticsResource(handler, database, resourceName, fieldName);
|
|
5811
|
+
}
|
|
5812
|
+
addHelperMethodsForHandler(handler, plugin, config);
|
|
5813
|
+
if (config.verbose) {
|
|
5814
|
+
console.log(
|
|
5815
|
+
`[EventualConsistency] ${resourceName}.${fieldName} - Setup complete. Resources: ${transactionResourceName}, ${lockResourceName}${config.enableAnalytics ? `, ${resourceName}_analytics_${fieldName}` : ""}`
|
|
5816
|
+
);
|
|
5817
|
+
}
|
|
5818
|
+
}
|
|
5819
|
+
async function createAnalyticsResource(handler, database, resourceName, fieldName) {
|
|
5820
|
+
const analyticsResourceName = `${resourceName}_analytics_${fieldName}`;
|
|
5821
|
+
const [ok, err, analyticsResource] = await tryFn(
|
|
5822
|
+
() => database.createResource({
|
|
5823
|
+
name: analyticsResourceName,
|
|
5824
|
+
attributes: {
|
|
5825
|
+
id: "string|required",
|
|
5826
|
+
period: "string|required",
|
|
5827
|
+
cohort: "string|required",
|
|
5828
|
+
transactionCount: "number|required",
|
|
5829
|
+
totalValue: "number|required",
|
|
5830
|
+
avgValue: "number|required",
|
|
5831
|
+
minValue: "number|required",
|
|
5832
|
+
maxValue: "number|required",
|
|
5833
|
+
operations: "object|optional",
|
|
5834
|
+
recordCount: "number|required",
|
|
5835
|
+
consolidatedAt: "string|required",
|
|
5836
|
+
updatedAt: "string|required"
|
|
5837
|
+
},
|
|
5838
|
+
behavior: "body-overflow",
|
|
5839
|
+
timestamps: false,
|
|
5840
|
+
createdBy: "EventualConsistencyPlugin"
|
|
5841
|
+
})
|
|
5842
|
+
);
|
|
5843
|
+
if (!ok && !database.resources[analyticsResourceName]) {
|
|
5844
|
+
throw new Error(`Failed to create analytics resource for ${resourceName}.${fieldName}: ${err?.message}`);
|
|
5845
|
+
}
|
|
5846
|
+
handler.analyticsResource = ok ? analyticsResource : database.resources[analyticsResourceName];
|
|
5847
|
+
}
|
|
5848
|
+
function addHelperMethodsForHandler(handler, plugin, config) {
|
|
5849
|
+
const resource = handler.targetResource;
|
|
5850
|
+
const fieldName = handler.field;
|
|
5851
|
+
if (!resource._eventualConsistencyPlugins) {
|
|
5852
|
+
resource._eventualConsistencyPlugins = {};
|
|
5853
|
+
}
|
|
5854
|
+
resource._eventualConsistencyPlugins[fieldName] = handler;
|
|
5855
|
+
if (!resource.add) {
|
|
5856
|
+
addHelperMethods(resource, plugin, config);
|
|
5857
|
+
}
|
|
5858
|
+
}
|
|
5859
|
+
async function onStart(fieldHandlers, config, runConsolidationFn, runGCFn, emitFn) {
|
|
5860
|
+
for (const [resourceName, resourceHandlers] of fieldHandlers) {
|
|
5861
|
+
for (const [fieldName, handler] of resourceHandlers) {
|
|
5862
|
+
if (!handler.deferredSetup) {
|
|
5863
|
+
if (config.autoConsolidate && config.mode === "async") {
|
|
5864
|
+
startConsolidationTimer(handler, resourceName, fieldName, runConsolidationFn, config);
|
|
5865
|
+
}
|
|
5866
|
+
if (config.transactionRetention && config.transactionRetention > 0) {
|
|
5867
|
+
startGarbageCollectionTimer(handler, resourceName, fieldName, runGCFn, config);
|
|
5868
|
+
}
|
|
5869
|
+
if (emitFn) {
|
|
5870
|
+
emitFn("eventual-consistency.started", {
|
|
5871
|
+
resource: resourceName,
|
|
5872
|
+
field: fieldName,
|
|
5873
|
+
cohort: config.cohort
|
|
5874
|
+
});
|
|
5337
5875
|
}
|
|
5338
5876
|
}
|
|
5339
|
-
return consolidatedValue;
|
|
5340
|
-
} finally {
|
|
5341
|
-
const [lockReleased, lockReleaseErr] = await tryFn(() => this.lockResource.delete(lockId));
|
|
5342
|
-
if (!lockReleased && this.config.verbose) {
|
|
5343
|
-
console.warn(`[EventualConsistency] Failed to release lock ${lockId}:`, lockReleaseErr?.message);
|
|
5344
|
-
}
|
|
5345
5877
|
}
|
|
5346
5878
|
}
|
|
5347
|
-
|
|
5348
|
-
|
|
5349
|
-
|
|
5350
|
-
const
|
|
5351
|
-
|
|
5352
|
-
|
|
5353
|
-
|
|
5354
|
-
|
|
5355
|
-
|
|
5356
|
-
|
|
5357
|
-
|
|
5358
|
-
|
|
5359
|
-
|
|
5360
|
-
|
|
5361
|
-
|
|
5362
|
-
if (
|
|
5363
|
-
|
|
5879
|
+
}
|
|
5880
|
+
async function onStop(fieldHandlers, emitFn) {
|
|
5881
|
+
for (const [resourceName, resourceHandlers] of fieldHandlers) {
|
|
5882
|
+
for (const [fieldName, handler] of resourceHandlers) {
|
|
5883
|
+
if (handler.consolidationTimer) {
|
|
5884
|
+
clearInterval(handler.consolidationTimer);
|
|
5885
|
+
handler.consolidationTimer = null;
|
|
5886
|
+
}
|
|
5887
|
+
if (handler.gcTimer) {
|
|
5888
|
+
clearInterval(handler.gcTimer);
|
|
5889
|
+
handler.gcTimer = null;
|
|
5890
|
+
}
|
|
5891
|
+
if (handler.pendingTransactions && handler.pendingTransactions.size > 0) {
|
|
5892
|
+
await flushPendingTransactions(handler);
|
|
5893
|
+
}
|
|
5894
|
+
if (emitFn) {
|
|
5895
|
+
emitFn("eventual-consistency.stopped", {
|
|
5896
|
+
resource: resourceName,
|
|
5897
|
+
field: fieldName
|
|
5898
|
+
});
|
|
5364
5899
|
}
|
|
5365
|
-
return 0;
|
|
5366
5900
|
}
|
|
5367
|
-
|
|
5368
|
-
|
|
5369
|
-
|
|
5370
|
-
|
|
5371
|
-
|
|
5372
|
-
|
|
5373
|
-
|
|
5374
|
-
|
|
5375
|
-
|
|
5376
|
-
|
|
5377
|
-
|
|
5378
|
-
)
|
|
5379
|
-
|
|
5380
|
-
|
|
5381
|
-
|
|
5382
|
-
|
|
5901
|
+
}
|
|
5902
|
+
}
|
|
5903
|
+
|
|
5904
|
+
class EventualConsistencyPlugin extends Plugin {
|
|
5905
|
+
constructor(options = {}) {
|
|
5906
|
+
super(options);
|
|
5907
|
+
validateResourcesConfig(options.resources);
|
|
5908
|
+
const detectedTimezone = detectTimezone();
|
|
5909
|
+
const timezoneAutoDetected = !options.cohort?.timezone;
|
|
5910
|
+
this.config = createConfig(options, detectedTimezone);
|
|
5911
|
+
this.fieldHandlers = /* @__PURE__ */ new Map();
|
|
5912
|
+
for (const [resourceName, fields] of Object.entries(options.resources)) {
|
|
5913
|
+
const resourceHandlers = /* @__PURE__ */ new Map();
|
|
5914
|
+
for (const fieldName of fields) {
|
|
5915
|
+
resourceHandlers.set(fieldName, createFieldHandler(resourceName, fieldName));
|
|
5916
|
+
}
|
|
5917
|
+
this.fieldHandlers.set(resourceName, resourceHandlers);
|
|
5383
5918
|
}
|
|
5384
|
-
|
|
5385
|
-
|
|
5919
|
+
logConfigWarnings(this.config);
|
|
5920
|
+
logInitialization(this.config, this.fieldHandlers, timezoneAutoDetected);
|
|
5921
|
+
}
|
|
5922
|
+
/**
|
|
5923
|
+
* Setup hook - create resources and register helpers
|
|
5924
|
+
*/
|
|
5925
|
+
async onSetup() {
|
|
5926
|
+
await onSetup(
|
|
5927
|
+
this.database,
|
|
5928
|
+
this.fieldHandlers,
|
|
5929
|
+
(handler) => completeFieldSetup(handler, this.database, this.config, this),
|
|
5930
|
+
(resourceName) => watchForResource(
|
|
5931
|
+
resourceName,
|
|
5932
|
+
this.database,
|
|
5933
|
+
this.fieldHandlers,
|
|
5934
|
+
(handler) => completeFieldSetup(handler, this.database, this.config, this)
|
|
5935
|
+
)
|
|
5386
5936
|
);
|
|
5387
|
-
return this.config.reducer(filtered);
|
|
5388
5937
|
}
|
|
5389
|
-
|
|
5390
|
-
|
|
5391
|
-
|
|
5392
|
-
|
|
5393
|
-
|
|
5394
|
-
|
|
5938
|
+
/**
|
|
5939
|
+
* Start hook - begin timers and emit events
|
|
5940
|
+
*/
|
|
5941
|
+
async onStart() {
|
|
5942
|
+
await onStart(
|
|
5943
|
+
this.fieldHandlers,
|
|
5944
|
+
this.config,
|
|
5945
|
+
(handler, resourceName, fieldName) => this._runConsolidationForHandler(handler, resourceName, fieldName),
|
|
5946
|
+
(handler, resourceName, fieldName) => this._runGarbageCollectionForHandler(handler, resourceName, fieldName),
|
|
5947
|
+
(event, data) => this.emit(event, data)
|
|
5395
5948
|
);
|
|
5396
|
-
if (!ok) return null;
|
|
5397
|
-
const stats = {
|
|
5398
|
-
date: cohortDate,
|
|
5399
|
-
transactionCount: transactions.length,
|
|
5400
|
-
totalValue: 0,
|
|
5401
|
-
byOperation: { set: 0, add: 0, sub: 0 },
|
|
5402
|
-
byOriginalId: {}
|
|
5403
|
-
};
|
|
5404
|
-
for (const txn of transactions) {
|
|
5405
|
-
stats.totalValue += txn.value || 0;
|
|
5406
|
-
stats.byOperation[txn.operation] = (stats.byOperation[txn.operation] || 0) + 1;
|
|
5407
|
-
if (!stats.byOriginalId[txn.originalId]) {
|
|
5408
|
-
stats.byOriginalId[txn.originalId] = {
|
|
5409
|
-
count: 0,
|
|
5410
|
-
value: 0
|
|
5411
|
-
};
|
|
5412
|
-
}
|
|
5413
|
-
stats.byOriginalId[txn.originalId].count++;
|
|
5414
|
-
stats.byOriginalId[txn.originalId].value += txn.value || 0;
|
|
5415
|
-
}
|
|
5416
|
-
return stats;
|
|
5417
5949
|
}
|
|
5418
5950
|
/**
|
|
5419
|
-
*
|
|
5420
|
-
* Uses distributed locking to prevent multiple containers from cleaning simultaneously
|
|
5951
|
+
* Stop hook - stop timers and flush pending
|
|
5421
5952
|
*/
|
|
5422
|
-
async
|
|
5423
|
-
|
|
5424
|
-
|
|
5425
|
-
|
|
5426
|
-
const cleanupLockId = `lock-cleanup-${this.config.resource}-${this.config.field}`;
|
|
5427
|
-
const [lockAcquired] = await tryFn(
|
|
5428
|
-
() => this.lockResource.insert({
|
|
5429
|
-
id: cleanupLockId,
|
|
5430
|
-
lockedAt: Date.now(),
|
|
5431
|
-
workerId: process.pid ? String(process.pid) : "unknown"
|
|
5432
|
-
})
|
|
5953
|
+
async onStop() {
|
|
5954
|
+
await onStop(
|
|
5955
|
+
this.fieldHandlers,
|
|
5956
|
+
(event, data) => this.emit(event, data)
|
|
5433
5957
|
);
|
|
5434
|
-
if (!lockAcquired) {
|
|
5435
|
-
if (this.config.verbose) {
|
|
5436
|
-
console.log(`[EventualConsistency] Lock cleanup already running in another container`);
|
|
5437
|
-
}
|
|
5438
|
-
return;
|
|
5439
|
-
}
|
|
5440
|
-
try {
|
|
5441
|
-
const [ok, err, locks] = await tryFn(() => this.lockResource.list());
|
|
5442
|
-
if (!ok || !locks || locks.length === 0) return;
|
|
5443
|
-
const staleLocks = locks.filter(
|
|
5444
|
-
(lock) => lock.id !== cleanupLockId && lock.lockedAt < cutoffTime
|
|
5445
|
-
);
|
|
5446
|
-
if (staleLocks.length === 0) return;
|
|
5447
|
-
if (this.config.verbose) {
|
|
5448
|
-
console.log(`[EventualConsistency] Cleaning up ${staleLocks.length} stale locks`);
|
|
5449
|
-
}
|
|
5450
|
-
const { results, errors } = await promisePool.PromisePool.for(staleLocks).withConcurrency(5).process(async (lock) => {
|
|
5451
|
-
const [deleted] = await tryFn(() => this.lockResource.delete(lock.id));
|
|
5452
|
-
return deleted;
|
|
5453
|
-
});
|
|
5454
|
-
if (errors && errors.length > 0 && this.config.verbose) {
|
|
5455
|
-
console.warn(`[EventualConsistency] ${errors.length} stale locks failed to delete`);
|
|
5456
|
-
}
|
|
5457
|
-
} catch (error) {
|
|
5458
|
-
if (this.config.verbose) {
|
|
5459
|
-
console.warn(`[EventualConsistency] Error cleaning up stale locks:`, error.message);
|
|
5460
|
-
}
|
|
5461
|
-
} finally {
|
|
5462
|
-
await tryFn(() => this.lockResource.delete(cleanupLockId));
|
|
5463
|
-
}
|
|
5464
5958
|
}
|
|
5465
5959
|
/**
|
|
5466
|
-
*
|
|
5960
|
+
* Create partition configuration
|
|
5961
|
+
* @returns {Object} Partition configuration
|
|
5962
|
+
*/
|
|
5963
|
+
createPartitionConfig() {
|
|
5964
|
+
return createPartitionConfig();
|
|
5965
|
+
}
|
|
5966
|
+
/**
|
|
5967
|
+
* Get cohort information for a date
|
|
5968
|
+
* @param {Date} date - Date to get cohort info for
|
|
5969
|
+
* @returns {Object} Cohort information
|
|
5970
|
+
*/
|
|
5971
|
+
getCohortInfo(date) {
|
|
5972
|
+
return getCohortInfo(date, this.config.cohort.timezone, this.config.verbose);
|
|
5973
|
+
}
|
|
5974
|
+
/**
|
|
5975
|
+
* Create a transaction for a field handler
|
|
5976
|
+
* @param {Object} handler - Field handler
|
|
5977
|
+
* @param {Object} data - Transaction data
|
|
5978
|
+
* @returns {Promise<Object|null>} Created transaction
|
|
5979
|
+
*/
|
|
5980
|
+
async createTransaction(handler, data) {
|
|
5981
|
+
return await createTransaction(handler, data, this.config);
|
|
5982
|
+
}
|
|
5983
|
+
/**
|
|
5984
|
+
* Consolidate a single record (internal method)
|
|
5985
|
+
* This is used internally by consolidation timers and helper methods
|
|
5986
|
+
* @private
|
|
5467
5987
|
*/
|
|
5468
|
-
|
|
5469
|
-
|
|
5470
|
-
|
|
5471
|
-
|
|
5472
|
-
|
|
5988
|
+
async consolidateRecord(originalId) {
|
|
5989
|
+
return await consolidateRecord(
|
|
5990
|
+
originalId,
|
|
5991
|
+
this.transactionResource,
|
|
5992
|
+
this.targetResource,
|
|
5993
|
+
this.lockResource,
|
|
5994
|
+
this.analyticsResource,
|
|
5995
|
+
(transactions) => this.updateAnalytics(transactions),
|
|
5996
|
+
this.config
|
|
5997
|
+
);
|
|
5473
5998
|
}
|
|
5474
|
-
|
|
5475
|
-
|
|
5476
|
-
|
|
5477
|
-
|
|
5478
|
-
|
|
5999
|
+
/**
|
|
6000
|
+
* Get consolidated value without applying (internal method)
|
|
6001
|
+
* @private
|
|
6002
|
+
*/
|
|
6003
|
+
async getConsolidatedValue(originalId, options = {}) {
|
|
6004
|
+
return await getConsolidatedValue(
|
|
6005
|
+
originalId,
|
|
6006
|
+
options,
|
|
6007
|
+
this.transactionResource,
|
|
6008
|
+
this.targetResource,
|
|
6009
|
+
this.config
|
|
6010
|
+
);
|
|
5479
6011
|
}
|
|
5480
|
-
|
|
5481
|
-
|
|
5482
|
-
|
|
5483
|
-
|
|
5484
|
-
|
|
5485
|
-
|
|
5486
|
-
this.
|
|
5487
|
-
this.config.field = fieldName;
|
|
5488
|
-
this.transactionResource = handler.transactionResource;
|
|
5489
|
-
this.targetResource = handler.targetResource;
|
|
5490
|
-
this.lockResource = handler.lockResource;
|
|
5491
|
-
try {
|
|
5492
|
-
await this.runGarbageCollection();
|
|
5493
|
-
} finally {
|
|
5494
|
-
this.config.resource = oldResource;
|
|
5495
|
-
this.config.field = oldField;
|
|
5496
|
-
this.transactionResource = oldTransactionResource;
|
|
5497
|
-
this.targetResource = oldTargetResource;
|
|
5498
|
-
this.lockResource = oldLockResource;
|
|
5499
|
-
}
|
|
6012
|
+
/**
|
|
6013
|
+
* Get cohort statistics
|
|
6014
|
+
* @param {string} cohortDate - Cohort date
|
|
6015
|
+
* @returns {Promise<Object|null>} Cohort statistics
|
|
6016
|
+
*/
|
|
6017
|
+
async getCohortStats(cohortDate) {
|
|
6018
|
+
return await getCohortStats(cohortDate, this.transactionResource);
|
|
5500
6019
|
}
|
|
5501
6020
|
/**
|
|
5502
|
-
*
|
|
5503
|
-
*
|
|
6021
|
+
* Recalculate from scratch (internal method)
|
|
6022
|
+
* @private
|
|
5504
6023
|
*/
|
|
5505
|
-
async
|
|
5506
|
-
|
|
5507
|
-
|
|
5508
|
-
|
|
5509
|
-
|
|
5510
|
-
|
|
5511
|
-
|
|
5512
|
-
|
|
6024
|
+
async recalculateRecord(originalId) {
|
|
6025
|
+
return await recalculateRecord(
|
|
6026
|
+
originalId,
|
|
6027
|
+
this.transactionResource,
|
|
6028
|
+
this.targetResource,
|
|
6029
|
+
this.lockResource,
|
|
6030
|
+
(id) => this.consolidateRecord(id),
|
|
6031
|
+
this.config
|
|
5513
6032
|
);
|
|
5514
|
-
if (!lockAcquired) {
|
|
5515
|
-
if (this.config.verbose) {
|
|
5516
|
-
console.log(`[EventualConsistency] GC already running in another container`);
|
|
5517
|
-
}
|
|
5518
|
-
return;
|
|
5519
|
-
}
|
|
5520
|
-
try {
|
|
5521
|
-
const now = Date.now();
|
|
5522
|
-
const retentionMs = this.config.transactionRetention * 24 * 60 * 60 * 1e3;
|
|
5523
|
-
const cutoffDate = new Date(now - retentionMs);
|
|
5524
|
-
const cutoffIso = cutoffDate.toISOString();
|
|
5525
|
-
if (this.config.verbose) {
|
|
5526
|
-
console.log(`[EventualConsistency] Running GC for transactions older than ${cutoffIso} (${this.config.transactionRetention} days)`);
|
|
5527
|
-
}
|
|
5528
|
-
const cutoffMonth = cutoffDate.toISOString().substring(0, 7);
|
|
5529
|
-
const [ok, err, oldTransactions] = await tryFn(
|
|
5530
|
-
() => this.transactionResource.query({
|
|
5531
|
-
applied: true,
|
|
5532
|
-
timestamp: { "<": cutoffIso }
|
|
5533
|
-
})
|
|
5534
|
-
);
|
|
5535
|
-
if (!ok) {
|
|
5536
|
-
if (this.config.verbose) {
|
|
5537
|
-
console.warn(`[EventualConsistency] GC failed to query transactions:`, err?.message);
|
|
5538
|
-
}
|
|
5539
|
-
return;
|
|
5540
|
-
}
|
|
5541
|
-
if (!oldTransactions || oldTransactions.length === 0) {
|
|
5542
|
-
if (this.config.verbose) {
|
|
5543
|
-
console.log(`[EventualConsistency] No old transactions to clean up`);
|
|
5544
|
-
}
|
|
5545
|
-
return;
|
|
5546
|
-
}
|
|
5547
|
-
if (this.config.verbose) {
|
|
5548
|
-
console.log(`[EventualConsistency] Deleting ${oldTransactions.length} old transactions`);
|
|
5549
|
-
}
|
|
5550
|
-
const { results, errors } = await promisePool.PromisePool.for(oldTransactions).withConcurrency(10).process(async (txn) => {
|
|
5551
|
-
const [deleted] = await tryFn(() => this.transactionResource.delete(txn.id));
|
|
5552
|
-
return deleted;
|
|
5553
|
-
});
|
|
5554
|
-
if (this.config.verbose) {
|
|
5555
|
-
console.log(`[EventualConsistency] GC completed: ${results.length} deleted, ${errors.length} errors`);
|
|
5556
|
-
}
|
|
5557
|
-
this.emit("eventual-consistency.gc-completed", {
|
|
5558
|
-
resource: this.config.resource,
|
|
5559
|
-
field: this.config.field,
|
|
5560
|
-
deletedCount: results.length,
|
|
5561
|
-
errorCount: errors.length
|
|
5562
|
-
});
|
|
5563
|
-
} catch (error) {
|
|
5564
|
-
if (this.config.verbose) {
|
|
5565
|
-
console.warn(`[EventualConsistency] GC error:`, error.message);
|
|
5566
|
-
}
|
|
5567
|
-
this.emit("eventual-consistency.gc-error", error);
|
|
5568
|
-
} finally {
|
|
5569
|
-
await tryFn(() => this.lockResource.delete(gcLockId));
|
|
5570
|
-
}
|
|
5571
6033
|
}
|
|
5572
6034
|
/**
|
|
5573
|
-
* Update analytics
|
|
5574
|
-
* @param {Array} transactions - Array of transactions that were just consolidated
|
|
6035
|
+
* Update analytics
|
|
5575
6036
|
* @private
|
|
5576
6037
|
*/
|
|
5577
6038
|
async updateAnalytics(transactions) {
|
|
5578
|
-
|
|
5579
|
-
if (this.config.verbose) {
|
|
5580
|
-
console.log(
|
|
5581
|
-
`[EventualConsistency] ${this.config.resource}.${this.config.field} - Updating analytics for ${transactions.length} transactions...`
|
|
5582
|
-
);
|
|
5583
|
-
}
|
|
5584
|
-
try {
|
|
5585
|
-
const byHour = this._groupByCohort(transactions, "cohortHour");
|
|
5586
|
-
const cohortCount = Object.keys(byHour).length;
|
|
5587
|
-
if (this.config.verbose) {
|
|
5588
|
-
console.log(
|
|
5589
|
-
`[EventualConsistency] ${this.config.resource}.${this.config.field} - Updating ${cohortCount} hourly analytics cohorts...`
|
|
5590
|
-
);
|
|
5591
|
-
}
|
|
5592
|
-
for (const [cohort, txns] of Object.entries(byHour)) {
|
|
5593
|
-
await this._upsertAnalytics("hour", cohort, txns);
|
|
5594
|
-
}
|
|
5595
|
-
if (this.config.analyticsConfig.rollupStrategy === "incremental") {
|
|
5596
|
-
const uniqueHours = Object.keys(byHour);
|
|
5597
|
-
if (this.config.verbose) {
|
|
5598
|
-
console.log(
|
|
5599
|
-
`[EventualConsistency] ${this.config.resource}.${this.config.field} - Rolling up ${uniqueHours.length} hours to daily/monthly analytics...`
|
|
5600
|
-
);
|
|
5601
|
-
}
|
|
5602
|
-
for (const cohortHour of uniqueHours) {
|
|
5603
|
-
await this._rollupAnalytics(cohortHour);
|
|
5604
|
-
}
|
|
5605
|
-
}
|
|
5606
|
-
if (this.config.verbose) {
|
|
5607
|
-
console.log(
|
|
5608
|
-
`[EventualConsistency] ${this.config.resource}.${this.config.field} - Analytics update complete for ${cohortCount} cohorts`
|
|
5609
|
-
);
|
|
5610
|
-
}
|
|
5611
|
-
} catch (error) {
|
|
5612
|
-
console.warn(
|
|
5613
|
-
`[EventualConsistency] ${this.config.resource}.${this.config.field} - Analytics update error:`,
|
|
5614
|
-
error.message
|
|
5615
|
-
);
|
|
5616
|
-
}
|
|
6039
|
+
return await updateAnalytics(transactions, this.analyticsResource, this.config);
|
|
5617
6040
|
}
|
|
5618
6041
|
/**
|
|
5619
|
-
*
|
|
6042
|
+
* Helper method for sync mode consolidation
|
|
5620
6043
|
* @private
|
|
5621
6044
|
*/
|
|
5622
|
-
|
|
5623
|
-
const
|
|
5624
|
-
|
|
5625
|
-
|
|
5626
|
-
|
|
5627
|
-
|
|
5628
|
-
|
|
5629
|
-
|
|
5630
|
-
|
|
5631
|
-
|
|
5632
|
-
|
|
6045
|
+
async _syncModeConsolidate(handler, id, field) {
|
|
6046
|
+
const oldResource = this.config.resource;
|
|
6047
|
+
const oldField = this.config.field;
|
|
6048
|
+
const oldTransactionResource = this.transactionResource;
|
|
6049
|
+
const oldTargetResource = this.targetResource;
|
|
6050
|
+
const oldLockResource = this.lockResource;
|
|
6051
|
+
const oldAnalyticsResource = this.analyticsResource;
|
|
6052
|
+
this.config.resource = handler.resource;
|
|
6053
|
+
this.config.field = handler.field;
|
|
6054
|
+
this.transactionResource = handler.transactionResource;
|
|
6055
|
+
this.targetResource = handler.targetResource;
|
|
6056
|
+
this.lockResource = handler.lockResource;
|
|
6057
|
+
this.analyticsResource = handler.analyticsResource;
|
|
6058
|
+
const result = await this.consolidateRecord(id);
|
|
6059
|
+
this.config.resource = oldResource;
|
|
6060
|
+
this.config.field = oldField;
|
|
6061
|
+
this.transactionResource = oldTransactionResource;
|
|
6062
|
+
this.targetResource = oldTargetResource;
|
|
6063
|
+
this.lockResource = oldLockResource;
|
|
6064
|
+
this.analyticsResource = oldAnalyticsResource;
|
|
6065
|
+
return result;
|
|
5633
6066
|
}
|
|
5634
6067
|
/**
|
|
5635
|
-
*
|
|
6068
|
+
* Helper method for consolidate with handler
|
|
5636
6069
|
* @private
|
|
5637
6070
|
*/
|
|
5638
|
-
async
|
|
5639
|
-
const
|
|
5640
|
-
const
|
|
5641
|
-
const
|
|
5642
|
-
|
|
5643
|
-
|
|
5644
|
-
|
|
5645
|
-
|
|
5646
|
-
|
|
5647
|
-
|
|
5648
|
-
|
|
5649
|
-
|
|
5650
|
-
|
|
5651
|
-
const
|
|
5652
|
-
|
|
5653
|
-
|
|
5654
|
-
|
|
5655
|
-
|
|
5656
|
-
|
|
5657
|
-
|
|
5658
|
-
|
|
5659
|
-
const newMinValue = Math.min(existing.minValue, minValue);
|
|
5660
|
-
const newMaxValue = Math.max(existing.maxValue, maxValue);
|
|
5661
|
-
const newOperations = { ...existing.operations };
|
|
5662
|
-
for (const [op, stats] of Object.entries(operations)) {
|
|
5663
|
-
if (!newOperations[op]) {
|
|
5664
|
-
newOperations[op] = { count: 0, sum: 0 };
|
|
5665
|
-
}
|
|
5666
|
-
newOperations[op].count += stats.count;
|
|
5667
|
-
newOperations[op].sum += stats.sum;
|
|
5668
|
-
}
|
|
5669
|
-
const newRecordCount = Math.max(existing.recordCount, recordCount);
|
|
5670
|
-
await tryFn(
|
|
5671
|
-
() => this.analyticsResource.update(id, {
|
|
5672
|
-
transactionCount: newTransactionCount,
|
|
5673
|
-
totalValue: newTotalValue,
|
|
5674
|
-
avgValue: newAvgValue,
|
|
5675
|
-
minValue: newMinValue,
|
|
5676
|
-
maxValue: newMaxValue,
|
|
5677
|
-
operations: newOperations,
|
|
5678
|
-
recordCount: newRecordCount,
|
|
5679
|
-
updatedAt: now
|
|
5680
|
-
})
|
|
5681
|
-
);
|
|
5682
|
-
} else {
|
|
5683
|
-
await tryFn(
|
|
5684
|
-
() => this.analyticsResource.insert({
|
|
5685
|
-
id,
|
|
5686
|
-
period,
|
|
5687
|
-
cohort,
|
|
5688
|
-
transactionCount,
|
|
5689
|
-
totalValue,
|
|
5690
|
-
avgValue,
|
|
5691
|
-
minValue,
|
|
5692
|
-
maxValue,
|
|
5693
|
-
operations,
|
|
5694
|
-
recordCount,
|
|
5695
|
-
consolidatedAt: now,
|
|
5696
|
-
updatedAt: now
|
|
5697
|
-
})
|
|
5698
|
-
);
|
|
5699
|
-
}
|
|
6071
|
+
async _consolidateWithHandler(handler, id) {
|
|
6072
|
+
const oldResource = this.config.resource;
|
|
6073
|
+
const oldField = this.config.field;
|
|
6074
|
+
const oldTransactionResource = this.transactionResource;
|
|
6075
|
+
const oldTargetResource = this.targetResource;
|
|
6076
|
+
const oldLockResource = this.lockResource;
|
|
6077
|
+
const oldAnalyticsResource = this.analyticsResource;
|
|
6078
|
+
this.config.resource = handler.resource;
|
|
6079
|
+
this.config.field = handler.field;
|
|
6080
|
+
this.transactionResource = handler.transactionResource;
|
|
6081
|
+
this.targetResource = handler.targetResource;
|
|
6082
|
+
this.lockResource = handler.lockResource;
|
|
6083
|
+
this.analyticsResource = handler.analyticsResource;
|
|
6084
|
+
const result = await this.consolidateRecord(id);
|
|
6085
|
+
this.config.resource = oldResource;
|
|
6086
|
+
this.config.field = oldField;
|
|
6087
|
+
this.transactionResource = oldTransactionResource;
|
|
6088
|
+
this.targetResource = oldTargetResource;
|
|
6089
|
+
this.lockResource = oldLockResource;
|
|
6090
|
+
this.analyticsResource = oldAnalyticsResource;
|
|
6091
|
+
return result;
|
|
5700
6092
|
}
|
|
5701
6093
|
/**
|
|
5702
|
-
*
|
|
6094
|
+
* Helper method for getConsolidatedValue with handler
|
|
5703
6095
|
* @private
|
|
5704
6096
|
*/
|
|
5705
|
-
|
|
5706
|
-
const
|
|
5707
|
-
|
|
5708
|
-
|
|
5709
|
-
|
|
5710
|
-
|
|
5711
|
-
|
|
5712
|
-
|
|
5713
|
-
|
|
5714
|
-
|
|
5715
|
-
|
|
5716
|
-
|
|
6097
|
+
async _getConsolidatedValueWithHandler(handler, id, options) {
|
|
6098
|
+
const oldResource = this.config.resource;
|
|
6099
|
+
const oldField = this.config.field;
|
|
6100
|
+
const oldTransactionResource = this.transactionResource;
|
|
6101
|
+
const oldTargetResource = this.targetResource;
|
|
6102
|
+
this.config.resource = handler.resource;
|
|
6103
|
+
this.config.field = handler.field;
|
|
6104
|
+
this.transactionResource = handler.transactionResource;
|
|
6105
|
+
this.targetResource = handler.targetResource;
|
|
6106
|
+
const result = await this.getConsolidatedValue(id, options);
|
|
6107
|
+
this.config.resource = oldResource;
|
|
6108
|
+
this.config.field = oldField;
|
|
6109
|
+
this.transactionResource = oldTransactionResource;
|
|
6110
|
+
this.targetResource = oldTargetResource;
|
|
6111
|
+
return result;
|
|
5717
6112
|
}
|
|
5718
6113
|
/**
|
|
5719
|
-
*
|
|
6114
|
+
* Helper method for recalculate with handler
|
|
5720
6115
|
* @private
|
|
5721
6116
|
*/
|
|
5722
|
-
async
|
|
5723
|
-
const
|
|
5724
|
-
const
|
|
5725
|
-
|
|
5726
|
-
|
|
6117
|
+
async _recalculateWithHandler(handler, id) {
|
|
6118
|
+
const oldResource = this.config.resource;
|
|
6119
|
+
const oldField = this.config.field;
|
|
6120
|
+
const oldTransactionResource = this.transactionResource;
|
|
6121
|
+
const oldTargetResource = this.targetResource;
|
|
6122
|
+
const oldLockResource = this.lockResource;
|
|
6123
|
+
const oldAnalyticsResource = this.analyticsResource;
|
|
6124
|
+
this.config.resource = handler.resource;
|
|
6125
|
+
this.config.field = handler.field;
|
|
6126
|
+
this.transactionResource = handler.transactionResource;
|
|
6127
|
+
this.targetResource = handler.targetResource;
|
|
6128
|
+
this.lockResource = handler.lockResource;
|
|
6129
|
+
this.analyticsResource = handler.analyticsResource;
|
|
6130
|
+
const result = await this.recalculateRecord(id);
|
|
6131
|
+
this.config.resource = oldResource;
|
|
6132
|
+
this.config.field = oldField;
|
|
6133
|
+
this.transactionResource = oldTransactionResource;
|
|
6134
|
+
this.targetResource = oldTargetResource;
|
|
6135
|
+
this.lockResource = oldLockResource;
|
|
6136
|
+
this.analyticsResource = oldAnalyticsResource;
|
|
6137
|
+
return result;
|
|
5727
6138
|
}
|
|
5728
6139
|
/**
|
|
5729
|
-
*
|
|
6140
|
+
* Run consolidation for a handler
|
|
5730
6141
|
* @private
|
|
5731
6142
|
*/
|
|
5732
|
-
async
|
|
5733
|
-
const
|
|
5734
|
-
const
|
|
5735
|
-
|
|
5736
|
-
|
|
5737
|
-
|
|
5738
|
-
const
|
|
5739
|
-
|
|
5740
|
-
|
|
5741
|
-
|
|
5742
|
-
|
|
5743
|
-
|
|
5744
|
-
|
|
5745
|
-
|
|
5746
|
-
|
|
5747
|
-
|
|
5748
|
-
|
|
5749
|
-
|
|
5750
|
-
|
|
5751
|
-
operations[op] = { count: 0, sum: 0 };
|
|
5752
|
-
}
|
|
5753
|
-
operations[op].count += stats.count;
|
|
5754
|
-
operations[op].sum += stats.sum;
|
|
5755
|
-
}
|
|
5756
|
-
}
|
|
5757
|
-
const recordCount = Math.max(...sourceAnalytics.map((a) => a.recordCount));
|
|
5758
|
-
const id = `${period}-${cohort}`;
|
|
5759
|
-
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
5760
|
-
const [existingOk, existingErr, existing] = await tryFn(
|
|
5761
|
-
() => this.analyticsResource.get(id)
|
|
5762
|
-
);
|
|
5763
|
-
if (existingOk && existing) {
|
|
5764
|
-
await tryFn(
|
|
5765
|
-
() => this.analyticsResource.update(id, {
|
|
5766
|
-
transactionCount,
|
|
5767
|
-
totalValue,
|
|
5768
|
-
avgValue,
|
|
5769
|
-
minValue,
|
|
5770
|
-
maxValue,
|
|
5771
|
-
operations,
|
|
5772
|
-
recordCount,
|
|
5773
|
-
updatedAt: now
|
|
5774
|
-
})
|
|
6143
|
+
async _runConsolidationForHandler(handler, resourceName, fieldName) {
|
|
6144
|
+
const oldResource = this.config.resource;
|
|
6145
|
+
const oldField = this.config.field;
|
|
6146
|
+
const oldTransactionResource = this.transactionResource;
|
|
6147
|
+
const oldTargetResource = this.targetResource;
|
|
6148
|
+
const oldLockResource = this.lockResource;
|
|
6149
|
+
const oldAnalyticsResource = this.analyticsResource;
|
|
6150
|
+
this.config.resource = resourceName;
|
|
6151
|
+
this.config.field = fieldName;
|
|
6152
|
+
this.transactionResource = handler.transactionResource;
|
|
6153
|
+
this.targetResource = handler.targetResource;
|
|
6154
|
+
this.lockResource = handler.lockResource;
|
|
6155
|
+
this.analyticsResource = handler.analyticsResource;
|
|
6156
|
+
try {
|
|
6157
|
+
await runConsolidation(
|
|
6158
|
+
this.transactionResource,
|
|
6159
|
+
(id) => this.consolidateRecord(id),
|
|
6160
|
+
(event, data) => this.emit(event, data),
|
|
6161
|
+
this.config
|
|
5775
6162
|
);
|
|
5776
|
-
}
|
|
5777
|
-
|
|
5778
|
-
|
|
5779
|
-
|
|
5780
|
-
|
|
5781
|
-
|
|
5782
|
-
|
|
5783
|
-
|
|
5784
|
-
|
|
5785
|
-
|
|
5786
|
-
|
|
5787
|
-
|
|
5788
|
-
|
|
5789
|
-
|
|
5790
|
-
|
|
5791
|
-
|
|
6163
|
+
} finally {
|
|
6164
|
+
this.config.resource = oldResource;
|
|
6165
|
+
this.config.field = oldField;
|
|
6166
|
+
this.transactionResource = oldTransactionResource;
|
|
6167
|
+
this.targetResource = oldTargetResource;
|
|
6168
|
+
this.lockResource = oldLockResource;
|
|
6169
|
+
this.analyticsResource = oldAnalyticsResource;
|
|
6170
|
+
}
|
|
6171
|
+
}
|
|
6172
|
+
/**
|
|
6173
|
+
* Run garbage collection for a handler
|
|
6174
|
+
* @private
|
|
6175
|
+
*/
|
|
6176
|
+
async _runGarbageCollectionForHandler(handler, resourceName, fieldName) {
|
|
6177
|
+
const oldResource = this.config.resource;
|
|
6178
|
+
const oldField = this.config.field;
|
|
6179
|
+
const oldTransactionResource = this.transactionResource;
|
|
6180
|
+
const oldTargetResource = this.targetResource;
|
|
6181
|
+
const oldLockResource = this.lockResource;
|
|
6182
|
+
this.config.resource = resourceName;
|
|
6183
|
+
this.config.field = fieldName;
|
|
6184
|
+
this.transactionResource = handler.transactionResource;
|
|
6185
|
+
this.targetResource = handler.targetResource;
|
|
6186
|
+
this.lockResource = handler.lockResource;
|
|
6187
|
+
try {
|
|
6188
|
+
await runGarbageCollection(
|
|
6189
|
+
this.transactionResource,
|
|
6190
|
+
this.lockResource,
|
|
6191
|
+
this.config,
|
|
6192
|
+
(event, data) => this.emit(event, data)
|
|
5792
6193
|
);
|
|
6194
|
+
} finally {
|
|
6195
|
+
this.config.resource = oldResource;
|
|
6196
|
+
this.config.field = oldField;
|
|
6197
|
+
this.transactionResource = oldTransactionResource;
|
|
6198
|
+
this.targetResource = oldTargetResource;
|
|
6199
|
+
this.lockResource = oldLockResource;
|
|
5793
6200
|
}
|
|
5794
6201
|
}
|
|
6202
|
+
// Public Analytics API
|
|
5795
6203
|
/**
|
|
5796
6204
|
* Get analytics for a specific period
|
|
5797
6205
|
* @param {string} resourceName - Resource name
|
|
@@ -5800,111 +6208,7 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
5800
6208
|
* @returns {Promise<Array>} Analytics data
|
|
5801
6209
|
*/
|
|
5802
6210
|
async getAnalytics(resourceName, field, options = {}) {
|
|
5803
|
-
|
|
5804
|
-
if (!fieldHandlers) {
|
|
5805
|
-
throw new Error(`No eventual consistency configured for resource: ${resourceName}`);
|
|
5806
|
-
}
|
|
5807
|
-
const handler = fieldHandlers.get(field);
|
|
5808
|
-
if (!handler) {
|
|
5809
|
-
throw new Error(`No eventual consistency configured for field: ${resourceName}.${field}`);
|
|
5810
|
-
}
|
|
5811
|
-
if (!handler.analyticsResource) {
|
|
5812
|
-
throw new Error("Analytics not enabled for this plugin");
|
|
5813
|
-
}
|
|
5814
|
-
const { period = "day", date, startDate, endDate, month, year, breakdown = false } = options;
|
|
5815
|
-
const [ok, err, allAnalytics] = await tryFn(
|
|
5816
|
-
() => handler.analyticsResource.list()
|
|
5817
|
-
);
|
|
5818
|
-
if (!ok || !allAnalytics) {
|
|
5819
|
-
return [];
|
|
5820
|
-
}
|
|
5821
|
-
let filtered = allAnalytics.filter((a) => a.period === period);
|
|
5822
|
-
if (date) {
|
|
5823
|
-
if (period === "hour") {
|
|
5824
|
-
filtered = filtered.filter((a) => a.cohort.startsWith(date));
|
|
5825
|
-
} else {
|
|
5826
|
-
filtered = filtered.filter((a) => a.cohort === date);
|
|
5827
|
-
}
|
|
5828
|
-
} else if (startDate && endDate) {
|
|
5829
|
-
filtered = filtered.filter((a) => a.cohort >= startDate && a.cohort <= endDate);
|
|
5830
|
-
} else if (month) {
|
|
5831
|
-
filtered = filtered.filter((a) => a.cohort.startsWith(month));
|
|
5832
|
-
} else if (year) {
|
|
5833
|
-
filtered = filtered.filter((a) => a.cohort.startsWith(String(year)));
|
|
5834
|
-
}
|
|
5835
|
-
filtered.sort((a, b) => a.cohort.localeCompare(b.cohort));
|
|
5836
|
-
if (breakdown === "operations") {
|
|
5837
|
-
return filtered.map((a) => ({
|
|
5838
|
-
cohort: a.cohort,
|
|
5839
|
-
...a.operations
|
|
5840
|
-
}));
|
|
5841
|
-
}
|
|
5842
|
-
return filtered.map((a) => ({
|
|
5843
|
-
cohort: a.cohort,
|
|
5844
|
-
count: a.transactionCount,
|
|
5845
|
-
sum: a.totalValue,
|
|
5846
|
-
avg: a.avgValue,
|
|
5847
|
-
min: a.minValue,
|
|
5848
|
-
max: a.maxValue,
|
|
5849
|
-
operations: a.operations,
|
|
5850
|
-
recordCount: a.recordCount
|
|
5851
|
-
}));
|
|
5852
|
-
}
|
|
5853
|
-
/**
|
|
5854
|
-
* Fill gaps in analytics data with zeros for continuous time series
|
|
5855
|
-
* @private
|
|
5856
|
-
* @param {Array} data - Sparse analytics data
|
|
5857
|
-
* @param {string} period - Period type ('hour', 'day', 'month')
|
|
5858
|
-
* @param {string} startDate - Start date (ISO format)
|
|
5859
|
-
* @param {string} endDate - End date (ISO format)
|
|
5860
|
-
* @returns {Array} Complete time series with gaps filled
|
|
5861
|
-
*/
|
|
5862
|
-
_fillGaps(data, period, startDate, endDate) {
|
|
5863
|
-
if (!data || data.length === 0) {
|
|
5864
|
-
data = [];
|
|
5865
|
-
}
|
|
5866
|
-
const dataMap = /* @__PURE__ */ new Map();
|
|
5867
|
-
data.forEach((item) => {
|
|
5868
|
-
dataMap.set(item.cohort, item);
|
|
5869
|
-
});
|
|
5870
|
-
const result = [];
|
|
5871
|
-
const emptyRecord = {
|
|
5872
|
-
count: 0,
|
|
5873
|
-
sum: 0,
|
|
5874
|
-
avg: 0,
|
|
5875
|
-
min: 0,
|
|
5876
|
-
max: 0,
|
|
5877
|
-
recordCount: 0
|
|
5878
|
-
};
|
|
5879
|
-
if (period === "hour") {
|
|
5880
|
-
const start = /* @__PURE__ */ new Date(startDate + "T00:00:00Z");
|
|
5881
|
-
const end = /* @__PURE__ */ new Date(endDate + "T23:59:59Z");
|
|
5882
|
-
for (let dt = new Date(start); dt <= end; dt.setHours(dt.getHours() + 1)) {
|
|
5883
|
-
const cohort = dt.toISOString().substring(0, 13);
|
|
5884
|
-
result.push(dataMap.get(cohort) || { cohort, ...emptyRecord });
|
|
5885
|
-
}
|
|
5886
|
-
} else if (period === "day") {
|
|
5887
|
-
const start = new Date(startDate);
|
|
5888
|
-
const end = new Date(endDate);
|
|
5889
|
-
for (let dt = new Date(start); dt <= end; dt.setDate(dt.getDate() + 1)) {
|
|
5890
|
-
const cohort = dt.toISOString().substring(0, 10);
|
|
5891
|
-
result.push(dataMap.get(cohort) || { cohort, ...emptyRecord });
|
|
5892
|
-
}
|
|
5893
|
-
} else if (period === "month") {
|
|
5894
|
-
const startYear = parseInt(startDate.substring(0, 4));
|
|
5895
|
-
const startMonth = parseInt(startDate.substring(5, 7));
|
|
5896
|
-
const endYear = parseInt(endDate.substring(0, 4));
|
|
5897
|
-
const endMonth = parseInt(endDate.substring(5, 7));
|
|
5898
|
-
for (let year = startYear; year <= endYear; year++) {
|
|
5899
|
-
const firstMonth = year === startYear ? startMonth : 1;
|
|
5900
|
-
const lastMonth = year === endYear ? endMonth : 12;
|
|
5901
|
-
for (let month = firstMonth; month <= lastMonth; month++) {
|
|
5902
|
-
const cohort = `${year}-${month.toString().padStart(2, "0")}`;
|
|
5903
|
-
result.push(dataMap.get(cohort) || { cohort, ...emptyRecord });
|
|
5904
|
-
}
|
|
5905
|
-
}
|
|
5906
|
-
}
|
|
5907
|
-
return result;
|
|
6211
|
+
return await getAnalytics(resourceName, field, options, this.fieldHandlers);
|
|
5908
6212
|
}
|
|
5909
6213
|
/**
|
|
5910
6214
|
* Get analytics for entire month, broken down by days
|
|
@@ -5912,25 +6216,10 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
5912
6216
|
* @param {string} field - Field name
|
|
5913
6217
|
* @param {string} month - Month in YYYY-MM format
|
|
5914
6218
|
* @param {Object} options - Options
|
|
5915
|
-
* @param {boolean} options.fillGaps - Fill missing days with zeros (default: false)
|
|
5916
6219
|
* @returns {Promise<Array>} Daily analytics for the month
|
|
5917
6220
|
*/
|
|
5918
6221
|
async getMonthByDay(resourceName, field, month, options = {}) {
|
|
5919
|
-
|
|
5920
|
-
const monthNum = parseInt(month.substring(5, 7));
|
|
5921
|
-
const firstDay = new Date(year, monthNum - 1, 1);
|
|
5922
|
-
const lastDay = new Date(year, monthNum, 0);
|
|
5923
|
-
const startDate = firstDay.toISOString().substring(0, 10);
|
|
5924
|
-
const endDate = lastDay.toISOString().substring(0, 10);
|
|
5925
|
-
const data = await this.getAnalytics(resourceName, field, {
|
|
5926
|
-
period: "day",
|
|
5927
|
-
startDate,
|
|
5928
|
-
endDate
|
|
5929
|
-
});
|
|
5930
|
-
if (options.fillGaps) {
|
|
5931
|
-
return this._fillGaps(data, "day", startDate, endDate);
|
|
5932
|
-
}
|
|
5933
|
-
return data;
|
|
6222
|
+
return await getMonthByDay(resourceName, field, month, options, this.fieldHandlers);
|
|
5934
6223
|
}
|
|
5935
6224
|
/**
|
|
5936
6225
|
* Get analytics for entire day, broken down by hours
|
|
@@ -5938,18 +6227,10 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
5938
6227
|
* @param {string} field - Field name
|
|
5939
6228
|
* @param {string} date - Date in YYYY-MM-DD format
|
|
5940
6229
|
* @param {Object} options - Options
|
|
5941
|
-
* @param {boolean} options.fillGaps - Fill missing hours with zeros (default: false)
|
|
5942
6230
|
* @returns {Promise<Array>} Hourly analytics for the day
|
|
5943
6231
|
*/
|
|
5944
6232
|
async getDayByHour(resourceName, field, date, options = {}) {
|
|
5945
|
-
|
|
5946
|
-
period: "hour",
|
|
5947
|
-
date
|
|
5948
|
-
});
|
|
5949
|
-
if (options.fillGaps) {
|
|
5950
|
-
return this._fillGaps(data, "hour", date, date);
|
|
5951
|
-
}
|
|
5952
|
-
return data;
|
|
6233
|
+
return await getDayByHour(resourceName, field, date, options, this.fieldHandlers);
|
|
5953
6234
|
}
|
|
5954
6235
|
/**
|
|
5955
6236
|
* Get analytics for last N days, broken down by days
|
|
@@ -5957,24 +6238,10 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
5957
6238
|
* @param {string} field - Field name
|
|
5958
6239
|
* @param {number} days - Number of days to look back (default: 7)
|
|
5959
6240
|
* @param {Object} options - Options
|
|
5960
|
-
* @param {boolean} options.fillGaps - Fill missing days with zeros (default: false)
|
|
5961
6241
|
* @returns {Promise<Array>} Daily analytics
|
|
5962
6242
|
*/
|
|
5963
6243
|
async getLastNDays(resourceName, field, days = 7, options = {}) {
|
|
5964
|
-
|
|
5965
|
-
const date = /* @__PURE__ */ new Date();
|
|
5966
|
-
date.setDate(date.getDate() - i);
|
|
5967
|
-
return date.toISOString().substring(0, 10);
|
|
5968
|
-
}).reverse();
|
|
5969
|
-
const data = await this.getAnalytics(resourceName, field, {
|
|
5970
|
-
period: "day",
|
|
5971
|
-
startDate: dates[0],
|
|
5972
|
-
endDate: dates[dates.length - 1]
|
|
5973
|
-
});
|
|
5974
|
-
if (options.fillGaps) {
|
|
5975
|
-
return this._fillGaps(data, "day", dates[0], dates[dates.length - 1]);
|
|
5976
|
-
}
|
|
5977
|
-
return data;
|
|
6244
|
+
return await getLastNDays(resourceName, field, days, options, this.fieldHandlers);
|
|
5978
6245
|
}
|
|
5979
6246
|
/**
|
|
5980
6247
|
* Get analytics for entire year, broken down by months
|
|
@@ -5982,20 +6249,10 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
5982
6249
|
* @param {string} field - Field name
|
|
5983
6250
|
* @param {number} year - Year (e.g., 2025)
|
|
5984
6251
|
* @param {Object} options - Options
|
|
5985
|
-
* @param {boolean} options.fillGaps - Fill missing months with zeros (default: false)
|
|
5986
6252
|
* @returns {Promise<Array>} Monthly analytics for the year
|
|
5987
6253
|
*/
|
|
5988
6254
|
async getYearByMonth(resourceName, field, year, options = {}) {
|
|
5989
|
-
|
|
5990
|
-
period: "month",
|
|
5991
|
-
year
|
|
5992
|
-
});
|
|
5993
|
-
if (options.fillGaps) {
|
|
5994
|
-
const startDate = `${year}-01`;
|
|
5995
|
-
const endDate = `${year}-12`;
|
|
5996
|
-
return this._fillGaps(data, "month", startDate, endDate);
|
|
5997
|
-
}
|
|
5998
|
-
return data;
|
|
6255
|
+
return await getYearByMonth(resourceName, field, year, options, this.fieldHandlers);
|
|
5999
6256
|
}
|
|
6000
6257
|
/**
|
|
6001
6258
|
* Get analytics for entire month, broken down by hours
|
|
@@ -6003,33 +6260,10 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
6003
6260
|
* @param {string} field - Field name
|
|
6004
6261
|
* @param {string} month - Month in YYYY-MM format (or 'last' for previous month)
|
|
6005
6262
|
* @param {Object} options - Options
|
|
6006
|
-
* @
|
|
6007
|
-
* @returns {Promise<Array>} Hourly analytics for the month (up to 24*31=744 records)
|
|
6263
|
+
* @returns {Promise<Array>} Hourly analytics for the month
|
|
6008
6264
|
*/
|
|
6009
6265
|
async getMonthByHour(resourceName, field, month, options = {}) {
|
|
6010
|
-
|
|
6011
|
-
if (month === "last") {
|
|
6012
|
-
const now = /* @__PURE__ */ new Date();
|
|
6013
|
-
now.setMonth(now.getMonth() - 1);
|
|
6014
|
-
year = now.getFullYear();
|
|
6015
|
-
monthNum = now.getMonth() + 1;
|
|
6016
|
-
} else {
|
|
6017
|
-
year = parseInt(month.substring(0, 4));
|
|
6018
|
-
monthNum = parseInt(month.substring(5, 7));
|
|
6019
|
-
}
|
|
6020
|
-
const firstDay = new Date(year, monthNum - 1, 1);
|
|
6021
|
-
const lastDay = new Date(year, monthNum, 0);
|
|
6022
|
-
const startDate = firstDay.toISOString().substring(0, 10);
|
|
6023
|
-
const endDate = lastDay.toISOString().substring(0, 10);
|
|
6024
|
-
const data = await this.getAnalytics(resourceName, field, {
|
|
6025
|
-
period: "hour",
|
|
6026
|
-
startDate,
|
|
6027
|
-
endDate
|
|
6028
|
-
});
|
|
6029
|
-
if (options.fillGaps) {
|
|
6030
|
-
return this._fillGaps(data, "hour", startDate, endDate);
|
|
6031
|
-
}
|
|
6032
|
-
return data;
|
|
6266
|
+
return await getMonthByHour(resourceName, field, month, options, this.fieldHandlers);
|
|
6033
6267
|
}
|
|
6034
6268
|
/**
|
|
6035
6269
|
* Get top records by volume
|
|
@@ -6039,57 +6273,7 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
6039
6273
|
* @returns {Promise<Array>} Top records
|
|
6040
6274
|
*/
|
|
6041
6275
|
async getTopRecords(resourceName, field, options = {}) {
|
|
6042
|
-
|
|
6043
|
-
if (!fieldHandlers) {
|
|
6044
|
-
throw new Error(`No eventual consistency configured for resource: ${resourceName}`);
|
|
6045
|
-
}
|
|
6046
|
-
const handler = fieldHandlers.get(field);
|
|
6047
|
-
if (!handler) {
|
|
6048
|
-
throw new Error(`No eventual consistency configured for field: ${resourceName}.${field}`);
|
|
6049
|
-
}
|
|
6050
|
-
if (!handler.transactionResource) {
|
|
6051
|
-
throw new Error("Transaction resource not initialized");
|
|
6052
|
-
}
|
|
6053
|
-
const { period = "day", date, metric = "transactionCount", limit = 10 } = options;
|
|
6054
|
-
const [ok, err, transactions] = await tryFn(
|
|
6055
|
-
() => handler.transactionResource.list()
|
|
6056
|
-
);
|
|
6057
|
-
if (!ok || !transactions) {
|
|
6058
|
-
return [];
|
|
6059
|
-
}
|
|
6060
|
-
let filtered = transactions;
|
|
6061
|
-
if (date) {
|
|
6062
|
-
if (period === "hour") {
|
|
6063
|
-
filtered = transactions.filter((t) => t.cohortHour && t.cohortHour.startsWith(date));
|
|
6064
|
-
} else if (period === "day") {
|
|
6065
|
-
filtered = transactions.filter((t) => t.cohortDate === date);
|
|
6066
|
-
} else if (period === "month") {
|
|
6067
|
-
filtered = transactions.filter((t) => t.cohortMonth && t.cohortMonth.startsWith(date));
|
|
6068
|
-
}
|
|
6069
|
-
}
|
|
6070
|
-
const byRecord = {};
|
|
6071
|
-
for (const txn of filtered) {
|
|
6072
|
-
const recordId = txn.originalId;
|
|
6073
|
-
if (!byRecord[recordId]) {
|
|
6074
|
-
byRecord[recordId] = { count: 0, sum: 0 };
|
|
6075
|
-
}
|
|
6076
|
-
byRecord[recordId].count++;
|
|
6077
|
-
byRecord[recordId].sum += txn.value;
|
|
6078
|
-
}
|
|
6079
|
-
const records = Object.entries(byRecord).map(([recordId, stats]) => ({
|
|
6080
|
-
recordId,
|
|
6081
|
-
count: stats.count,
|
|
6082
|
-
sum: stats.sum
|
|
6083
|
-
}));
|
|
6084
|
-
records.sort((a, b) => {
|
|
6085
|
-
if (metric === "transactionCount") {
|
|
6086
|
-
return b.count - a.count;
|
|
6087
|
-
} else if (metric === "totalValue") {
|
|
6088
|
-
return b.sum - a.sum;
|
|
6089
|
-
}
|
|
6090
|
-
return 0;
|
|
6091
|
-
});
|
|
6092
|
-
return records.slice(0, limit);
|
|
6276
|
+
return await getTopRecords(resourceName, field, options, this.fieldHandlers);
|
|
6093
6277
|
}
|
|
6094
6278
|
}
|
|
6095
6279
|
|
|
@@ -12160,7 +12344,7 @@ class Database extends EventEmitter {
|
|
|
12160
12344
|
this.id = idGenerator(7);
|
|
12161
12345
|
this.version = "1";
|
|
12162
12346
|
this.s3dbVersion = (() => {
|
|
12163
|
-
const [ok, err, version] = tryFn(() => true ? "10.0.
|
|
12347
|
+
const [ok, err, version] = tryFn(() => true ? "10.0.17" : "latest");
|
|
12164
12348
|
return ok ? version : "latest";
|
|
12165
12349
|
})();
|
|
12166
12350
|
this.resources = {};
|