s3db.js 10.0.1 → 10.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/s3db.cjs.js +709 -2
- package/dist/s3db.cjs.js.map +1 -1
- package/dist/s3db.d.ts +70 -3
- package/dist/s3db.es.js +709 -3
- package/dist/s3db.es.js.map +1 -1
- package/package.json +1 -1
- package/src/plugins/audit.plugin.js +4 -2
- package/src/plugins/backup.plugin.js +3 -1
- package/src/plugins/eventual-consistency.plugin.js +574 -2
- package/src/plugins/queue-consumer.plugin.js +4 -2
- package/src/s3db.d.ts +70 -3
package/dist/s3db.cjs.js
CHANGED
|
@@ -4343,10 +4343,20 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
4343
4343
|
// Days to keep applied transactions
|
|
4344
4344
|
gcInterval: options.gcInterval || 86400,
|
|
4345
4345
|
// 24 hours (in seconds)
|
|
4346
|
-
verbose: options.verbose || false
|
|
4346
|
+
verbose: options.verbose || false,
|
|
4347
|
+
// Analytics configuration
|
|
4348
|
+
enableAnalytics: options.enableAnalytics || false,
|
|
4349
|
+
analyticsConfig: {
|
|
4350
|
+
periods: options.analyticsConfig?.periods || ["hour", "day", "month"],
|
|
4351
|
+
metrics: options.analyticsConfig?.metrics || ["count", "sum", "avg", "min", "max"],
|
|
4352
|
+
rollupStrategy: options.analyticsConfig?.rollupStrategy || "incremental",
|
|
4353
|
+
// 'incremental' or 'batch'
|
|
4354
|
+
retentionDays: options.analyticsConfig?.retentionDays || 365
|
|
4355
|
+
}
|
|
4347
4356
|
};
|
|
4348
4357
|
this.transactionResource = null;
|
|
4349
4358
|
this.targetResource = null;
|
|
4359
|
+
this.analyticsResource = null;
|
|
4350
4360
|
this.consolidationTimer = null;
|
|
4351
4361
|
this.gcTimer = null;
|
|
4352
4362
|
this.pendingTransactions = /* @__PURE__ */ new Map();
|
|
@@ -4433,6 +4443,9 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
4433
4443
|
throw new Error(`Failed to create lock resource: ${lockErr?.message}`);
|
|
4434
4444
|
}
|
|
4435
4445
|
this.lockResource = lockOk ? lockResource : this.database.resources[lockResourceName];
|
|
4446
|
+
if (this.config.enableAnalytics) {
|
|
4447
|
+
await this.createAnalyticsResource();
|
|
4448
|
+
}
|
|
4436
4449
|
this.addHelperMethods();
|
|
4437
4450
|
if (this.config.autoConsolidate) {
|
|
4438
4451
|
this.startConsolidationTimer();
|
|
@@ -4484,6 +4497,53 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
4484
4497
|
};
|
|
4485
4498
|
return partitions;
|
|
4486
4499
|
}
|
|
4500
|
+
async createAnalyticsResource() {
|
|
4501
|
+
const analyticsResourceName = `${this.config.resource}_analytics_${this.config.field}`;
|
|
4502
|
+
const [ok, err, analyticsResource] = await tryFn(
|
|
4503
|
+
() => this.database.createResource({
|
|
4504
|
+
name: analyticsResourceName,
|
|
4505
|
+
attributes: {
|
|
4506
|
+
id: "string|required",
|
|
4507
|
+
period: "string|required",
|
|
4508
|
+
// 'hour', 'day', 'month'
|
|
4509
|
+
cohort: "string|required",
|
|
4510
|
+
// ISO format: '2025-10-09T14', '2025-10-09', '2025-10'
|
|
4511
|
+
// Aggregated metrics
|
|
4512
|
+
transactionCount: "number|required",
|
|
4513
|
+
totalValue: "number|required",
|
|
4514
|
+
avgValue: "number|required",
|
|
4515
|
+
minValue: "number|required",
|
|
4516
|
+
maxValue: "number|required",
|
|
4517
|
+
// Operation breakdown
|
|
4518
|
+
operations: "object|optional",
|
|
4519
|
+
// { add: { count, sum }, sub: { count, sum }, set: { count, sum } }
|
|
4520
|
+
// Metadata
|
|
4521
|
+
recordCount: "number|required",
|
|
4522
|
+
// Distinct originalIds
|
|
4523
|
+
consolidatedAt: "string|required",
|
|
4524
|
+
updatedAt: "string|required"
|
|
4525
|
+
},
|
|
4526
|
+
behavior: "body-overflow",
|
|
4527
|
+
timestamps: false,
|
|
4528
|
+
partitions: {
|
|
4529
|
+
byPeriod: {
|
|
4530
|
+
fields: { period: "string" }
|
|
4531
|
+
},
|
|
4532
|
+
byCohort: {
|
|
4533
|
+
fields: { cohort: "string" }
|
|
4534
|
+
}
|
|
4535
|
+
}
|
|
4536
|
+
})
|
|
4537
|
+
);
|
|
4538
|
+
if (!ok && !this.database.resources[analyticsResourceName]) {
|
|
4539
|
+
console.warn(`[EventualConsistency] Failed to create analytics resource: ${err?.message}`);
|
|
4540
|
+
return;
|
|
4541
|
+
}
|
|
4542
|
+
this.analyticsResource = ok ? analyticsResource : this.database.resources[analyticsResourceName];
|
|
4543
|
+
if (this.config.verbose) {
|
|
4544
|
+
console.log(`[EventualConsistency] Analytics resource created: ${analyticsResourceName}`);
|
|
4545
|
+
}
|
|
4546
|
+
}
|
|
4487
4547
|
/**
|
|
4488
4548
|
* Auto-detect timezone from environment or system
|
|
4489
4549
|
* @private
|
|
@@ -4831,6 +4891,9 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
4831
4891
|
if (errors && errors.length > 0 && this.config.verbose) {
|
|
4832
4892
|
console.warn(`[EventualConsistency] ${errors.length} transactions failed to mark as applied`);
|
|
4833
4893
|
}
|
|
4894
|
+
if (this.config.enableAnalytics && transactionsToUpdate.length > 0) {
|
|
4895
|
+
await this.updateAnalytics(transactionsToUpdate);
|
|
4896
|
+
}
|
|
4834
4897
|
}
|
|
4835
4898
|
return consolidatedValue;
|
|
4836
4899
|
} finally {
|
|
@@ -5038,6 +5101,402 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
5038
5101
|
await tryFn(() => this.lockResource.delete(gcLockId));
|
|
5039
5102
|
}
|
|
5040
5103
|
}
|
|
5104
|
+
/**
|
|
5105
|
+
* Update analytics with consolidated transactions
|
|
5106
|
+
* @param {Array} transactions - Array of transactions that were just consolidated
|
|
5107
|
+
* @private
|
|
5108
|
+
*/
|
|
5109
|
+
async updateAnalytics(transactions) {
|
|
5110
|
+
if (!this.analyticsResource || transactions.length === 0) return;
|
|
5111
|
+
try {
|
|
5112
|
+
const byHour = this._groupByCohort(transactions, "cohortHour");
|
|
5113
|
+
for (const [cohort, txns] of Object.entries(byHour)) {
|
|
5114
|
+
await this._upsertAnalytics("hour", cohort, txns);
|
|
5115
|
+
}
|
|
5116
|
+
if (this.config.analyticsConfig.rollupStrategy === "incremental") {
|
|
5117
|
+
const uniqueHours = Object.keys(byHour);
|
|
5118
|
+
for (const cohortHour of uniqueHours) {
|
|
5119
|
+
await this._rollupAnalytics(cohortHour);
|
|
5120
|
+
}
|
|
5121
|
+
}
|
|
5122
|
+
} catch (error) {
|
|
5123
|
+
if (this.config.verbose) {
|
|
5124
|
+
console.warn(`[EventualConsistency] Analytics update error:`, error.message);
|
|
5125
|
+
}
|
|
5126
|
+
}
|
|
5127
|
+
}
|
|
5128
|
+
/**
|
|
5129
|
+
* Group transactions by cohort
|
|
5130
|
+
* @private
|
|
5131
|
+
*/
|
|
5132
|
+
_groupByCohort(transactions, cohortField) {
|
|
5133
|
+
const groups = {};
|
|
5134
|
+
for (const txn of transactions) {
|
|
5135
|
+
const cohort = txn[cohortField];
|
|
5136
|
+
if (!cohort) continue;
|
|
5137
|
+
if (!groups[cohort]) {
|
|
5138
|
+
groups[cohort] = [];
|
|
5139
|
+
}
|
|
5140
|
+
groups[cohort].push(txn);
|
|
5141
|
+
}
|
|
5142
|
+
return groups;
|
|
5143
|
+
}
|
|
5144
|
+
/**
|
|
5145
|
+
* Upsert analytics for a specific period and cohort
|
|
5146
|
+
* @private
|
|
5147
|
+
*/
|
|
5148
|
+
async _upsertAnalytics(period, cohort, transactions) {
|
|
5149
|
+
const id = `${period}-${cohort}`;
|
|
5150
|
+
const transactionCount = transactions.length;
|
|
5151
|
+
const signedValues = transactions.map((t) => {
|
|
5152
|
+
if (t.operation === "sub") return -t.value;
|
|
5153
|
+
return t.value;
|
|
5154
|
+
});
|
|
5155
|
+
const totalValue = signedValues.reduce((sum, v) => sum + v, 0);
|
|
5156
|
+
const avgValue = totalValue / transactionCount;
|
|
5157
|
+
const minValue = Math.min(...signedValues);
|
|
5158
|
+
const maxValue = Math.max(...signedValues);
|
|
5159
|
+
const operations = this._calculateOperationBreakdown(transactions);
|
|
5160
|
+
const recordCount = new Set(transactions.map((t) => t.originalId)).size;
|
|
5161
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
5162
|
+
const [existingOk, existingErr, existing] = await tryFn(
|
|
5163
|
+
() => this.analyticsResource.get(id)
|
|
5164
|
+
);
|
|
5165
|
+
if (existingOk && existing) {
|
|
5166
|
+
const newTransactionCount = existing.transactionCount + transactionCount;
|
|
5167
|
+
const newTotalValue = existing.totalValue + totalValue;
|
|
5168
|
+
const newAvgValue = newTotalValue / newTransactionCount;
|
|
5169
|
+
const newMinValue = Math.min(existing.minValue, minValue);
|
|
5170
|
+
const newMaxValue = Math.max(existing.maxValue, maxValue);
|
|
5171
|
+
const newOperations = { ...existing.operations };
|
|
5172
|
+
for (const [op, stats] of Object.entries(operations)) {
|
|
5173
|
+
if (!newOperations[op]) {
|
|
5174
|
+
newOperations[op] = { count: 0, sum: 0 };
|
|
5175
|
+
}
|
|
5176
|
+
newOperations[op].count += stats.count;
|
|
5177
|
+
newOperations[op].sum += stats.sum;
|
|
5178
|
+
}
|
|
5179
|
+
const newRecordCount = Math.max(existing.recordCount, recordCount);
|
|
5180
|
+
await tryFn(
|
|
5181
|
+
() => this.analyticsResource.update(id, {
|
|
5182
|
+
transactionCount: newTransactionCount,
|
|
5183
|
+
totalValue: newTotalValue,
|
|
5184
|
+
avgValue: newAvgValue,
|
|
5185
|
+
minValue: newMinValue,
|
|
5186
|
+
maxValue: newMaxValue,
|
|
5187
|
+
operations: newOperations,
|
|
5188
|
+
recordCount: newRecordCount,
|
|
5189
|
+
updatedAt: now
|
|
5190
|
+
})
|
|
5191
|
+
);
|
|
5192
|
+
} else {
|
|
5193
|
+
await tryFn(
|
|
5194
|
+
() => this.analyticsResource.insert({
|
|
5195
|
+
id,
|
|
5196
|
+
period,
|
|
5197
|
+
cohort,
|
|
5198
|
+
transactionCount,
|
|
5199
|
+
totalValue,
|
|
5200
|
+
avgValue,
|
|
5201
|
+
minValue,
|
|
5202
|
+
maxValue,
|
|
5203
|
+
operations,
|
|
5204
|
+
recordCount,
|
|
5205
|
+
consolidatedAt: now,
|
|
5206
|
+
updatedAt: now
|
|
5207
|
+
})
|
|
5208
|
+
);
|
|
5209
|
+
}
|
|
5210
|
+
}
|
|
5211
|
+
/**
|
|
5212
|
+
* Calculate operation breakdown
|
|
5213
|
+
* @private
|
|
5214
|
+
*/
|
|
5215
|
+
_calculateOperationBreakdown(transactions) {
|
|
5216
|
+
const breakdown = {};
|
|
5217
|
+
for (const txn of transactions) {
|
|
5218
|
+
const op = txn.operation;
|
|
5219
|
+
if (!breakdown[op]) {
|
|
5220
|
+
breakdown[op] = { count: 0, sum: 0 };
|
|
5221
|
+
}
|
|
5222
|
+
breakdown[op].count++;
|
|
5223
|
+
const signedValue = op === "sub" ? -txn.value : txn.value;
|
|
5224
|
+
breakdown[op].sum += signedValue;
|
|
5225
|
+
}
|
|
5226
|
+
return breakdown;
|
|
5227
|
+
}
|
|
5228
|
+
/**
|
|
5229
|
+
* Roll up hourly analytics to daily and monthly
|
|
5230
|
+
* @private
|
|
5231
|
+
*/
|
|
5232
|
+
async _rollupAnalytics(cohortHour) {
|
|
5233
|
+
const cohortDate = cohortHour.substring(0, 10);
|
|
5234
|
+
const cohortMonth = cohortHour.substring(0, 7);
|
|
5235
|
+
await this._rollupPeriod("day", cohortDate, cohortDate);
|
|
5236
|
+
await this._rollupPeriod("month", cohortMonth, cohortMonth);
|
|
5237
|
+
}
|
|
5238
|
+
/**
|
|
5239
|
+
* Roll up analytics for a specific period
|
|
5240
|
+
* @private
|
|
5241
|
+
*/
|
|
5242
|
+
async _rollupPeriod(period, cohort, sourcePrefix) {
|
|
5243
|
+
const sourcePeriod = period === "day" ? "hour" : "day";
|
|
5244
|
+
const [ok, err, allAnalytics] = await tryFn(
|
|
5245
|
+
() => this.analyticsResource.list()
|
|
5246
|
+
);
|
|
5247
|
+
if (!ok || !allAnalytics) return;
|
|
5248
|
+
const sourceAnalytics = allAnalytics.filter(
|
|
5249
|
+
(a) => a.period === sourcePeriod && a.cohort.startsWith(sourcePrefix)
|
|
5250
|
+
);
|
|
5251
|
+
if (sourceAnalytics.length === 0) return;
|
|
5252
|
+
const transactionCount = sourceAnalytics.reduce((sum, a) => sum + a.transactionCount, 0);
|
|
5253
|
+
const totalValue = sourceAnalytics.reduce((sum, a) => sum + a.totalValue, 0);
|
|
5254
|
+
const avgValue = totalValue / transactionCount;
|
|
5255
|
+
const minValue = Math.min(...sourceAnalytics.map((a) => a.minValue));
|
|
5256
|
+
const maxValue = Math.max(...sourceAnalytics.map((a) => a.maxValue));
|
|
5257
|
+
const operations = {};
|
|
5258
|
+
for (const analytics of sourceAnalytics) {
|
|
5259
|
+
for (const [op, stats] of Object.entries(analytics.operations || {})) {
|
|
5260
|
+
if (!operations[op]) {
|
|
5261
|
+
operations[op] = { count: 0, sum: 0 };
|
|
5262
|
+
}
|
|
5263
|
+
operations[op].count += stats.count;
|
|
5264
|
+
operations[op].sum += stats.sum;
|
|
5265
|
+
}
|
|
5266
|
+
}
|
|
5267
|
+
const recordCount = Math.max(...sourceAnalytics.map((a) => a.recordCount));
|
|
5268
|
+
const id = `${period}-${cohort}`;
|
|
5269
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
5270
|
+
const [existingOk, existingErr, existing] = await tryFn(
|
|
5271
|
+
() => this.analyticsResource.get(id)
|
|
5272
|
+
);
|
|
5273
|
+
if (existingOk && existing) {
|
|
5274
|
+
await tryFn(
|
|
5275
|
+
() => this.analyticsResource.update(id, {
|
|
5276
|
+
transactionCount,
|
|
5277
|
+
totalValue,
|
|
5278
|
+
avgValue,
|
|
5279
|
+
minValue,
|
|
5280
|
+
maxValue,
|
|
5281
|
+
operations,
|
|
5282
|
+
recordCount,
|
|
5283
|
+
updatedAt: now
|
|
5284
|
+
})
|
|
5285
|
+
);
|
|
5286
|
+
} else {
|
|
5287
|
+
await tryFn(
|
|
5288
|
+
() => this.analyticsResource.insert({
|
|
5289
|
+
id,
|
|
5290
|
+
period,
|
|
5291
|
+
cohort,
|
|
5292
|
+
transactionCount,
|
|
5293
|
+
totalValue,
|
|
5294
|
+
avgValue,
|
|
5295
|
+
minValue,
|
|
5296
|
+
maxValue,
|
|
5297
|
+
operations,
|
|
5298
|
+
recordCount,
|
|
5299
|
+
consolidatedAt: now,
|
|
5300
|
+
updatedAt: now
|
|
5301
|
+
})
|
|
5302
|
+
);
|
|
5303
|
+
}
|
|
5304
|
+
}
|
|
5305
|
+
/**
|
|
5306
|
+
* Get analytics for a specific period
|
|
5307
|
+
* @param {string} resourceName - Resource name
|
|
5308
|
+
* @param {string} field - Field name
|
|
5309
|
+
* @param {Object} options - Query options
|
|
5310
|
+
* @returns {Promise<Array>} Analytics data
|
|
5311
|
+
*/
|
|
5312
|
+
async getAnalytics(resourceName, field, options = {}) {
|
|
5313
|
+
if (!this.analyticsResource) {
|
|
5314
|
+
throw new Error("Analytics not enabled for this plugin");
|
|
5315
|
+
}
|
|
5316
|
+
const { period = "day", date, startDate, endDate, month, year, breakdown = false } = options;
|
|
5317
|
+
const [ok, err, allAnalytics] = await tryFn(
|
|
5318
|
+
() => this.analyticsResource.list()
|
|
5319
|
+
);
|
|
5320
|
+
if (!ok || !allAnalytics) {
|
|
5321
|
+
return [];
|
|
5322
|
+
}
|
|
5323
|
+
let filtered = allAnalytics.filter((a) => a.period === period);
|
|
5324
|
+
if (date) {
|
|
5325
|
+
if (period === "hour") {
|
|
5326
|
+
filtered = filtered.filter((a) => a.cohort.startsWith(date));
|
|
5327
|
+
} else {
|
|
5328
|
+
filtered = filtered.filter((a) => a.cohort === date);
|
|
5329
|
+
}
|
|
5330
|
+
} else if (startDate && endDate) {
|
|
5331
|
+
filtered = filtered.filter((a) => a.cohort >= startDate && a.cohort <= endDate);
|
|
5332
|
+
} else if (month) {
|
|
5333
|
+
filtered = filtered.filter((a) => a.cohort.startsWith(month));
|
|
5334
|
+
} else if (year) {
|
|
5335
|
+
filtered = filtered.filter((a) => a.cohort.startsWith(String(year)));
|
|
5336
|
+
}
|
|
5337
|
+
filtered.sort((a, b) => a.cohort.localeCompare(b.cohort));
|
|
5338
|
+
if (breakdown === "operations") {
|
|
5339
|
+
return filtered.map((a) => ({
|
|
5340
|
+
cohort: a.cohort,
|
|
5341
|
+
...a.operations
|
|
5342
|
+
}));
|
|
5343
|
+
}
|
|
5344
|
+
return filtered.map((a) => ({
|
|
5345
|
+
cohort: a.cohort,
|
|
5346
|
+
count: a.transactionCount,
|
|
5347
|
+
sum: a.totalValue,
|
|
5348
|
+
avg: a.avgValue,
|
|
5349
|
+
min: a.minValue,
|
|
5350
|
+
max: a.maxValue,
|
|
5351
|
+
operations: a.operations,
|
|
5352
|
+
recordCount: a.recordCount
|
|
5353
|
+
}));
|
|
5354
|
+
}
|
|
5355
|
+
/**
|
|
5356
|
+
* Get analytics for entire month, broken down by days
|
|
5357
|
+
* @param {string} resourceName - Resource name
|
|
5358
|
+
* @param {string} field - Field name
|
|
5359
|
+
* @param {string} month - Month in YYYY-MM format
|
|
5360
|
+
* @returns {Promise<Array>} Daily analytics for the month
|
|
5361
|
+
*/
|
|
5362
|
+
async getMonthByDay(resourceName, field, month) {
|
|
5363
|
+
const year = parseInt(month.substring(0, 4));
|
|
5364
|
+
const monthNum = parseInt(month.substring(5, 7));
|
|
5365
|
+
const firstDay = new Date(year, monthNum - 1, 1);
|
|
5366
|
+
const lastDay = new Date(year, monthNum, 0);
|
|
5367
|
+
const startDate = firstDay.toISOString().substring(0, 10);
|
|
5368
|
+
const endDate = lastDay.toISOString().substring(0, 10);
|
|
5369
|
+
return await this.getAnalytics(resourceName, field, {
|
|
5370
|
+
period: "day",
|
|
5371
|
+
startDate,
|
|
5372
|
+
endDate
|
|
5373
|
+
});
|
|
5374
|
+
}
|
|
5375
|
+
/**
|
|
5376
|
+
* Get analytics for entire day, broken down by hours
|
|
5377
|
+
* @param {string} resourceName - Resource name
|
|
5378
|
+
* @param {string} field - Field name
|
|
5379
|
+
* @param {string} date - Date in YYYY-MM-DD format
|
|
5380
|
+
* @returns {Promise<Array>} Hourly analytics for the day
|
|
5381
|
+
*/
|
|
5382
|
+
async getDayByHour(resourceName, field, date) {
|
|
5383
|
+
return await this.getAnalytics(resourceName, field, {
|
|
5384
|
+
period: "hour",
|
|
5385
|
+
date
|
|
5386
|
+
});
|
|
5387
|
+
}
|
|
5388
|
+
/**
|
|
5389
|
+
* Get analytics for last N days, broken down by days
|
|
5390
|
+
* @param {string} resourceName - Resource name
|
|
5391
|
+
* @param {string} field - Field name
|
|
5392
|
+
* @param {number} days - Number of days to look back (default: 7)
|
|
5393
|
+
* @returns {Promise<Array>} Daily analytics
|
|
5394
|
+
*/
|
|
5395
|
+
async getLastNDays(resourceName, field, days = 7) {
|
|
5396
|
+
const dates = Array.from({ length: days }, (_, i) => {
|
|
5397
|
+
const date = /* @__PURE__ */ new Date();
|
|
5398
|
+
date.setDate(date.getDate() - i);
|
|
5399
|
+
return date.toISOString().substring(0, 10);
|
|
5400
|
+
}).reverse();
|
|
5401
|
+
return await this.getAnalytics(resourceName, field, {
|
|
5402
|
+
period: "day",
|
|
5403
|
+
startDate: dates[0],
|
|
5404
|
+
endDate: dates[dates.length - 1]
|
|
5405
|
+
});
|
|
5406
|
+
}
|
|
5407
|
+
/**
|
|
5408
|
+
* Get analytics for entire year, broken down by months
|
|
5409
|
+
* @param {string} resourceName - Resource name
|
|
5410
|
+
* @param {string} field - Field name
|
|
5411
|
+
* @param {number} year - Year (e.g., 2025)
|
|
5412
|
+
* @returns {Promise<Array>} Monthly analytics for the year
|
|
5413
|
+
*/
|
|
5414
|
+
async getYearByMonth(resourceName, field, year) {
|
|
5415
|
+
return await this.getAnalytics(resourceName, field, {
|
|
5416
|
+
period: "month",
|
|
5417
|
+
year
|
|
5418
|
+
});
|
|
5419
|
+
}
|
|
5420
|
+
/**
|
|
5421
|
+
* Get analytics for entire month, broken down by hours
|
|
5422
|
+
* @param {string} resourceName - Resource name
|
|
5423
|
+
* @param {string} field - Field name
|
|
5424
|
+
* @param {string} month - Month in YYYY-MM format (or 'last' for previous month)
|
|
5425
|
+
* @returns {Promise<Array>} Hourly analytics for the month (up to 24*31=744 records)
|
|
5426
|
+
*/
|
|
5427
|
+
async getMonthByHour(resourceName, field, month) {
|
|
5428
|
+
let year, monthNum;
|
|
5429
|
+
if (month === "last") {
|
|
5430
|
+
const now = /* @__PURE__ */ new Date();
|
|
5431
|
+
now.setMonth(now.getMonth() - 1);
|
|
5432
|
+
year = now.getFullYear();
|
|
5433
|
+
monthNum = now.getMonth() + 1;
|
|
5434
|
+
} else {
|
|
5435
|
+
year = parseInt(month.substring(0, 4));
|
|
5436
|
+
monthNum = parseInt(month.substring(5, 7));
|
|
5437
|
+
}
|
|
5438
|
+
const firstDay = new Date(year, monthNum - 1, 1);
|
|
5439
|
+
const lastDay = new Date(year, monthNum, 0);
|
|
5440
|
+
const startDate = firstDay.toISOString().substring(0, 10);
|
|
5441
|
+
const endDate = lastDay.toISOString().substring(0, 10);
|
|
5442
|
+
return await this.getAnalytics(resourceName, field, {
|
|
5443
|
+
period: "hour",
|
|
5444
|
+
startDate,
|
|
5445
|
+
endDate
|
|
5446
|
+
});
|
|
5447
|
+
}
|
|
5448
|
+
/**
|
|
5449
|
+
* Get top records by volume
|
|
5450
|
+
* @param {string} resourceName - Resource name
|
|
5451
|
+
* @param {string} field - Field name
|
|
5452
|
+
* @param {Object} options - Query options
|
|
5453
|
+
* @returns {Promise<Array>} Top records
|
|
5454
|
+
*/
|
|
5455
|
+
async getTopRecords(resourceName, field, options = {}) {
|
|
5456
|
+
if (!this.transactionResource) {
|
|
5457
|
+
throw new Error("Transaction resource not initialized");
|
|
5458
|
+
}
|
|
5459
|
+
const { period = "day", date, metric = "transactionCount", limit = 10 } = options;
|
|
5460
|
+
const [ok, err, transactions] = await tryFn(
|
|
5461
|
+
() => this.transactionResource.list()
|
|
5462
|
+
);
|
|
5463
|
+
if (!ok || !transactions) {
|
|
5464
|
+
return [];
|
|
5465
|
+
}
|
|
5466
|
+
let filtered = transactions;
|
|
5467
|
+
if (date) {
|
|
5468
|
+
if (period === "hour") {
|
|
5469
|
+
filtered = transactions.filter((t) => t.cohortHour && t.cohortHour.startsWith(date));
|
|
5470
|
+
} else if (period === "day") {
|
|
5471
|
+
filtered = transactions.filter((t) => t.cohortDate === date);
|
|
5472
|
+
} else if (period === "month") {
|
|
5473
|
+
filtered = transactions.filter((t) => t.cohortMonth && t.cohortMonth.startsWith(date));
|
|
5474
|
+
}
|
|
5475
|
+
}
|
|
5476
|
+
const byRecord = {};
|
|
5477
|
+
for (const txn of filtered) {
|
|
5478
|
+
const recordId = txn.originalId;
|
|
5479
|
+
if (!byRecord[recordId]) {
|
|
5480
|
+
byRecord[recordId] = { count: 0, sum: 0 };
|
|
5481
|
+
}
|
|
5482
|
+
byRecord[recordId].count++;
|
|
5483
|
+
byRecord[recordId].sum += txn.value;
|
|
5484
|
+
}
|
|
5485
|
+
const records = Object.entries(byRecord).map(([recordId, stats]) => ({
|
|
5486
|
+
recordId,
|
|
5487
|
+
count: stats.count,
|
|
5488
|
+
sum: stats.sum
|
|
5489
|
+
}));
|
|
5490
|
+
records.sort((a, b) => {
|
|
5491
|
+
if (metric === "transactionCount") {
|
|
5492
|
+
return b.count - a.count;
|
|
5493
|
+
} else if (metric === "totalValue") {
|
|
5494
|
+
return b.sum - a.sum;
|
|
5495
|
+
}
|
|
5496
|
+
return 0;
|
|
5497
|
+
});
|
|
5498
|
+
return records.slice(0, limit);
|
|
5499
|
+
}
|
|
5041
5500
|
}
|
|
5042
5501
|
|
|
5043
5502
|
class FullTextPlugin extends Plugin {
|
|
@@ -5902,6 +6361,253 @@ class MetricsPlugin extends Plugin {
|
|
|
5902
6361
|
}
|
|
5903
6362
|
}
|
|
5904
6363
|
|
|
6364
|
+
class SqsConsumer {
|
|
6365
|
+
constructor({ queueUrl, onMessage, onError, poolingInterval = 5e3, maxMessages = 10, region = "us-east-1", credentials, endpoint, driver = "sqs" }) {
|
|
6366
|
+
this.driver = driver;
|
|
6367
|
+
this.queueUrl = queueUrl;
|
|
6368
|
+
this.onMessage = onMessage;
|
|
6369
|
+
this.onError = onError;
|
|
6370
|
+
this.poolingInterval = poolingInterval;
|
|
6371
|
+
this.maxMessages = maxMessages;
|
|
6372
|
+
this.region = region;
|
|
6373
|
+
this.credentials = credentials;
|
|
6374
|
+
this.endpoint = endpoint;
|
|
6375
|
+
this.sqs = null;
|
|
6376
|
+
this._stopped = false;
|
|
6377
|
+
this._timer = null;
|
|
6378
|
+
this._pollPromise = null;
|
|
6379
|
+
this._pollResolve = null;
|
|
6380
|
+
this._SQSClient = null;
|
|
6381
|
+
this._ReceiveMessageCommand = null;
|
|
6382
|
+
this._DeleteMessageCommand = null;
|
|
6383
|
+
}
|
|
6384
|
+
async start() {
|
|
6385
|
+
const [ok, err, sdk] = await tryFn(() => import('@aws-sdk/client-sqs'));
|
|
6386
|
+
if (!ok) throw new Error("SqsConsumer: @aws-sdk/client-sqs is not installed. Please install it to use the SQS consumer.");
|
|
6387
|
+
const { SQSClient, ReceiveMessageCommand, DeleteMessageCommand } = sdk;
|
|
6388
|
+
this._SQSClient = SQSClient;
|
|
6389
|
+
this._ReceiveMessageCommand = ReceiveMessageCommand;
|
|
6390
|
+
this._DeleteMessageCommand = DeleteMessageCommand;
|
|
6391
|
+
this.sqs = new SQSClient({ region: this.region, credentials: this.credentials, endpoint: this.endpoint });
|
|
6392
|
+
this._stopped = false;
|
|
6393
|
+
this._pollPromise = new Promise((resolve) => {
|
|
6394
|
+
this._pollResolve = resolve;
|
|
6395
|
+
});
|
|
6396
|
+
this._poll();
|
|
6397
|
+
}
|
|
6398
|
+
async stop() {
|
|
6399
|
+
this._stopped = true;
|
|
6400
|
+
if (this._timer) {
|
|
6401
|
+
clearTimeout(this._timer);
|
|
6402
|
+
this._timer = null;
|
|
6403
|
+
}
|
|
6404
|
+
if (this._pollResolve) {
|
|
6405
|
+
this._pollResolve();
|
|
6406
|
+
}
|
|
6407
|
+
}
|
|
6408
|
+
async _poll() {
|
|
6409
|
+
if (this._stopped) {
|
|
6410
|
+
if (this._pollResolve) this._pollResolve();
|
|
6411
|
+
return;
|
|
6412
|
+
}
|
|
6413
|
+
const [ok, err, result] = await tryFn(async () => {
|
|
6414
|
+
const cmd = new this._ReceiveMessageCommand({
|
|
6415
|
+
QueueUrl: this.queueUrl,
|
|
6416
|
+
MaxNumberOfMessages: this.maxMessages,
|
|
6417
|
+
WaitTimeSeconds: 10,
|
|
6418
|
+
MessageAttributeNames: ["All"]
|
|
6419
|
+
});
|
|
6420
|
+
const { Messages } = await this.sqs.send(cmd);
|
|
6421
|
+
if (Messages && Messages.length > 0) {
|
|
6422
|
+
for (const msg of Messages) {
|
|
6423
|
+
const [okMsg, errMsg] = await tryFn(async () => {
|
|
6424
|
+
const parsedMsg = this._parseMessage(msg);
|
|
6425
|
+
await this.onMessage(parsedMsg, msg);
|
|
6426
|
+
await this.sqs.send(new this._DeleteMessageCommand({
|
|
6427
|
+
QueueUrl: this.queueUrl,
|
|
6428
|
+
ReceiptHandle: msg.ReceiptHandle
|
|
6429
|
+
}));
|
|
6430
|
+
});
|
|
6431
|
+
if (!okMsg && this.onError) {
|
|
6432
|
+
this.onError(errMsg, msg);
|
|
6433
|
+
}
|
|
6434
|
+
}
|
|
6435
|
+
}
|
|
6436
|
+
});
|
|
6437
|
+
if (!ok && this.onError) {
|
|
6438
|
+
this.onError(err);
|
|
6439
|
+
}
|
|
6440
|
+
this._timer = setTimeout(() => this._poll(), this.poolingInterval);
|
|
6441
|
+
}
|
|
6442
|
+
_parseMessage(msg) {
|
|
6443
|
+
let body;
|
|
6444
|
+
const [ok, err, parsed] = tryFn(() => JSON.parse(msg.Body));
|
|
6445
|
+
body = ok ? parsed : msg.Body;
|
|
6446
|
+
const attributes = {};
|
|
6447
|
+
if (msg.MessageAttributes) {
|
|
6448
|
+
for (const [k, v] of Object.entries(msg.MessageAttributes)) {
|
|
6449
|
+
attributes[k] = v.StringValue;
|
|
6450
|
+
}
|
|
6451
|
+
}
|
|
6452
|
+
return { $body: body, $attributes: attributes, $raw: msg };
|
|
6453
|
+
}
|
|
6454
|
+
}
|
|
6455
|
+
|
|
6456
|
+
class RabbitMqConsumer {
|
|
6457
|
+
constructor({ amqpUrl, queue, prefetch = 10, reconnectInterval = 2e3, onMessage, onError, driver = "rabbitmq" }) {
|
|
6458
|
+
this.amqpUrl = amqpUrl;
|
|
6459
|
+
this.queue = queue;
|
|
6460
|
+
this.prefetch = prefetch;
|
|
6461
|
+
this.reconnectInterval = reconnectInterval;
|
|
6462
|
+
this.onMessage = onMessage;
|
|
6463
|
+
this.onError = onError;
|
|
6464
|
+
this.driver = driver;
|
|
6465
|
+
this.connection = null;
|
|
6466
|
+
this.channel = null;
|
|
6467
|
+
this._stopped = false;
|
|
6468
|
+
}
|
|
6469
|
+
async start() {
|
|
6470
|
+
this._stopped = false;
|
|
6471
|
+
await this._connect();
|
|
6472
|
+
}
|
|
6473
|
+
async stop() {
|
|
6474
|
+
this._stopped = true;
|
|
6475
|
+
if (this.channel) await this.channel.close();
|
|
6476
|
+
if (this.connection) await this.connection.close();
|
|
6477
|
+
}
|
|
6478
|
+
async _connect() {
|
|
6479
|
+
const [ok, err] = await tryFn(async () => {
|
|
6480
|
+
const amqp = (await import('amqplib')).default;
|
|
6481
|
+
this.connection = await amqp.connect(this.amqpUrl);
|
|
6482
|
+
this.channel = await this.connection.createChannel();
|
|
6483
|
+
await this.channel.assertQueue(this.queue, { durable: true });
|
|
6484
|
+
this.channel.prefetch(this.prefetch);
|
|
6485
|
+
this.channel.consume(this.queue, async (msg) => {
|
|
6486
|
+
if (msg !== null) {
|
|
6487
|
+
const [okMsg, errMsg] = await tryFn(async () => {
|
|
6488
|
+
const content = JSON.parse(msg.content.toString());
|
|
6489
|
+
await this.onMessage({ $body: content, $raw: msg });
|
|
6490
|
+
this.channel.ack(msg);
|
|
6491
|
+
});
|
|
6492
|
+
if (!okMsg) {
|
|
6493
|
+
if (this.onError) this.onError(errMsg, msg);
|
|
6494
|
+
this.channel.nack(msg, false, false);
|
|
6495
|
+
}
|
|
6496
|
+
}
|
|
6497
|
+
});
|
|
6498
|
+
});
|
|
6499
|
+
if (!ok) {
|
|
6500
|
+
if (this.onError) this.onError(err);
|
|
6501
|
+
if (!this._stopped) {
|
|
6502
|
+
setTimeout(() => this._connect(), this.reconnectInterval);
|
|
6503
|
+
}
|
|
6504
|
+
}
|
|
6505
|
+
}
|
|
6506
|
+
}
|
|
6507
|
+
|
|
6508
|
+
const CONSUMER_DRIVERS = {
|
|
6509
|
+
sqs: SqsConsumer,
|
|
6510
|
+
rabbitmq: RabbitMqConsumer
|
|
6511
|
+
// kafka: KafkaConsumer, // futuro
|
|
6512
|
+
};
|
|
6513
|
+
function createConsumer(driver, config) {
|
|
6514
|
+
const ConsumerClass = CONSUMER_DRIVERS[driver];
|
|
6515
|
+
if (!ConsumerClass) {
|
|
6516
|
+
throw new Error(`Unknown consumer driver: ${driver}. Available: ${Object.keys(CONSUMER_DRIVERS).join(", ")}`);
|
|
6517
|
+
}
|
|
6518
|
+
return new ConsumerClass(config);
|
|
6519
|
+
}
|
|
6520
|
+
|
|
6521
|
+
class QueueConsumerPlugin {
|
|
6522
|
+
constructor(options = {}) {
|
|
6523
|
+
this.options = options;
|
|
6524
|
+
this.driversConfig = Array.isArray(options.consumers) ? options.consumers : [];
|
|
6525
|
+
this.consumers = [];
|
|
6526
|
+
}
|
|
6527
|
+
async setup(database) {
|
|
6528
|
+
this.database = database;
|
|
6529
|
+
for (const driverDef of this.driversConfig) {
|
|
6530
|
+
const { driver, config: driverConfig = {}, consumers: consumerDefs = [] } = driverDef;
|
|
6531
|
+
if (consumerDefs.length === 0 && driverDef.resources) {
|
|
6532
|
+
const { resources, driver: defDriver, config: nestedConfig, ...directConfig } = driverDef;
|
|
6533
|
+
const resourceList = Array.isArray(resources) ? resources : [resources];
|
|
6534
|
+
const flatConfig = nestedConfig ? { ...directConfig, ...nestedConfig } : directConfig;
|
|
6535
|
+
for (const resource of resourceList) {
|
|
6536
|
+
const consumer = createConsumer(driver, {
|
|
6537
|
+
...flatConfig,
|
|
6538
|
+
onMessage: (msg) => this._handleMessage(msg, resource),
|
|
6539
|
+
onError: (err, raw) => this._handleError(err, raw, resource)
|
|
6540
|
+
});
|
|
6541
|
+
await consumer.start();
|
|
6542
|
+
this.consumers.push(consumer);
|
|
6543
|
+
}
|
|
6544
|
+
} else {
|
|
6545
|
+
for (const consumerDef of consumerDefs) {
|
|
6546
|
+
const { resources, ...consumerConfig } = consumerDef;
|
|
6547
|
+
const resourceList = Array.isArray(resources) ? resources : [resources];
|
|
6548
|
+
for (const resource of resourceList) {
|
|
6549
|
+
const mergedConfig = { ...driverConfig, ...consumerConfig };
|
|
6550
|
+
const consumer = createConsumer(driver, {
|
|
6551
|
+
...mergedConfig,
|
|
6552
|
+
onMessage: (msg) => this._handleMessage(msg, resource),
|
|
6553
|
+
onError: (err, raw) => this._handleError(err, raw, resource)
|
|
6554
|
+
});
|
|
6555
|
+
await consumer.start();
|
|
6556
|
+
this.consumers.push(consumer);
|
|
6557
|
+
}
|
|
6558
|
+
}
|
|
6559
|
+
}
|
|
6560
|
+
}
|
|
6561
|
+
}
|
|
6562
|
+
async stop() {
|
|
6563
|
+
if (!Array.isArray(this.consumers)) this.consumers = [];
|
|
6564
|
+
for (const consumer of this.consumers) {
|
|
6565
|
+
if (consumer && typeof consumer.stop === "function") {
|
|
6566
|
+
await consumer.stop();
|
|
6567
|
+
}
|
|
6568
|
+
}
|
|
6569
|
+
this.consumers = [];
|
|
6570
|
+
}
|
|
6571
|
+
async _handleMessage(msg, configuredResource) {
|
|
6572
|
+
this.options;
|
|
6573
|
+
let body = msg.$body || msg;
|
|
6574
|
+
if (body.$body && !body.resource && !body.action && !body.data) {
|
|
6575
|
+
body = body.$body;
|
|
6576
|
+
}
|
|
6577
|
+
let resource = body.resource || msg.resource;
|
|
6578
|
+
let action = body.action || msg.action;
|
|
6579
|
+
let data = body.data || msg.data;
|
|
6580
|
+
if (!resource) {
|
|
6581
|
+
throw new Error("QueueConsumerPlugin: resource not found in message");
|
|
6582
|
+
}
|
|
6583
|
+
if (!action) {
|
|
6584
|
+
throw new Error("QueueConsumerPlugin: action not found in message");
|
|
6585
|
+
}
|
|
6586
|
+
const resourceObj = this.database.resources[resource];
|
|
6587
|
+
if (!resourceObj) throw new Error(`QueueConsumerPlugin: resource '${resource}' not found`);
|
|
6588
|
+
let result;
|
|
6589
|
+
const [ok, err, res] = await tryFn(async () => {
|
|
6590
|
+
if (action === "insert") {
|
|
6591
|
+
result = await resourceObj.insert(data);
|
|
6592
|
+
} else if (action === "update") {
|
|
6593
|
+
const { id: updateId, ...updateAttributes } = data;
|
|
6594
|
+
result = await resourceObj.update(updateId, updateAttributes);
|
|
6595
|
+
} else if (action === "delete") {
|
|
6596
|
+
result = await resourceObj.delete(data.id);
|
|
6597
|
+
} else {
|
|
6598
|
+
throw new Error(`QueueConsumerPlugin: unsupported action '${action}'`);
|
|
6599
|
+
}
|
|
6600
|
+
return result;
|
|
6601
|
+
});
|
|
6602
|
+
if (!ok) {
|
|
6603
|
+
throw err;
|
|
6604
|
+
}
|
|
6605
|
+
return res;
|
|
6606
|
+
}
|
|
6607
|
+
_handleError(err, raw, resourceName) {
|
|
6608
|
+
}
|
|
6609
|
+
}
|
|
6610
|
+
|
|
5905
6611
|
class BaseReplicator extends EventEmitter {
|
|
5906
6612
|
constructor(config = {}) {
|
|
5907
6613
|
super();
|
|
@@ -10858,7 +11564,7 @@ class Database extends EventEmitter {
|
|
|
10858
11564
|
this.id = idGenerator(7);
|
|
10859
11565
|
this.version = "1";
|
|
10860
11566
|
this.s3dbVersion = (() => {
|
|
10861
|
-
const [ok, err, version] = tryFn(() => true ? "10.0.
|
|
11567
|
+
const [ok, err, version] = tryFn(() => true ? "10.0.4" : "latest");
|
|
10862
11568
|
return ok ? version : "latest";
|
|
10863
11569
|
})();
|
|
10864
11570
|
this.resources = {};
|
|
@@ -14597,6 +15303,7 @@ exports.PartitionError = PartitionError;
|
|
|
14597
15303
|
exports.PermissionError = PermissionError;
|
|
14598
15304
|
exports.Plugin = Plugin;
|
|
14599
15305
|
exports.PluginObject = PluginObject;
|
|
15306
|
+
exports.QueueConsumerPlugin = QueueConsumerPlugin;
|
|
14600
15307
|
exports.ReplicatorPlugin = ReplicatorPlugin;
|
|
14601
15308
|
exports.Resource = Resource;
|
|
14602
15309
|
exports.ResourceError = ResourceError;
|