s3db.js 11.2.0 → 11.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/s3db-cli.js +55029 -0
- package/dist/s3db.cjs.js +532 -20
- package/dist/s3db.cjs.js.map +1 -1
- package/dist/s3db.es.js +532 -21
- package/dist/s3db.es.js.map +1 -1
- package/package.json +1 -4
- package/src/database.class.js +3 -0
- package/src/errors.js +115 -4
- package/src/plugins/cache/memory-cache.class.js +216 -33
- package/src/plugins/cache.plugin.js +85 -1
- package/src/plugins/eventual-consistency/analytics.js +145 -0
- package/src/plugins/eventual-consistency/config.js +4 -1
- package/src/plugins/eventual-consistency/consolidation.js +38 -4
- package/src/plugins/eventual-consistency/index.js +203 -1
- package/src/plugins/eventual-consistency/install.js +22 -0
- package/src/resource.class.js +8 -1
package/dist/s3db.cjs.js
CHANGED
|
@@ -15,6 +15,7 @@ var jsonStableStringify = require('json-stable-stringify');
|
|
|
15
15
|
var stream = require('stream');
|
|
16
16
|
var promisePool = require('@supercharge/promise-pool');
|
|
17
17
|
var web = require('node:stream/web');
|
|
18
|
+
var os$1 = require('node:os');
|
|
18
19
|
var lodashEs = require('lodash-es');
|
|
19
20
|
var http = require('http');
|
|
20
21
|
var https = require('https');
|
|
@@ -221,7 +222,7 @@ function calculateEffectiveLimit(config = {}) {
|
|
|
221
222
|
}
|
|
222
223
|
|
|
223
224
|
class BaseError extends Error {
|
|
224
|
-
constructor({ verbose, bucket, key, message, code, statusCode, requestId, awsMessage, original, commandName, commandInput, metadata, suggestion, ...rest }) {
|
|
225
|
+
constructor({ verbose, bucket, key, message, code, statusCode, requestId, awsMessage, original, commandName, commandInput, metadata, suggestion, description, ...rest }) {
|
|
225
226
|
if (verbose) message = message + `
|
|
226
227
|
|
|
227
228
|
Verbose:
|
|
@@ -247,6 +248,7 @@ ${JSON.stringify(rest, null, 2)}`;
|
|
|
247
248
|
this.commandInput = commandInput;
|
|
248
249
|
this.metadata = metadata;
|
|
249
250
|
this.suggestion = suggestion;
|
|
251
|
+
this.description = description;
|
|
250
252
|
this.data = { bucket, key, ...rest, verbose, message };
|
|
251
253
|
}
|
|
252
254
|
toJson() {
|
|
@@ -264,6 +266,7 @@ ${JSON.stringify(rest, null, 2)}`;
|
|
|
264
266
|
commandInput: this.commandInput,
|
|
265
267
|
metadata: this.metadata,
|
|
266
268
|
suggestion: this.suggestion,
|
|
269
|
+
description: this.description,
|
|
267
270
|
data: this.data,
|
|
268
271
|
original: this.original,
|
|
269
272
|
stack: this.stack
|
|
@@ -456,7 +459,107 @@ class ResourceError extends S3dbError {
|
|
|
456
459
|
}
|
|
457
460
|
class PartitionError extends S3dbError {
|
|
458
461
|
constructor(message, details = {}) {
|
|
459
|
-
|
|
462
|
+
let description = details.description;
|
|
463
|
+
if (!description && details.resourceName && details.partitionName && details.fieldName) {
|
|
464
|
+
const { resourceName, partitionName, fieldName, availableFields = [] } = details;
|
|
465
|
+
description = `
|
|
466
|
+
Partition Field Validation Error
|
|
467
|
+
|
|
468
|
+
Resource: ${resourceName}
|
|
469
|
+
Partition: ${partitionName}
|
|
470
|
+
Missing Field: ${fieldName}
|
|
471
|
+
|
|
472
|
+
Available fields in schema:
|
|
473
|
+
${availableFields.map((f) => ` \u2022 ${f}`).join("\n") || " (no fields defined)"}
|
|
474
|
+
|
|
475
|
+
Possible causes:
|
|
476
|
+
1. Field was removed from schema but partition still references it
|
|
477
|
+
2. Typo in partition field name
|
|
478
|
+
3. Nested field path is incorrect (use dot notation like 'utm.source')
|
|
479
|
+
|
|
480
|
+
Solution:
|
|
481
|
+
${details.strictValidation === false ? " \u2022 Update partition definition to use existing fields" : ` \u2022 Add missing field to schema, OR
|
|
482
|
+
\u2022 Update partition definition to use existing fields, OR
|
|
483
|
+
\u2022 Use strictValidation: false to skip this check during testing`}
|
|
484
|
+
|
|
485
|
+
Docs: https://docs.s3db.js.org/resources/partitions#validation
|
|
486
|
+
`.trim();
|
|
487
|
+
}
|
|
488
|
+
super(message, {
|
|
489
|
+
...details,
|
|
490
|
+
description,
|
|
491
|
+
suggestion: details.suggestion || "Check partition definition, fields, and input values."
|
|
492
|
+
});
|
|
493
|
+
}
|
|
494
|
+
}
|
|
495
|
+
class AnalyticsNotEnabledError extends S3dbError {
|
|
496
|
+
constructor(details = {}) {
|
|
497
|
+
const {
|
|
498
|
+
pluginName = "EventualConsistency",
|
|
499
|
+
resourceName = "unknown",
|
|
500
|
+
field = "unknown",
|
|
501
|
+
configuredResources = [],
|
|
502
|
+
registeredResources = [],
|
|
503
|
+
pluginInitialized = false,
|
|
504
|
+
...rest
|
|
505
|
+
} = details;
|
|
506
|
+
const message = `Analytics not enabled for ${resourceName}.${field}`;
|
|
507
|
+
const description = `
|
|
508
|
+
Analytics Not Enabled
|
|
509
|
+
|
|
510
|
+
Plugin: ${pluginName}
|
|
511
|
+
Resource: ${resourceName}
|
|
512
|
+
Field: ${field}
|
|
513
|
+
|
|
514
|
+
Diagnostics:
|
|
515
|
+
\u2022 Plugin initialized: ${pluginInitialized ? "\u2713 Yes" : "\u2717 No"}
|
|
516
|
+
\u2022 Analytics resources created: ${registeredResources.length}/${configuredResources.length}
|
|
517
|
+
${configuredResources.map((r) => {
|
|
518
|
+
const exists = registeredResources.includes(r);
|
|
519
|
+
return ` ${exists ? "\u2713" : "\u2717"} ${r}${!exists ? " (missing)" : ""}`;
|
|
520
|
+
}).join("\n")}
|
|
521
|
+
|
|
522
|
+
Possible causes:
|
|
523
|
+
1. Resource not created yet - Analytics resources are created when db.createResource() is called
|
|
524
|
+
2. Resource created before plugin initialization - Plugin must be initialized before resources
|
|
525
|
+
3. Field not configured in analytics.resources config
|
|
526
|
+
|
|
527
|
+
Correct initialization order:
|
|
528
|
+
1. Create database: const db = new Database({ ... })
|
|
529
|
+
2. Install plugins: await db.connect() (triggers plugin.install())
|
|
530
|
+
3. Create resources: await db.createResource({ name: '${resourceName}', ... })
|
|
531
|
+
4. Analytics resources are auto-created by plugin
|
|
532
|
+
|
|
533
|
+
Example fix:
|
|
534
|
+
const db = new Database({
|
|
535
|
+
bucket: 'my-bucket',
|
|
536
|
+
plugins: [new EventualConsistencyPlugin({
|
|
537
|
+
resources: {
|
|
538
|
+
'${resourceName}': {
|
|
539
|
+
fields: {
|
|
540
|
+
'${field}': { type: 'counter', analytics: true }
|
|
541
|
+
}
|
|
542
|
+
}
|
|
543
|
+
}
|
|
544
|
+
})]
|
|
545
|
+
});
|
|
546
|
+
|
|
547
|
+
await db.connect(); // Plugin initialized here
|
|
548
|
+
await db.createResource({ name: '${resourceName}', ... }); // Analytics resource created here
|
|
549
|
+
|
|
550
|
+
Docs: https://docs.s3db.js.org/plugins/eventual-consistency#troubleshooting
|
|
551
|
+
`.trim();
|
|
552
|
+
super(message, {
|
|
553
|
+
...rest,
|
|
554
|
+
pluginName,
|
|
555
|
+
resourceName,
|
|
556
|
+
field,
|
|
557
|
+
configuredResources,
|
|
558
|
+
registeredResources,
|
|
559
|
+
pluginInitialized,
|
|
560
|
+
description,
|
|
561
|
+
suggestion: "Ensure resources are created after plugin initialization. Check plugin configuration and resource creation order."
|
|
562
|
+
});
|
|
460
563
|
}
|
|
461
564
|
}
|
|
462
565
|
|
|
@@ -3605,6 +3708,24 @@ class MemoryCache extends Cache {
|
|
|
3605
3708
|
this.cache = {};
|
|
3606
3709
|
this.meta = {};
|
|
3607
3710
|
this.maxSize = config.maxSize !== void 0 ? config.maxSize : 1e3;
|
|
3711
|
+
if (config.maxMemoryBytes && config.maxMemoryBytes > 0 && config.maxMemoryPercent && config.maxMemoryPercent > 0) {
|
|
3712
|
+
throw new Error(
|
|
3713
|
+
"[MemoryCache] Cannot use both maxMemoryBytes and maxMemoryPercent. Choose one: maxMemoryBytes (absolute) or maxMemoryPercent (0...1 fraction)."
|
|
3714
|
+
);
|
|
3715
|
+
}
|
|
3716
|
+
if (config.maxMemoryPercent && config.maxMemoryPercent > 0) {
|
|
3717
|
+
if (config.maxMemoryPercent > 1) {
|
|
3718
|
+
throw new Error(
|
|
3719
|
+
`[MemoryCache] maxMemoryPercent must be between 0 and 1 (e.g., 0.1 for 10%). Received: ${config.maxMemoryPercent}`
|
|
3720
|
+
);
|
|
3721
|
+
}
|
|
3722
|
+
const totalMemory = os$1.totalmem();
|
|
3723
|
+
this.maxMemoryBytes = Math.floor(totalMemory * config.maxMemoryPercent);
|
|
3724
|
+
this.maxMemoryPercent = config.maxMemoryPercent;
|
|
3725
|
+
} else {
|
|
3726
|
+
this.maxMemoryBytes = config.maxMemoryBytes !== void 0 ? config.maxMemoryBytes : 0;
|
|
3727
|
+
this.maxMemoryPercent = 0;
|
|
3728
|
+
}
|
|
3608
3729
|
this.ttl = config.ttl !== void 0 ? config.ttl : 3e5;
|
|
3609
3730
|
this.enableCompression = config.enableCompression !== void 0 ? config.enableCompression : false;
|
|
3610
3731
|
this.compressionThreshold = config.compressionThreshold !== void 0 ? config.compressionThreshold : 1024;
|
|
@@ -3614,23 +3735,18 @@ class MemoryCache extends Cache {
|
|
|
3614
3735
|
totalCompressedSize: 0,
|
|
3615
3736
|
compressionRatio: 0
|
|
3616
3737
|
};
|
|
3738
|
+
this.currentMemoryBytes = 0;
|
|
3739
|
+
this.evictedDueToMemory = 0;
|
|
3617
3740
|
}
|
|
3618
3741
|
async _set(key, data) {
|
|
3619
|
-
if (this.maxSize > 0 && Object.keys(this.cache).length >= this.maxSize) {
|
|
3620
|
-
const oldestKey = Object.entries(this.meta).sort((a, b) => a[1].ts - b[1].ts)[0]?.[0];
|
|
3621
|
-
if (oldestKey) {
|
|
3622
|
-
delete this.cache[oldestKey];
|
|
3623
|
-
delete this.meta[oldestKey];
|
|
3624
|
-
}
|
|
3625
|
-
}
|
|
3626
3742
|
let finalData = data;
|
|
3627
3743
|
let compressed = false;
|
|
3628
3744
|
let originalSize = 0;
|
|
3629
3745
|
let compressedSize = 0;
|
|
3746
|
+
const serialized = JSON.stringify(data);
|
|
3747
|
+
originalSize = Buffer.byteLength(serialized, "utf8");
|
|
3630
3748
|
if (this.enableCompression) {
|
|
3631
3749
|
try {
|
|
3632
|
-
const serialized = JSON.stringify(data);
|
|
3633
|
-
originalSize = Buffer.byteLength(serialized, "utf8");
|
|
3634
3750
|
if (originalSize >= this.compressionThreshold) {
|
|
3635
3751
|
const compressedBuffer = zlib.gzipSync(Buffer.from(serialized, "utf8"));
|
|
3636
3752
|
finalData = {
|
|
@@ -3649,13 +3765,42 @@ class MemoryCache extends Cache {
|
|
|
3649
3765
|
console.warn(`[MemoryCache] Compression failed for key '${key}':`, error.message);
|
|
3650
3766
|
}
|
|
3651
3767
|
}
|
|
3768
|
+
const itemSize = compressed ? compressedSize : originalSize;
|
|
3769
|
+
if (Object.prototype.hasOwnProperty.call(this.cache, key)) {
|
|
3770
|
+
const oldSize = this.meta[key]?.compressedSize || 0;
|
|
3771
|
+
this.currentMemoryBytes -= oldSize;
|
|
3772
|
+
}
|
|
3773
|
+
if (this.maxMemoryBytes > 0) {
|
|
3774
|
+
while (this.currentMemoryBytes + itemSize > this.maxMemoryBytes && Object.keys(this.cache).length > 0) {
|
|
3775
|
+
const oldestKey = Object.entries(this.meta).sort((a, b) => a[1].ts - b[1].ts)[0]?.[0];
|
|
3776
|
+
if (oldestKey) {
|
|
3777
|
+
const evictedSize = this.meta[oldestKey]?.compressedSize || 0;
|
|
3778
|
+
delete this.cache[oldestKey];
|
|
3779
|
+
delete this.meta[oldestKey];
|
|
3780
|
+
this.currentMemoryBytes -= evictedSize;
|
|
3781
|
+
this.evictedDueToMemory++;
|
|
3782
|
+
} else {
|
|
3783
|
+
break;
|
|
3784
|
+
}
|
|
3785
|
+
}
|
|
3786
|
+
}
|
|
3787
|
+
if (this.maxSize > 0 && Object.keys(this.cache).length >= this.maxSize) {
|
|
3788
|
+
const oldestKey = Object.entries(this.meta).sort((a, b) => a[1].ts - b[1].ts)[0]?.[0];
|
|
3789
|
+
if (oldestKey) {
|
|
3790
|
+
const evictedSize = this.meta[oldestKey]?.compressedSize || 0;
|
|
3791
|
+
delete this.cache[oldestKey];
|
|
3792
|
+
delete this.meta[oldestKey];
|
|
3793
|
+
this.currentMemoryBytes -= evictedSize;
|
|
3794
|
+
}
|
|
3795
|
+
}
|
|
3652
3796
|
this.cache[key] = finalData;
|
|
3653
3797
|
this.meta[key] = {
|
|
3654
3798
|
ts: Date.now(),
|
|
3655
3799
|
compressed,
|
|
3656
3800
|
originalSize,
|
|
3657
|
-
compressedSize:
|
|
3801
|
+
compressedSize: itemSize
|
|
3658
3802
|
};
|
|
3803
|
+
this.currentMemoryBytes += itemSize;
|
|
3659
3804
|
return data;
|
|
3660
3805
|
}
|
|
3661
3806
|
async _get(key) {
|
|
@@ -3663,7 +3808,9 @@ class MemoryCache extends Cache {
|
|
|
3663
3808
|
if (this.ttl > 0) {
|
|
3664
3809
|
const now = Date.now();
|
|
3665
3810
|
const meta = this.meta[key];
|
|
3666
|
-
if (meta && now - meta.ts > this.ttl
|
|
3811
|
+
if (meta && now - meta.ts > this.ttl) {
|
|
3812
|
+
const itemSize = meta.compressedSize || 0;
|
|
3813
|
+
this.currentMemoryBytes -= itemSize;
|
|
3667
3814
|
delete this.cache[key];
|
|
3668
3815
|
delete this.meta[key];
|
|
3669
3816
|
return null;
|
|
@@ -3685,6 +3832,10 @@ class MemoryCache extends Cache {
|
|
|
3685
3832
|
return rawData;
|
|
3686
3833
|
}
|
|
3687
3834
|
async _del(key) {
|
|
3835
|
+
if (Object.prototype.hasOwnProperty.call(this.cache, key)) {
|
|
3836
|
+
const itemSize = this.meta[key]?.compressedSize || 0;
|
|
3837
|
+
this.currentMemoryBytes -= itemSize;
|
|
3838
|
+
}
|
|
3688
3839
|
delete this.cache[key];
|
|
3689
3840
|
delete this.meta[key];
|
|
3690
3841
|
return true;
|
|
@@ -3693,10 +3844,13 @@ class MemoryCache extends Cache {
|
|
|
3693
3844
|
if (!prefix) {
|
|
3694
3845
|
this.cache = {};
|
|
3695
3846
|
this.meta = {};
|
|
3847
|
+
this.currentMemoryBytes = 0;
|
|
3696
3848
|
return true;
|
|
3697
3849
|
}
|
|
3698
3850
|
for (const key of Object.keys(this.cache)) {
|
|
3699
3851
|
if (key.startsWith(prefix)) {
|
|
3852
|
+
const itemSize = this.meta[key]?.compressedSize || 0;
|
|
3853
|
+
this.currentMemoryBytes -= itemSize;
|
|
3700
3854
|
delete this.cache[key];
|
|
3701
3855
|
delete this.meta[key];
|
|
3702
3856
|
}
|
|
@@ -3734,6 +3888,53 @@ class MemoryCache extends Cache {
|
|
|
3734
3888
|
}
|
|
3735
3889
|
};
|
|
3736
3890
|
}
|
|
3891
|
+
/**
|
|
3892
|
+
* Get memory usage statistics
|
|
3893
|
+
* @returns {Object} Memory stats including current usage, limits, and eviction counts
|
|
3894
|
+
*/
|
|
3895
|
+
getMemoryStats() {
|
|
3896
|
+
const totalItems = Object.keys(this.cache).length;
|
|
3897
|
+
const memoryUsagePercent = this.maxMemoryBytes > 0 ? (this.currentMemoryBytes / this.maxMemoryBytes * 100).toFixed(2) : 0;
|
|
3898
|
+
const systemMemory = {
|
|
3899
|
+
total: os$1.totalmem(),
|
|
3900
|
+
free: os$1.freemem(),
|
|
3901
|
+
used: os$1.totalmem() - os$1.freemem()
|
|
3902
|
+
};
|
|
3903
|
+
const cachePercentOfTotal = systemMemory.total > 0 ? (this.currentMemoryBytes / systemMemory.total * 100).toFixed(2) : 0;
|
|
3904
|
+
return {
|
|
3905
|
+
currentMemoryBytes: this.currentMemoryBytes,
|
|
3906
|
+
maxMemoryBytes: this.maxMemoryBytes,
|
|
3907
|
+
maxMemoryPercent: this.maxMemoryPercent,
|
|
3908
|
+
memoryUsagePercent: parseFloat(memoryUsagePercent),
|
|
3909
|
+
cachePercentOfSystemMemory: parseFloat(cachePercentOfTotal),
|
|
3910
|
+
totalItems,
|
|
3911
|
+
maxSize: this.maxSize,
|
|
3912
|
+
evictedDueToMemory: this.evictedDueToMemory,
|
|
3913
|
+
averageItemSize: totalItems > 0 ? Math.round(this.currentMemoryBytes / totalItems) : 0,
|
|
3914
|
+
memoryUsage: {
|
|
3915
|
+
current: this._formatBytes(this.currentMemoryBytes),
|
|
3916
|
+
max: this.maxMemoryBytes > 0 ? this._formatBytes(this.maxMemoryBytes) : "unlimited",
|
|
3917
|
+
available: this.maxMemoryBytes > 0 ? this._formatBytes(this.maxMemoryBytes - this.currentMemoryBytes) : "unlimited"
|
|
3918
|
+
},
|
|
3919
|
+
systemMemory: {
|
|
3920
|
+
total: this._formatBytes(systemMemory.total),
|
|
3921
|
+
free: this._formatBytes(systemMemory.free),
|
|
3922
|
+
used: this._formatBytes(systemMemory.used),
|
|
3923
|
+
cachePercent: `${cachePercentOfTotal}%`
|
|
3924
|
+
}
|
|
3925
|
+
};
|
|
3926
|
+
}
|
|
3927
|
+
/**
|
|
3928
|
+
* Format bytes to human-readable format
|
|
3929
|
+
* @private
|
|
3930
|
+
*/
|
|
3931
|
+
_formatBytes(bytes) {
|
|
3932
|
+
if (bytes === 0) return "0 B";
|
|
3933
|
+
const k = 1024;
|
|
3934
|
+
const sizes = ["B", "KB", "MB", "GB"];
|
|
3935
|
+
const i = Math.floor(Math.log(bytes) / Math.log(k));
|
|
3936
|
+
return `${(bytes / Math.pow(k, i)).toFixed(2)} ${sizes[i]}`;
|
|
3937
|
+
}
|
|
3737
3938
|
}
|
|
3738
3939
|
|
|
3739
3940
|
class FilesystemCache extends Cache {
|
|
@@ -4563,8 +4764,10 @@ class CachePlugin extends Plugin {
|
|
|
4563
4764
|
config: {
|
|
4564
4765
|
ttl: options.ttl,
|
|
4565
4766
|
maxSize: options.maxSize,
|
|
4767
|
+
maxMemoryBytes: options.maxMemoryBytes,
|
|
4768
|
+
maxMemoryPercent: options.maxMemoryPercent,
|
|
4566
4769
|
...options.config
|
|
4567
|
-
// Driver-specific config (can override ttl/maxSize)
|
|
4770
|
+
// Driver-specific config (can override ttl/maxSize/maxMemoryBytes/maxMemoryPercent)
|
|
4568
4771
|
},
|
|
4569
4772
|
// Resource filtering
|
|
4570
4773
|
include: options.include || null,
|
|
@@ -5105,8 +5308,10 @@ function createConfig(options, detectedTimezone) {
|
|
|
5105
5308
|
consolidationWindow: consolidation.window ?? 24,
|
|
5106
5309
|
autoConsolidate: consolidation.auto !== false,
|
|
5107
5310
|
mode: consolidation.mode || "async",
|
|
5108
|
-
// ✅
|
|
5311
|
+
// ✅ Performance tuning - Mark applied concurrency (default 50, up from 10)
|
|
5109
5312
|
markAppliedConcurrency: consolidation.markAppliedConcurrency ?? 50,
|
|
5313
|
+
// ✅ Performance tuning - Recalculate concurrency (default 50, up from 10)
|
|
5314
|
+
recalculateConcurrency: consolidation.recalculateConcurrency ?? 50,
|
|
5110
5315
|
// Late arrivals
|
|
5111
5316
|
lateArrivalStrategy: lateArrivals.strategy || "warn",
|
|
5112
5317
|
// Batch transactions
|
|
@@ -5935,11 +6140,33 @@ async function consolidateRecord(originalId, transactionResource, targetResource
|
|
|
5935
6140
|
const transactionsToUpdate = transactions.filter((txn) => txn.id !== "__synthetic__");
|
|
5936
6141
|
const markAppliedConcurrency = config.markAppliedConcurrency || 50;
|
|
5937
6142
|
const { results, errors } = await promisePool.PromisePool.for(transactionsToUpdate).withConcurrency(markAppliedConcurrency).process(async (txn) => {
|
|
6143
|
+
const txnWithCohorts = ensureCohortHour(txn, config.cohort.timezone, false);
|
|
6144
|
+
const updateData = { applied: true };
|
|
6145
|
+
if (txnWithCohorts.cohortHour && !txn.cohortHour) {
|
|
6146
|
+
updateData.cohortHour = txnWithCohorts.cohortHour;
|
|
6147
|
+
}
|
|
6148
|
+
if (txnWithCohorts.cohortDate && !txn.cohortDate) {
|
|
6149
|
+
updateData.cohortDate = txnWithCohorts.cohortDate;
|
|
6150
|
+
}
|
|
6151
|
+
if (txnWithCohorts.cohortWeek && !txn.cohortWeek) {
|
|
6152
|
+
updateData.cohortWeek = txnWithCohorts.cohortWeek;
|
|
6153
|
+
}
|
|
6154
|
+
if (txnWithCohorts.cohortMonth && !txn.cohortMonth) {
|
|
6155
|
+
updateData.cohortMonth = txnWithCohorts.cohortMonth;
|
|
6156
|
+
}
|
|
6157
|
+
if (txn.value === null || txn.value === void 0) {
|
|
6158
|
+
updateData.value = 1;
|
|
6159
|
+
}
|
|
5938
6160
|
const [ok2, err2] = await tryFn(
|
|
5939
|
-
() => transactionResource.update(txn.id,
|
|
6161
|
+
() => transactionResource.update(txn.id, updateData)
|
|
5940
6162
|
);
|
|
5941
6163
|
if (!ok2 && config.verbose) {
|
|
5942
|
-
console.warn(
|
|
6164
|
+
console.warn(
|
|
6165
|
+
`[EventualConsistency] Failed to mark transaction ${txn.id} as applied:`,
|
|
6166
|
+
err2?.message,
|
|
6167
|
+
"Update data:",
|
|
6168
|
+
updateData
|
|
6169
|
+
);
|
|
5943
6170
|
}
|
|
5944
6171
|
return ok2;
|
|
5945
6172
|
});
|
|
@@ -6131,7 +6358,8 @@ async function recalculateRecord(originalId, transactionResource, targetResource
|
|
|
6131
6358
|
}
|
|
6132
6359
|
}
|
|
6133
6360
|
const transactionsToReset = allTransactions.filter((txn) => txn.source !== "anchor");
|
|
6134
|
-
const
|
|
6361
|
+
const recalculateConcurrency = config.recalculateConcurrency || 50;
|
|
6362
|
+
const { results, errors } = await promisePool.PromisePool.for(transactionsToReset).withConcurrency(recalculateConcurrency).process(async (txn) => {
|
|
6135
6363
|
const [ok, err] = await tryFn(
|
|
6136
6364
|
() => transactionResource.update(txn.id, { applied: false })
|
|
6137
6365
|
);
|
|
@@ -6998,6 +7226,80 @@ async function getLastNMonths(resourceName, field, months = 12, options, fieldHa
|
|
|
6998
7226
|
}
|
|
6999
7227
|
return data;
|
|
7000
7228
|
}
|
|
7229
|
+
async function getRawEvents(resourceName, field, options, fieldHandlers) {
|
|
7230
|
+
const resourceHandlers = fieldHandlers.get(resourceName);
|
|
7231
|
+
if (!resourceHandlers) {
|
|
7232
|
+
throw new Error(`No eventual consistency configured for resource: ${resourceName}`);
|
|
7233
|
+
}
|
|
7234
|
+
const handler = resourceHandlers.get(field);
|
|
7235
|
+
if (!handler) {
|
|
7236
|
+
throw new Error(`No eventual consistency configured for field: ${resourceName}.${field}`);
|
|
7237
|
+
}
|
|
7238
|
+
if (!handler.transactionResource) {
|
|
7239
|
+
throw new Error("Transaction resource not initialized");
|
|
7240
|
+
}
|
|
7241
|
+
const {
|
|
7242
|
+
recordId,
|
|
7243
|
+
startDate,
|
|
7244
|
+
endDate,
|
|
7245
|
+
cohortDate,
|
|
7246
|
+
cohortHour,
|
|
7247
|
+
cohortMonth,
|
|
7248
|
+
applied,
|
|
7249
|
+
operation,
|
|
7250
|
+
limit
|
|
7251
|
+
} = options;
|
|
7252
|
+
const query = {};
|
|
7253
|
+
if (recordId !== void 0) {
|
|
7254
|
+
query.originalId = recordId;
|
|
7255
|
+
}
|
|
7256
|
+
if (applied !== void 0) {
|
|
7257
|
+
query.applied = applied;
|
|
7258
|
+
}
|
|
7259
|
+
const [ok, err, allTransactions] = await tryFn(
|
|
7260
|
+
() => handler.transactionResource.query(query)
|
|
7261
|
+
);
|
|
7262
|
+
if (!ok || !allTransactions) {
|
|
7263
|
+
return [];
|
|
7264
|
+
}
|
|
7265
|
+
let filtered = allTransactions;
|
|
7266
|
+
if (operation !== void 0) {
|
|
7267
|
+
filtered = filtered.filter((t) => t.operation === operation);
|
|
7268
|
+
}
|
|
7269
|
+
if (cohortDate) {
|
|
7270
|
+
filtered = filtered.filter((t) => t.cohortDate === cohortDate);
|
|
7271
|
+
}
|
|
7272
|
+
if (cohortHour) {
|
|
7273
|
+
filtered = filtered.filter((t) => t.cohortHour === cohortHour);
|
|
7274
|
+
}
|
|
7275
|
+
if (cohortMonth) {
|
|
7276
|
+
filtered = filtered.filter((t) => t.cohortMonth === cohortMonth);
|
|
7277
|
+
}
|
|
7278
|
+
if (startDate && endDate) {
|
|
7279
|
+
const isHourly = startDate.length > 10;
|
|
7280
|
+
const cohortField = isHourly ? "cohortHour" : "cohortDate";
|
|
7281
|
+
filtered = filtered.filter(
|
|
7282
|
+
(t) => t[cohortField] && t[cohortField] >= startDate && t[cohortField] <= endDate
|
|
7283
|
+
);
|
|
7284
|
+
} else if (startDate) {
|
|
7285
|
+
const isHourly = startDate.length > 10;
|
|
7286
|
+
const cohortField = isHourly ? "cohortHour" : "cohortDate";
|
|
7287
|
+
filtered = filtered.filter((t) => t[cohortField] && t[cohortField] >= startDate);
|
|
7288
|
+
} else if (endDate) {
|
|
7289
|
+
const isHourly = endDate.length > 10;
|
|
7290
|
+
const cohortField = isHourly ? "cohortHour" : "cohortDate";
|
|
7291
|
+
filtered = filtered.filter((t) => t[cohortField] && t[cohortField] <= endDate);
|
|
7292
|
+
}
|
|
7293
|
+
filtered.sort((a, b) => {
|
|
7294
|
+
const aTime = new Date(a.timestamp || a.createdAt).getTime();
|
|
7295
|
+
const bTime = new Date(b.timestamp || b.createdAt).getTime();
|
|
7296
|
+
return bTime - aTime;
|
|
7297
|
+
});
|
|
7298
|
+
if (limit && limit > 0) {
|
|
7299
|
+
filtered = filtered.slice(0, limit);
|
|
7300
|
+
}
|
|
7301
|
+
return filtered;
|
|
7302
|
+
}
|
|
7001
7303
|
|
|
7002
7304
|
function addHelperMethods(resource, plugin, config) {
|
|
7003
7305
|
resource.set = async (id, field, value) => {
|
|
@@ -7222,6 +7524,28 @@ async function createAnalyticsResource(handler, database, resourceName, fieldNam
|
|
|
7222
7524
|
},
|
|
7223
7525
|
behavior: "body-overflow",
|
|
7224
7526
|
timestamps: false,
|
|
7527
|
+
asyncPartitions: true,
|
|
7528
|
+
// ✅ Multi-attribute partitions for optimal analytics query performance
|
|
7529
|
+
partitions: {
|
|
7530
|
+
// Query by period (hour/day/week/month)
|
|
7531
|
+
byPeriod: {
|
|
7532
|
+
fields: { period: "string" }
|
|
7533
|
+
},
|
|
7534
|
+
// Query by period + cohort (e.g., all hour records for specific hours)
|
|
7535
|
+
byPeriodCohort: {
|
|
7536
|
+
fields: {
|
|
7537
|
+
period: "string",
|
|
7538
|
+
cohort: "string"
|
|
7539
|
+
}
|
|
7540
|
+
},
|
|
7541
|
+
// Query by field + period (e.g., all daily analytics for clicks field)
|
|
7542
|
+
byFieldPeriod: {
|
|
7543
|
+
fields: {
|
|
7544
|
+
field: "string",
|
|
7545
|
+
period: "string"
|
|
7546
|
+
}
|
|
7547
|
+
}
|
|
7548
|
+
},
|
|
7225
7549
|
createdBy: "EventualConsistencyPlugin"
|
|
7226
7550
|
})
|
|
7227
7551
|
);
|
|
@@ -7733,6 +8057,185 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
7733
8057
|
async getLastNMonths(resourceName, field, months = 12, options = {}) {
|
|
7734
8058
|
return await getLastNMonths(resourceName, field, months, options, this.fieldHandlers);
|
|
7735
8059
|
}
|
|
8060
|
+
/**
|
|
8061
|
+
* Get raw transaction events for custom aggregation
|
|
8062
|
+
*
|
|
8063
|
+
* This method provides direct access to the underlying transaction events,
|
|
8064
|
+
* allowing developers to perform custom aggregations beyond the pre-built analytics.
|
|
8065
|
+
* Useful for complex queries, custom metrics, or when you need the raw event data.
|
|
8066
|
+
*
|
|
8067
|
+
* @param {string} resourceName - Resource name
|
|
8068
|
+
* @param {string} field - Field name
|
|
8069
|
+
* @param {Object} options - Query options
|
|
8070
|
+
* @param {string} options.recordId - Filter by specific record ID
|
|
8071
|
+
* @param {string} options.startDate - Start date filter (YYYY-MM-DD or YYYY-MM-DDTHH)
|
|
8072
|
+
* @param {string} options.endDate - End date filter (YYYY-MM-DD or YYYY-MM-DDTHH)
|
|
8073
|
+
* @param {string} options.cohortDate - Filter by cohort date (YYYY-MM-DD)
|
|
8074
|
+
* @param {string} options.cohortHour - Filter by cohort hour (YYYY-MM-DDTHH)
|
|
8075
|
+
* @param {string} options.cohortMonth - Filter by cohort month (YYYY-MM)
|
|
8076
|
+
* @param {boolean} options.applied - Filter by applied status (true/false/undefined for both)
|
|
8077
|
+
* @param {string} options.operation - Filter by operation type ('add', 'sub', 'set')
|
|
8078
|
+
* @param {number} options.limit - Maximum number of events to return
|
|
8079
|
+
* @returns {Promise<Array>} Raw transaction events
|
|
8080
|
+
*
|
|
8081
|
+
* @example
|
|
8082
|
+
* // Get all events for a specific record
|
|
8083
|
+
* const events = await plugin.getRawEvents('wallets', 'balance', {
|
|
8084
|
+
* recordId: 'wallet1'
|
|
8085
|
+
* });
|
|
8086
|
+
*
|
|
8087
|
+
* @example
|
|
8088
|
+
* // Get events for a specific time range
|
|
8089
|
+
* const events = await plugin.getRawEvents('wallets', 'balance', {
|
|
8090
|
+
* startDate: '2025-10-01',
|
|
8091
|
+
* endDate: '2025-10-31'
|
|
8092
|
+
* });
|
|
8093
|
+
*
|
|
8094
|
+
* @example
|
|
8095
|
+
* // Get only pending (unapplied) transactions
|
|
8096
|
+
* const pending = await plugin.getRawEvents('wallets', 'balance', {
|
|
8097
|
+
* applied: false
|
|
8098
|
+
* });
|
|
8099
|
+
*/
|
|
8100
|
+
async getRawEvents(resourceName, field, options = {}) {
|
|
8101
|
+
return await getRawEvents(resourceName, field, options, this.fieldHandlers);
|
|
8102
|
+
}
|
|
8103
|
+
/**
|
|
8104
|
+
* Get diagnostics information about the plugin state
|
|
8105
|
+
*
|
|
8106
|
+
* This method provides comprehensive diagnostic information about the EventualConsistencyPlugin,
|
|
8107
|
+
* including configured resources, field handlers, timers, and overall health status.
|
|
8108
|
+
* Useful for debugging initialization issues, configuration problems, or runtime errors.
|
|
8109
|
+
*
|
|
8110
|
+
* @param {Object} options - Diagnostic options
|
|
8111
|
+
* @param {string} options.resourceName - Optional: limit diagnostics to specific resource
|
|
8112
|
+
* @param {string} options.field - Optional: limit diagnostics to specific field
|
|
8113
|
+
* @param {boolean} options.includeStats - Include transaction statistics (default: false)
|
|
8114
|
+
* @returns {Promise<Object>} Diagnostic information
|
|
8115
|
+
*
|
|
8116
|
+
* @example
|
|
8117
|
+
* // Get overall plugin diagnostics
|
|
8118
|
+
* const diagnostics = await plugin.getDiagnostics();
|
|
8119
|
+
* console.log(diagnostics);
|
|
8120
|
+
*
|
|
8121
|
+
* @example
|
|
8122
|
+
* // Get diagnostics for specific resource/field with stats
|
|
8123
|
+
* const diagnostics = await plugin.getDiagnostics({
|
|
8124
|
+
* resourceName: 'wallets',
|
|
8125
|
+
* field: 'balance',
|
|
8126
|
+
* includeStats: true
|
|
8127
|
+
* });
|
|
8128
|
+
*/
|
|
8129
|
+
async getDiagnostics(options = {}) {
|
|
8130
|
+
const { resourceName, field, includeStats = false } = options;
|
|
8131
|
+
const diagnostics = {
|
|
8132
|
+
plugin: {
|
|
8133
|
+
name: "EventualConsistencyPlugin",
|
|
8134
|
+
initialized: this.database !== null && this.database !== void 0,
|
|
8135
|
+
verbose: this.config.verbose || false,
|
|
8136
|
+
timezone: this.config.cohort?.timezone || "UTC",
|
|
8137
|
+
consolidation: {
|
|
8138
|
+
mode: this.config.consolidation?.mode || "timer",
|
|
8139
|
+
interval: this.config.consolidation?.interval || 6e4,
|
|
8140
|
+
batchSize: this.config.consolidation?.batchSize || 100
|
|
8141
|
+
},
|
|
8142
|
+
garbageCollection: {
|
|
8143
|
+
enabled: this.config.garbageCollection?.enabled !== false,
|
|
8144
|
+
retentionDays: this.config.garbageCollection?.retentionDays || 30,
|
|
8145
|
+
interval: this.config.garbageCollection?.interval || 36e5
|
|
8146
|
+
}
|
|
8147
|
+
},
|
|
8148
|
+
resources: [],
|
|
8149
|
+
errors: [],
|
|
8150
|
+
warnings: []
|
|
8151
|
+
};
|
|
8152
|
+
for (const [resName, resourceHandlers] of this.fieldHandlers.entries()) {
|
|
8153
|
+
if (resourceName && resName !== resourceName) {
|
|
8154
|
+
continue;
|
|
8155
|
+
}
|
|
8156
|
+
const resourceDiag = {
|
|
8157
|
+
name: resName,
|
|
8158
|
+
fields: []
|
|
8159
|
+
};
|
|
8160
|
+
for (const [fieldName, handler] of resourceHandlers.entries()) {
|
|
8161
|
+
if (field && fieldName !== field) {
|
|
8162
|
+
continue;
|
|
8163
|
+
}
|
|
8164
|
+
const fieldDiag = {
|
|
8165
|
+
name: fieldName,
|
|
8166
|
+
type: handler.type || "counter",
|
|
8167
|
+
analyticsEnabled: handler.analyticsResource !== null && handler.analyticsResource !== void 0,
|
|
8168
|
+
resources: {
|
|
8169
|
+
transaction: handler.transactionResource?.name || null,
|
|
8170
|
+
target: handler.targetResource?.name || null,
|
|
8171
|
+
analytics: handler.analyticsResource?.name || null
|
|
8172
|
+
},
|
|
8173
|
+
timers: {
|
|
8174
|
+
consolidation: handler.consolidationTimer !== null && handler.consolidationTimer !== void 0,
|
|
8175
|
+
garbageCollection: handler.garbageCollectionTimer !== null && handler.garbageCollectionTimer !== void 0
|
|
8176
|
+
}
|
|
8177
|
+
};
|
|
8178
|
+
if (!handler.transactionResource) {
|
|
8179
|
+
diagnostics.errors.push({
|
|
8180
|
+
resource: resName,
|
|
8181
|
+
field: fieldName,
|
|
8182
|
+
issue: "Missing transaction resource",
|
|
8183
|
+
suggestion: "Ensure plugin is installed and resources are created after plugin installation"
|
|
8184
|
+
});
|
|
8185
|
+
}
|
|
8186
|
+
if (!handler.targetResource) {
|
|
8187
|
+
diagnostics.warnings.push({
|
|
8188
|
+
resource: resName,
|
|
8189
|
+
field: fieldName,
|
|
8190
|
+
issue: "Missing target resource",
|
|
8191
|
+
suggestion: "Target resource may not have been created yet"
|
|
8192
|
+
});
|
|
8193
|
+
}
|
|
8194
|
+
if (handler.analyticsResource && !handler.analyticsResource.name) {
|
|
8195
|
+
diagnostics.errors.push({
|
|
8196
|
+
resource: resName,
|
|
8197
|
+
field: fieldName,
|
|
8198
|
+
issue: "Invalid analytics resource",
|
|
8199
|
+
suggestion: "Analytics resource exists but has no name - possible initialization failure"
|
|
8200
|
+
});
|
|
8201
|
+
}
|
|
8202
|
+
if (includeStats && handler.transactionResource) {
|
|
8203
|
+
try {
|
|
8204
|
+
const [okPending, errPending, pendingTxns] = await handler.transactionResource.query({ applied: false }).catch(() => [false, null, []]);
|
|
8205
|
+
const [okApplied, errApplied, appliedTxns] = await handler.transactionResource.query({ applied: true }).catch(() => [false, null, []]);
|
|
8206
|
+
fieldDiag.stats = {
|
|
8207
|
+
pendingTransactions: okPending ? pendingTxns?.length || 0 : "error",
|
|
8208
|
+
appliedTransactions: okApplied ? appliedTxns?.length || 0 : "error",
|
|
8209
|
+
totalTransactions: okPending && okApplied ? (pendingTxns?.length || 0) + (appliedTxns?.length || 0) : "error"
|
|
8210
|
+
};
|
|
8211
|
+
if (handler.analyticsResource) {
|
|
8212
|
+
const [okAnalytics, errAnalytics, analyticsRecords] = await handler.analyticsResource.list().catch(() => [false, null, []]);
|
|
8213
|
+
fieldDiag.stats.analyticsRecords = okAnalytics ? analyticsRecords?.length || 0 : "error";
|
|
8214
|
+
}
|
|
8215
|
+
} catch (error) {
|
|
8216
|
+
diagnostics.warnings.push({
|
|
8217
|
+
resource: resName,
|
|
8218
|
+
field: fieldName,
|
|
8219
|
+
issue: "Failed to fetch statistics",
|
|
8220
|
+
error: error.message
|
|
8221
|
+
});
|
|
8222
|
+
}
|
|
8223
|
+
}
|
|
8224
|
+
resourceDiag.fields.push(fieldDiag);
|
|
8225
|
+
}
|
|
8226
|
+
if (resourceDiag.fields.length > 0) {
|
|
8227
|
+
diagnostics.resources.push(resourceDiag);
|
|
8228
|
+
}
|
|
8229
|
+
}
|
|
8230
|
+
diagnostics.health = {
|
|
8231
|
+
status: diagnostics.errors.length === 0 ? diagnostics.warnings.length === 0 ? "healthy" : "warning" : "error",
|
|
8232
|
+
totalResources: diagnostics.resources.length,
|
|
8233
|
+
totalFields: diagnostics.resources.reduce((sum, r) => sum + r.fields.length, 0),
|
|
8234
|
+
errorCount: diagnostics.errors.length,
|
|
8235
|
+
warningCount: diagnostics.warnings.length
|
|
8236
|
+
};
|
|
8237
|
+
return diagnostics;
|
|
8238
|
+
}
|
|
7736
8239
|
}
|
|
7737
8240
|
|
|
7738
8241
|
class FullTextPlugin extends Plugin {
|
|
@@ -11354,6 +11857,7 @@ ${errorDetails}`,
|
|
|
11354
11857
|
idGenerator: customIdGenerator,
|
|
11355
11858
|
idSize = 22,
|
|
11356
11859
|
versioningEnabled = false,
|
|
11860
|
+
strictValidation = true,
|
|
11357
11861
|
events = {},
|
|
11358
11862
|
asyncEvents = true,
|
|
11359
11863
|
asyncPartitions = true,
|
|
@@ -11367,6 +11871,7 @@ ${errorDetails}`,
|
|
|
11367
11871
|
this.parallelism = parallelism;
|
|
11368
11872
|
this.passphrase = passphrase ?? "secret";
|
|
11369
11873
|
this.versioningEnabled = versioningEnabled;
|
|
11874
|
+
this.strictValidation = strictValidation;
|
|
11370
11875
|
this.setAsyncMode(asyncEvents);
|
|
11371
11876
|
this.idGenerator = this.configureIdGenerator(customIdGenerator, idSize);
|
|
11372
11877
|
if (typeof customIdGenerator === "number" && customIdGenerator > 0) {
|
|
@@ -11614,9 +12119,12 @@ ${errorDetails}`,
|
|
|
11614
12119
|
}
|
|
11615
12120
|
/**
|
|
11616
12121
|
* Validate that all partition fields exist in current resource attributes
|
|
11617
|
-
* @throws {Error} If partition fields don't exist in current schema
|
|
12122
|
+
* @throws {Error} If partition fields don't exist in current schema (only when strictValidation is true)
|
|
11618
12123
|
*/
|
|
11619
12124
|
validatePartitions() {
|
|
12125
|
+
if (!this.strictValidation) {
|
|
12126
|
+
return;
|
|
12127
|
+
}
|
|
11620
12128
|
if (!this.config.partitions) {
|
|
11621
12129
|
return;
|
|
11622
12130
|
}
|
|
@@ -13751,7 +14259,7 @@ class Database extends EventEmitter {
|
|
|
13751
14259
|
this.id = idGenerator(7);
|
|
13752
14260
|
this.version = "1";
|
|
13753
14261
|
this.s3dbVersion = (() => {
|
|
13754
|
-
const [ok, err, version] = tryFn(() => true ? "11.2.
|
|
14262
|
+
const [ok, err, version] = tryFn(() => true ? "11.2.3" : "latest");
|
|
13755
14263
|
return ok ? version : "latest";
|
|
13756
14264
|
})();
|
|
13757
14265
|
this.resources = {};
|
|
@@ -13766,6 +14274,7 @@ class Database extends EventEmitter {
|
|
|
13766
14274
|
this.passphrase = options.passphrase || "secret";
|
|
13767
14275
|
this.versioningEnabled = options.versioningEnabled || false;
|
|
13768
14276
|
this.persistHooks = options.persistHooks || false;
|
|
14277
|
+
this.strictValidation = options.strictValidation !== false;
|
|
13769
14278
|
this._initHooks();
|
|
13770
14279
|
let connectionString = options.connectionString;
|
|
13771
14280
|
if (!connectionString && (options.bucket || options.accessKeyId || options.secretAccessKey)) {
|
|
@@ -13887,6 +14396,7 @@ class Database extends EventEmitter {
|
|
|
13887
14396
|
asyncEvents: versionData.asyncEvents !== void 0 ? versionData.asyncEvents : true,
|
|
13888
14397
|
hooks: this.persistHooks ? this._deserializeHooks(versionData.hooks || {}) : versionData.hooks || {},
|
|
13889
14398
|
versioningEnabled: this.versioningEnabled,
|
|
14399
|
+
strictValidation: this.strictValidation,
|
|
13890
14400
|
map: versionData.map,
|
|
13891
14401
|
idGenerator: restoredIdGenerator,
|
|
13892
14402
|
idSize: restoredIdSize
|
|
@@ -14548,6 +15058,7 @@ class Database extends EventEmitter {
|
|
|
14548
15058
|
autoDecrypt: config.autoDecrypt !== void 0 ? config.autoDecrypt : true,
|
|
14549
15059
|
hooks: hooks || {},
|
|
14550
15060
|
versioningEnabled: this.versioningEnabled,
|
|
15061
|
+
strictValidation: this.strictValidation,
|
|
14551
15062
|
map: config.map,
|
|
14552
15063
|
idGenerator: config.idGenerator,
|
|
14553
15064
|
idSize: config.idSize,
|
|
@@ -17383,6 +17894,7 @@ class StateMachinePlugin extends Plugin {
|
|
|
17383
17894
|
}
|
|
17384
17895
|
|
|
17385
17896
|
exports.AVAILABLE_BEHAVIORS = AVAILABLE_BEHAVIORS;
|
|
17897
|
+
exports.AnalyticsNotEnabledError = AnalyticsNotEnabledError;
|
|
17386
17898
|
exports.AuditPlugin = AuditPlugin;
|
|
17387
17899
|
exports.AuthenticationError = AuthenticationError;
|
|
17388
17900
|
exports.BackupPlugin = BackupPlugin;
|