s3db.js 11.2.2 → 11.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/s3db.cjs.js +481 -16
- package/dist/s3db.cjs.js.map +1 -1
- package/dist/s3db.es.js +481 -17
- package/dist/s3db.es.js.map +1 -1
- package/package.json +1 -1
- package/src/database.class.js +3 -0
- package/src/errors.js +115 -4
- package/src/plugins/cache/memory-cache.class.js +216 -33
- package/src/plugins/cache.plugin.js +85 -1
- package/src/plugins/eventual-consistency/analytics.js +145 -0
- package/src/plugins/eventual-consistency/index.js +203 -1
- package/src/resource.class.js +8 -1
package/dist/s3db.es.js
CHANGED
|
@@ -11,6 +11,7 @@ import jsonStableStringify from 'json-stable-stringify';
|
|
|
11
11
|
import { Transform, Writable } from 'stream';
|
|
12
12
|
import { PromisePool } from '@supercharge/promise-pool';
|
|
13
13
|
import { ReadableStream } from 'node:stream/web';
|
|
14
|
+
import os$1 from 'node:os';
|
|
14
15
|
import { chunk, merge, isString, isEmpty, invert, uniq, cloneDeep, get, set, isObject, isFunction } from 'lodash-es';
|
|
15
16
|
import { Agent } from 'http';
|
|
16
17
|
import { Agent as Agent$1 } from 'https';
|
|
@@ -217,7 +218,7 @@ function calculateEffectiveLimit(config = {}) {
|
|
|
217
218
|
}
|
|
218
219
|
|
|
219
220
|
class BaseError extends Error {
|
|
220
|
-
constructor({ verbose, bucket, key, message, code, statusCode, requestId, awsMessage, original, commandName, commandInput, metadata, suggestion, ...rest }) {
|
|
221
|
+
constructor({ verbose, bucket, key, message, code, statusCode, requestId, awsMessage, original, commandName, commandInput, metadata, suggestion, description, ...rest }) {
|
|
221
222
|
if (verbose) message = message + `
|
|
222
223
|
|
|
223
224
|
Verbose:
|
|
@@ -243,6 +244,7 @@ ${JSON.stringify(rest, null, 2)}`;
|
|
|
243
244
|
this.commandInput = commandInput;
|
|
244
245
|
this.metadata = metadata;
|
|
245
246
|
this.suggestion = suggestion;
|
|
247
|
+
this.description = description;
|
|
246
248
|
this.data = { bucket, key, ...rest, verbose, message };
|
|
247
249
|
}
|
|
248
250
|
toJson() {
|
|
@@ -260,6 +262,7 @@ ${JSON.stringify(rest, null, 2)}`;
|
|
|
260
262
|
commandInput: this.commandInput,
|
|
261
263
|
metadata: this.metadata,
|
|
262
264
|
suggestion: this.suggestion,
|
|
265
|
+
description: this.description,
|
|
263
266
|
data: this.data,
|
|
264
267
|
original: this.original,
|
|
265
268
|
stack: this.stack
|
|
@@ -452,7 +455,107 @@ class ResourceError extends S3dbError {
|
|
|
452
455
|
}
|
|
453
456
|
class PartitionError extends S3dbError {
|
|
454
457
|
constructor(message, details = {}) {
|
|
455
|
-
|
|
458
|
+
let description = details.description;
|
|
459
|
+
if (!description && details.resourceName && details.partitionName && details.fieldName) {
|
|
460
|
+
const { resourceName, partitionName, fieldName, availableFields = [] } = details;
|
|
461
|
+
description = `
|
|
462
|
+
Partition Field Validation Error
|
|
463
|
+
|
|
464
|
+
Resource: ${resourceName}
|
|
465
|
+
Partition: ${partitionName}
|
|
466
|
+
Missing Field: ${fieldName}
|
|
467
|
+
|
|
468
|
+
Available fields in schema:
|
|
469
|
+
${availableFields.map((f) => ` \u2022 ${f}`).join("\n") || " (no fields defined)"}
|
|
470
|
+
|
|
471
|
+
Possible causes:
|
|
472
|
+
1. Field was removed from schema but partition still references it
|
|
473
|
+
2. Typo in partition field name
|
|
474
|
+
3. Nested field path is incorrect (use dot notation like 'utm.source')
|
|
475
|
+
|
|
476
|
+
Solution:
|
|
477
|
+
${details.strictValidation === false ? " \u2022 Update partition definition to use existing fields" : ` \u2022 Add missing field to schema, OR
|
|
478
|
+
\u2022 Update partition definition to use existing fields, OR
|
|
479
|
+
\u2022 Use strictValidation: false to skip this check during testing`}
|
|
480
|
+
|
|
481
|
+
Docs: https://docs.s3db.js.org/resources/partitions#validation
|
|
482
|
+
`.trim();
|
|
483
|
+
}
|
|
484
|
+
super(message, {
|
|
485
|
+
...details,
|
|
486
|
+
description,
|
|
487
|
+
suggestion: details.suggestion || "Check partition definition, fields, and input values."
|
|
488
|
+
});
|
|
489
|
+
}
|
|
490
|
+
}
|
|
491
|
+
class AnalyticsNotEnabledError extends S3dbError {
|
|
492
|
+
constructor(details = {}) {
|
|
493
|
+
const {
|
|
494
|
+
pluginName = "EventualConsistency",
|
|
495
|
+
resourceName = "unknown",
|
|
496
|
+
field = "unknown",
|
|
497
|
+
configuredResources = [],
|
|
498
|
+
registeredResources = [],
|
|
499
|
+
pluginInitialized = false,
|
|
500
|
+
...rest
|
|
501
|
+
} = details;
|
|
502
|
+
const message = `Analytics not enabled for ${resourceName}.${field}`;
|
|
503
|
+
const description = `
|
|
504
|
+
Analytics Not Enabled
|
|
505
|
+
|
|
506
|
+
Plugin: ${pluginName}
|
|
507
|
+
Resource: ${resourceName}
|
|
508
|
+
Field: ${field}
|
|
509
|
+
|
|
510
|
+
Diagnostics:
|
|
511
|
+
\u2022 Plugin initialized: ${pluginInitialized ? "\u2713 Yes" : "\u2717 No"}
|
|
512
|
+
\u2022 Analytics resources created: ${registeredResources.length}/${configuredResources.length}
|
|
513
|
+
${configuredResources.map((r) => {
|
|
514
|
+
const exists = registeredResources.includes(r);
|
|
515
|
+
return ` ${exists ? "\u2713" : "\u2717"} ${r}${!exists ? " (missing)" : ""}`;
|
|
516
|
+
}).join("\n")}
|
|
517
|
+
|
|
518
|
+
Possible causes:
|
|
519
|
+
1. Resource not created yet - Analytics resources are created when db.createResource() is called
|
|
520
|
+
2. Resource created before plugin initialization - Plugin must be initialized before resources
|
|
521
|
+
3. Field not configured in analytics.resources config
|
|
522
|
+
|
|
523
|
+
Correct initialization order:
|
|
524
|
+
1. Create database: const db = new Database({ ... })
|
|
525
|
+
2. Install plugins: await db.connect() (triggers plugin.install())
|
|
526
|
+
3. Create resources: await db.createResource({ name: '${resourceName}', ... })
|
|
527
|
+
4. Analytics resources are auto-created by plugin
|
|
528
|
+
|
|
529
|
+
Example fix:
|
|
530
|
+
const db = new Database({
|
|
531
|
+
bucket: 'my-bucket',
|
|
532
|
+
plugins: [new EventualConsistencyPlugin({
|
|
533
|
+
resources: {
|
|
534
|
+
'${resourceName}': {
|
|
535
|
+
fields: {
|
|
536
|
+
'${field}': { type: 'counter', analytics: true }
|
|
537
|
+
}
|
|
538
|
+
}
|
|
539
|
+
}
|
|
540
|
+
})]
|
|
541
|
+
});
|
|
542
|
+
|
|
543
|
+
await db.connect(); // Plugin initialized here
|
|
544
|
+
await db.createResource({ name: '${resourceName}', ... }); // Analytics resource created here
|
|
545
|
+
|
|
546
|
+
Docs: https://docs.s3db.js.org/plugins/eventual-consistency#troubleshooting
|
|
547
|
+
`.trim();
|
|
548
|
+
super(message, {
|
|
549
|
+
...rest,
|
|
550
|
+
pluginName,
|
|
551
|
+
resourceName,
|
|
552
|
+
field,
|
|
553
|
+
configuredResources,
|
|
554
|
+
registeredResources,
|
|
555
|
+
pluginInitialized,
|
|
556
|
+
description,
|
|
557
|
+
suggestion: "Ensure resources are created after plugin initialization. Check plugin configuration and resource creation order."
|
|
558
|
+
});
|
|
456
559
|
}
|
|
457
560
|
}
|
|
458
561
|
|
|
@@ -3601,6 +3704,24 @@ class MemoryCache extends Cache {
|
|
|
3601
3704
|
this.cache = {};
|
|
3602
3705
|
this.meta = {};
|
|
3603
3706
|
this.maxSize = config.maxSize !== void 0 ? config.maxSize : 1e3;
|
|
3707
|
+
if (config.maxMemoryBytes && config.maxMemoryBytes > 0 && config.maxMemoryPercent && config.maxMemoryPercent > 0) {
|
|
3708
|
+
throw new Error(
|
|
3709
|
+
"[MemoryCache] Cannot use both maxMemoryBytes and maxMemoryPercent. Choose one: maxMemoryBytes (absolute) or maxMemoryPercent (0...1 fraction)."
|
|
3710
|
+
);
|
|
3711
|
+
}
|
|
3712
|
+
if (config.maxMemoryPercent && config.maxMemoryPercent > 0) {
|
|
3713
|
+
if (config.maxMemoryPercent > 1) {
|
|
3714
|
+
throw new Error(
|
|
3715
|
+
`[MemoryCache] maxMemoryPercent must be between 0 and 1 (e.g., 0.1 for 10%). Received: ${config.maxMemoryPercent}`
|
|
3716
|
+
);
|
|
3717
|
+
}
|
|
3718
|
+
const totalMemory = os$1.totalmem();
|
|
3719
|
+
this.maxMemoryBytes = Math.floor(totalMemory * config.maxMemoryPercent);
|
|
3720
|
+
this.maxMemoryPercent = config.maxMemoryPercent;
|
|
3721
|
+
} else {
|
|
3722
|
+
this.maxMemoryBytes = config.maxMemoryBytes !== void 0 ? config.maxMemoryBytes : 0;
|
|
3723
|
+
this.maxMemoryPercent = 0;
|
|
3724
|
+
}
|
|
3604
3725
|
this.ttl = config.ttl !== void 0 ? config.ttl : 3e5;
|
|
3605
3726
|
this.enableCompression = config.enableCompression !== void 0 ? config.enableCompression : false;
|
|
3606
3727
|
this.compressionThreshold = config.compressionThreshold !== void 0 ? config.compressionThreshold : 1024;
|
|
@@ -3610,23 +3731,18 @@ class MemoryCache extends Cache {
|
|
|
3610
3731
|
totalCompressedSize: 0,
|
|
3611
3732
|
compressionRatio: 0
|
|
3612
3733
|
};
|
|
3734
|
+
this.currentMemoryBytes = 0;
|
|
3735
|
+
this.evictedDueToMemory = 0;
|
|
3613
3736
|
}
|
|
3614
3737
|
async _set(key, data) {
|
|
3615
|
-
if (this.maxSize > 0 && Object.keys(this.cache).length >= this.maxSize) {
|
|
3616
|
-
const oldestKey = Object.entries(this.meta).sort((a, b) => a[1].ts - b[1].ts)[0]?.[0];
|
|
3617
|
-
if (oldestKey) {
|
|
3618
|
-
delete this.cache[oldestKey];
|
|
3619
|
-
delete this.meta[oldestKey];
|
|
3620
|
-
}
|
|
3621
|
-
}
|
|
3622
3738
|
let finalData = data;
|
|
3623
3739
|
let compressed = false;
|
|
3624
3740
|
let originalSize = 0;
|
|
3625
3741
|
let compressedSize = 0;
|
|
3742
|
+
const serialized = JSON.stringify(data);
|
|
3743
|
+
originalSize = Buffer.byteLength(serialized, "utf8");
|
|
3626
3744
|
if (this.enableCompression) {
|
|
3627
3745
|
try {
|
|
3628
|
-
const serialized = JSON.stringify(data);
|
|
3629
|
-
originalSize = Buffer.byteLength(serialized, "utf8");
|
|
3630
3746
|
if (originalSize >= this.compressionThreshold) {
|
|
3631
3747
|
const compressedBuffer = zlib.gzipSync(Buffer.from(serialized, "utf8"));
|
|
3632
3748
|
finalData = {
|
|
@@ -3645,13 +3761,42 @@ class MemoryCache extends Cache {
|
|
|
3645
3761
|
console.warn(`[MemoryCache] Compression failed for key '${key}':`, error.message);
|
|
3646
3762
|
}
|
|
3647
3763
|
}
|
|
3764
|
+
const itemSize = compressed ? compressedSize : originalSize;
|
|
3765
|
+
if (Object.prototype.hasOwnProperty.call(this.cache, key)) {
|
|
3766
|
+
const oldSize = this.meta[key]?.compressedSize || 0;
|
|
3767
|
+
this.currentMemoryBytes -= oldSize;
|
|
3768
|
+
}
|
|
3769
|
+
if (this.maxMemoryBytes > 0) {
|
|
3770
|
+
while (this.currentMemoryBytes + itemSize > this.maxMemoryBytes && Object.keys(this.cache).length > 0) {
|
|
3771
|
+
const oldestKey = Object.entries(this.meta).sort((a, b) => a[1].ts - b[1].ts)[0]?.[0];
|
|
3772
|
+
if (oldestKey) {
|
|
3773
|
+
const evictedSize = this.meta[oldestKey]?.compressedSize || 0;
|
|
3774
|
+
delete this.cache[oldestKey];
|
|
3775
|
+
delete this.meta[oldestKey];
|
|
3776
|
+
this.currentMemoryBytes -= evictedSize;
|
|
3777
|
+
this.evictedDueToMemory++;
|
|
3778
|
+
} else {
|
|
3779
|
+
break;
|
|
3780
|
+
}
|
|
3781
|
+
}
|
|
3782
|
+
}
|
|
3783
|
+
if (this.maxSize > 0 && Object.keys(this.cache).length >= this.maxSize) {
|
|
3784
|
+
const oldestKey = Object.entries(this.meta).sort((a, b) => a[1].ts - b[1].ts)[0]?.[0];
|
|
3785
|
+
if (oldestKey) {
|
|
3786
|
+
const evictedSize = this.meta[oldestKey]?.compressedSize || 0;
|
|
3787
|
+
delete this.cache[oldestKey];
|
|
3788
|
+
delete this.meta[oldestKey];
|
|
3789
|
+
this.currentMemoryBytes -= evictedSize;
|
|
3790
|
+
}
|
|
3791
|
+
}
|
|
3648
3792
|
this.cache[key] = finalData;
|
|
3649
3793
|
this.meta[key] = {
|
|
3650
3794
|
ts: Date.now(),
|
|
3651
3795
|
compressed,
|
|
3652
3796
|
originalSize,
|
|
3653
|
-
compressedSize:
|
|
3797
|
+
compressedSize: itemSize
|
|
3654
3798
|
};
|
|
3799
|
+
this.currentMemoryBytes += itemSize;
|
|
3655
3800
|
return data;
|
|
3656
3801
|
}
|
|
3657
3802
|
async _get(key) {
|
|
@@ -3659,7 +3804,9 @@ class MemoryCache extends Cache {
|
|
|
3659
3804
|
if (this.ttl > 0) {
|
|
3660
3805
|
const now = Date.now();
|
|
3661
3806
|
const meta = this.meta[key];
|
|
3662
|
-
if (meta && now - meta.ts > this.ttl
|
|
3807
|
+
if (meta && now - meta.ts > this.ttl) {
|
|
3808
|
+
const itemSize = meta.compressedSize || 0;
|
|
3809
|
+
this.currentMemoryBytes -= itemSize;
|
|
3663
3810
|
delete this.cache[key];
|
|
3664
3811
|
delete this.meta[key];
|
|
3665
3812
|
return null;
|
|
@@ -3681,6 +3828,10 @@ class MemoryCache extends Cache {
|
|
|
3681
3828
|
return rawData;
|
|
3682
3829
|
}
|
|
3683
3830
|
async _del(key) {
|
|
3831
|
+
if (Object.prototype.hasOwnProperty.call(this.cache, key)) {
|
|
3832
|
+
const itemSize = this.meta[key]?.compressedSize || 0;
|
|
3833
|
+
this.currentMemoryBytes -= itemSize;
|
|
3834
|
+
}
|
|
3684
3835
|
delete this.cache[key];
|
|
3685
3836
|
delete this.meta[key];
|
|
3686
3837
|
return true;
|
|
@@ -3689,10 +3840,13 @@ class MemoryCache extends Cache {
|
|
|
3689
3840
|
if (!prefix) {
|
|
3690
3841
|
this.cache = {};
|
|
3691
3842
|
this.meta = {};
|
|
3843
|
+
this.currentMemoryBytes = 0;
|
|
3692
3844
|
return true;
|
|
3693
3845
|
}
|
|
3694
3846
|
for (const key of Object.keys(this.cache)) {
|
|
3695
3847
|
if (key.startsWith(prefix)) {
|
|
3848
|
+
const itemSize = this.meta[key]?.compressedSize || 0;
|
|
3849
|
+
this.currentMemoryBytes -= itemSize;
|
|
3696
3850
|
delete this.cache[key];
|
|
3697
3851
|
delete this.meta[key];
|
|
3698
3852
|
}
|
|
@@ -3730,6 +3884,53 @@ class MemoryCache extends Cache {
|
|
|
3730
3884
|
}
|
|
3731
3885
|
};
|
|
3732
3886
|
}
|
|
3887
|
+
/**
|
|
3888
|
+
* Get memory usage statistics
|
|
3889
|
+
* @returns {Object} Memory stats including current usage, limits, and eviction counts
|
|
3890
|
+
*/
|
|
3891
|
+
getMemoryStats() {
|
|
3892
|
+
const totalItems = Object.keys(this.cache).length;
|
|
3893
|
+
const memoryUsagePercent = this.maxMemoryBytes > 0 ? (this.currentMemoryBytes / this.maxMemoryBytes * 100).toFixed(2) : 0;
|
|
3894
|
+
const systemMemory = {
|
|
3895
|
+
total: os$1.totalmem(),
|
|
3896
|
+
free: os$1.freemem(),
|
|
3897
|
+
used: os$1.totalmem() - os$1.freemem()
|
|
3898
|
+
};
|
|
3899
|
+
const cachePercentOfTotal = systemMemory.total > 0 ? (this.currentMemoryBytes / systemMemory.total * 100).toFixed(2) : 0;
|
|
3900
|
+
return {
|
|
3901
|
+
currentMemoryBytes: this.currentMemoryBytes,
|
|
3902
|
+
maxMemoryBytes: this.maxMemoryBytes,
|
|
3903
|
+
maxMemoryPercent: this.maxMemoryPercent,
|
|
3904
|
+
memoryUsagePercent: parseFloat(memoryUsagePercent),
|
|
3905
|
+
cachePercentOfSystemMemory: parseFloat(cachePercentOfTotal),
|
|
3906
|
+
totalItems,
|
|
3907
|
+
maxSize: this.maxSize,
|
|
3908
|
+
evictedDueToMemory: this.evictedDueToMemory,
|
|
3909
|
+
averageItemSize: totalItems > 0 ? Math.round(this.currentMemoryBytes / totalItems) : 0,
|
|
3910
|
+
memoryUsage: {
|
|
3911
|
+
current: this._formatBytes(this.currentMemoryBytes),
|
|
3912
|
+
max: this.maxMemoryBytes > 0 ? this._formatBytes(this.maxMemoryBytes) : "unlimited",
|
|
3913
|
+
available: this.maxMemoryBytes > 0 ? this._formatBytes(this.maxMemoryBytes - this.currentMemoryBytes) : "unlimited"
|
|
3914
|
+
},
|
|
3915
|
+
systemMemory: {
|
|
3916
|
+
total: this._formatBytes(systemMemory.total),
|
|
3917
|
+
free: this._formatBytes(systemMemory.free),
|
|
3918
|
+
used: this._formatBytes(systemMemory.used),
|
|
3919
|
+
cachePercent: `${cachePercentOfTotal}%`
|
|
3920
|
+
}
|
|
3921
|
+
};
|
|
3922
|
+
}
|
|
3923
|
+
/**
|
|
3924
|
+
* Format bytes to human-readable format
|
|
3925
|
+
* @private
|
|
3926
|
+
*/
|
|
3927
|
+
_formatBytes(bytes) {
|
|
3928
|
+
if (bytes === 0) return "0 B";
|
|
3929
|
+
const k = 1024;
|
|
3930
|
+
const sizes = ["B", "KB", "MB", "GB"];
|
|
3931
|
+
const i = Math.floor(Math.log(bytes) / Math.log(k));
|
|
3932
|
+
return `${(bytes / Math.pow(k, i)).toFixed(2)} ${sizes[i]}`;
|
|
3933
|
+
}
|
|
3733
3934
|
}
|
|
3734
3935
|
|
|
3735
3936
|
class FilesystemCache extends Cache {
|
|
@@ -4559,8 +4760,10 @@ class CachePlugin extends Plugin {
|
|
|
4559
4760
|
config: {
|
|
4560
4761
|
ttl: options.ttl,
|
|
4561
4762
|
maxSize: options.maxSize,
|
|
4763
|
+
maxMemoryBytes: options.maxMemoryBytes,
|
|
4764
|
+
maxMemoryPercent: options.maxMemoryPercent,
|
|
4562
4765
|
...options.config
|
|
4563
|
-
// Driver-specific config (can override ttl/maxSize)
|
|
4766
|
+
// Driver-specific config (can override ttl/maxSize/maxMemoryBytes/maxMemoryPercent)
|
|
4564
4767
|
},
|
|
4565
4768
|
// Resource filtering
|
|
4566
4769
|
include: options.include || null,
|
|
@@ -7019,6 +7222,80 @@ async function getLastNMonths(resourceName, field, months = 12, options, fieldHa
|
|
|
7019
7222
|
}
|
|
7020
7223
|
return data;
|
|
7021
7224
|
}
|
|
7225
|
+
async function getRawEvents(resourceName, field, options, fieldHandlers) {
|
|
7226
|
+
const resourceHandlers = fieldHandlers.get(resourceName);
|
|
7227
|
+
if (!resourceHandlers) {
|
|
7228
|
+
throw new Error(`No eventual consistency configured for resource: ${resourceName}`);
|
|
7229
|
+
}
|
|
7230
|
+
const handler = resourceHandlers.get(field);
|
|
7231
|
+
if (!handler) {
|
|
7232
|
+
throw new Error(`No eventual consistency configured for field: ${resourceName}.${field}`);
|
|
7233
|
+
}
|
|
7234
|
+
if (!handler.transactionResource) {
|
|
7235
|
+
throw new Error("Transaction resource not initialized");
|
|
7236
|
+
}
|
|
7237
|
+
const {
|
|
7238
|
+
recordId,
|
|
7239
|
+
startDate,
|
|
7240
|
+
endDate,
|
|
7241
|
+
cohortDate,
|
|
7242
|
+
cohortHour,
|
|
7243
|
+
cohortMonth,
|
|
7244
|
+
applied,
|
|
7245
|
+
operation,
|
|
7246
|
+
limit
|
|
7247
|
+
} = options;
|
|
7248
|
+
const query = {};
|
|
7249
|
+
if (recordId !== void 0) {
|
|
7250
|
+
query.originalId = recordId;
|
|
7251
|
+
}
|
|
7252
|
+
if (applied !== void 0) {
|
|
7253
|
+
query.applied = applied;
|
|
7254
|
+
}
|
|
7255
|
+
const [ok, err, allTransactions] = await tryFn(
|
|
7256
|
+
() => handler.transactionResource.query(query)
|
|
7257
|
+
);
|
|
7258
|
+
if (!ok || !allTransactions) {
|
|
7259
|
+
return [];
|
|
7260
|
+
}
|
|
7261
|
+
let filtered = allTransactions;
|
|
7262
|
+
if (operation !== void 0) {
|
|
7263
|
+
filtered = filtered.filter((t) => t.operation === operation);
|
|
7264
|
+
}
|
|
7265
|
+
if (cohortDate) {
|
|
7266
|
+
filtered = filtered.filter((t) => t.cohortDate === cohortDate);
|
|
7267
|
+
}
|
|
7268
|
+
if (cohortHour) {
|
|
7269
|
+
filtered = filtered.filter((t) => t.cohortHour === cohortHour);
|
|
7270
|
+
}
|
|
7271
|
+
if (cohortMonth) {
|
|
7272
|
+
filtered = filtered.filter((t) => t.cohortMonth === cohortMonth);
|
|
7273
|
+
}
|
|
7274
|
+
if (startDate && endDate) {
|
|
7275
|
+
const isHourly = startDate.length > 10;
|
|
7276
|
+
const cohortField = isHourly ? "cohortHour" : "cohortDate";
|
|
7277
|
+
filtered = filtered.filter(
|
|
7278
|
+
(t) => t[cohortField] && t[cohortField] >= startDate && t[cohortField] <= endDate
|
|
7279
|
+
);
|
|
7280
|
+
} else if (startDate) {
|
|
7281
|
+
const isHourly = startDate.length > 10;
|
|
7282
|
+
const cohortField = isHourly ? "cohortHour" : "cohortDate";
|
|
7283
|
+
filtered = filtered.filter((t) => t[cohortField] && t[cohortField] >= startDate);
|
|
7284
|
+
} else if (endDate) {
|
|
7285
|
+
const isHourly = endDate.length > 10;
|
|
7286
|
+
const cohortField = isHourly ? "cohortHour" : "cohortDate";
|
|
7287
|
+
filtered = filtered.filter((t) => t[cohortField] && t[cohortField] <= endDate);
|
|
7288
|
+
}
|
|
7289
|
+
filtered.sort((a, b) => {
|
|
7290
|
+
const aTime = new Date(a.timestamp || a.createdAt).getTime();
|
|
7291
|
+
const bTime = new Date(b.timestamp || b.createdAt).getTime();
|
|
7292
|
+
return bTime - aTime;
|
|
7293
|
+
});
|
|
7294
|
+
if (limit && limit > 0) {
|
|
7295
|
+
filtered = filtered.slice(0, limit);
|
|
7296
|
+
}
|
|
7297
|
+
return filtered;
|
|
7298
|
+
}
|
|
7022
7299
|
|
|
7023
7300
|
function addHelperMethods(resource, plugin, config) {
|
|
7024
7301
|
resource.set = async (id, field, value) => {
|
|
@@ -7776,6 +8053,185 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
7776
8053
|
async getLastNMonths(resourceName, field, months = 12, options = {}) {
|
|
7777
8054
|
return await getLastNMonths(resourceName, field, months, options, this.fieldHandlers);
|
|
7778
8055
|
}
|
|
8056
|
+
/**
|
|
8057
|
+
* Get raw transaction events for custom aggregation
|
|
8058
|
+
*
|
|
8059
|
+
* This method provides direct access to the underlying transaction events,
|
|
8060
|
+
* allowing developers to perform custom aggregations beyond the pre-built analytics.
|
|
8061
|
+
* Useful for complex queries, custom metrics, or when you need the raw event data.
|
|
8062
|
+
*
|
|
8063
|
+
* @param {string} resourceName - Resource name
|
|
8064
|
+
* @param {string} field - Field name
|
|
8065
|
+
* @param {Object} options - Query options
|
|
8066
|
+
* @param {string} options.recordId - Filter by specific record ID
|
|
8067
|
+
* @param {string} options.startDate - Start date filter (YYYY-MM-DD or YYYY-MM-DDTHH)
|
|
8068
|
+
* @param {string} options.endDate - End date filter (YYYY-MM-DD or YYYY-MM-DDTHH)
|
|
8069
|
+
* @param {string} options.cohortDate - Filter by cohort date (YYYY-MM-DD)
|
|
8070
|
+
* @param {string} options.cohortHour - Filter by cohort hour (YYYY-MM-DDTHH)
|
|
8071
|
+
* @param {string} options.cohortMonth - Filter by cohort month (YYYY-MM)
|
|
8072
|
+
* @param {boolean} options.applied - Filter by applied status (true/false/undefined for both)
|
|
8073
|
+
* @param {string} options.operation - Filter by operation type ('add', 'sub', 'set')
|
|
8074
|
+
* @param {number} options.limit - Maximum number of events to return
|
|
8075
|
+
* @returns {Promise<Array>} Raw transaction events
|
|
8076
|
+
*
|
|
8077
|
+
* @example
|
|
8078
|
+
* // Get all events for a specific record
|
|
8079
|
+
* const events = await plugin.getRawEvents('wallets', 'balance', {
|
|
8080
|
+
* recordId: 'wallet1'
|
|
8081
|
+
* });
|
|
8082
|
+
*
|
|
8083
|
+
* @example
|
|
8084
|
+
* // Get events for a specific time range
|
|
8085
|
+
* const events = await plugin.getRawEvents('wallets', 'balance', {
|
|
8086
|
+
* startDate: '2025-10-01',
|
|
8087
|
+
* endDate: '2025-10-31'
|
|
8088
|
+
* });
|
|
8089
|
+
*
|
|
8090
|
+
* @example
|
|
8091
|
+
* // Get only pending (unapplied) transactions
|
|
8092
|
+
* const pending = await plugin.getRawEvents('wallets', 'balance', {
|
|
8093
|
+
* applied: false
|
|
8094
|
+
* });
|
|
8095
|
+
*/
|
|
8096
|
+
async getRawEvents(resourceName, field, options = {}) {
|
|
8097
|
+
return await getRawEvents(resourceName, field, options, this.fieldHandlers);
|
|
8098
|
+
}
|
|
8099
|
+
/**
|
|
8100
|
+
* Get diagnostics information about the plugin state
|
|
8101
|
+
*
|
|
8102
|
+
* This method provides comprehensive diagnostic information about the EventualConsistencyPlugin,
|
|
8103
|
+
* including configured resources, field handlers, timers, and overall health status.
|
|
8104
|
+
* Useful for debugging initialization issues, configuration problems, or runtime errors.
|
|
8105
|
+
*
|
|
8106
|
+
* @param {Object} options - Diagnostic options
|
|
8107
|
+
* @param {string} options.resourceName - Optional: limit diagnostics to specific resource
|
|
8108
|
+
* @param {string} options.field - Optional: limit diagnostics to specific field
|
|
8109
|
+
* @param {boolean} options.includeStats - Include transaction statistics (default: false)
|
|
8110
|
+
* @returns {Promise<Object>} Diagnostic information
|
|
8111
|
+
*
|
|
8112
|
+
* @example
|
|
8113
|
+
* // Get overall plugin diagnostics
|
|
8114
|
+
* const diagnostics = await plugin.getDiagnostics();
|
|
8115
|
+
* console.log(diagnostics);
|
|
8116
|
+
*
|
|
8117
|
+
* @example
|
|
8118
|
+
* // Get diagnostics for specific resource/field with stats
|
|
8119
|
+
* const diagnostics = await plugin.getDiagnostics({
|
|
8120
|
+
* resourceName: 'wallets',
|
|
8121
|
+
* field: 'balance',
|
|
8122
|
+
* includeStats: true
|
|
8123
|
+
* });
|
|
8124
|
+
*/
|
|
8125
|
+
async getDiagnostics(options = {}) {
|
|
8126
|
+
const { resourceName, field, includeStats = false } = options;
|
|
8127
|
+
const diagnostics = {
|
|
8128
|
+
plugin: {
|
|
8129
|
+
name: "EventualConsistencyPlugin",
|
|
8130
|
+
initialized: this.database !== null && this.database !== void 0,
|
|
8131
|
+
verbose: this.config.verbose || false,
|
|
8132
|
+
timezone: this.config.cohort?.timezone || "UTC",
|
|
8133
|
+
consolidation: {
|
|
8134
|
+
mode: this.config.consolidation?.mode || "timer",
|
|
8135
|
+
interval: this.config.consolidation?.interval || 6e4,
|
|
8136
|
+
batchSize: this.config.consolidation?.batchSize || 100
|
|
8137
|
+
},
|
|
8138
|
+
garbageCollection: {
|
|
8139
|
+
enabled: this.config.garbageCollection?.enabled !== false,
|
|
8140
|
+
retentionDays: this.config.garbageCollection?.retentionDays || 30,
|
|
8141
|
+
interval: this.config.garbageCollection?.interval || 36e5
|
|
8142
|
+
}
|
|
8143
|
+
},
|
|
8144
|
+
resources: [],
|
|
8145
|
+
errors: [],
|
|
8146
|
+
warnings: []
|
|
8147
|
+
};
|
|
8148
|
+
for (const [resName, resourceHandlers] of this.fieldHandlers.entries()) {
|
|
8149
|
+
if (resourceName && resName !== resourceName) {
|
|
8150
|
+
continue;
|
|
8151
|
+
}
|
|
8152
|
+
const resourceDiag = {
|
|
8153
|
+
name: resName,
|
|
8154
|
+
fields: []
|
|
8155
|
+
};
|
|
8156
|
+
for (const [fieldName, handler] of resourceHandlers.entries()) {
|
|
8157
|
+
if (field && fieldName !== field) {
|
|
8158
|
+
continue;
|
|
8159
|
+
}
|
|
8160
|
+
const fieldDiag = {
|
|
8161
|
+
name: fieldName,
|
|
8162
|
+
type: handler.type || "counter",
|
|
8163
|
+
analyticsEnabled: handler.analyticsResource !== null && handler.analyticsResource !== void 0,
|
|
8164
|
+
resources: {
|
|
8165
|
+
transaction: handler.transactionResource?.name || null,
|
|
8166
|
+
target: handler.targetResource?.name || null,
|
|
8167
|
+
analytics: handler.analyticsResource?.name || null
|
|
8168
|
+
},
|
|
8169
|
+
timers: {
|
|
8170
|
+
consolidation: handler.consolidationTimer !== null && handler.consolidationTimer !== void 0,
|
|
8171
|
+
garbageCollection: handler.garbageCollectionTimer !== null && handler.garbageCollectionTimer !== void 0
|
|
8172
|
+
}
|
|
8173
|
+
};
|
|
8174
|
+
if (!handler.transactionResource) {
|
|
8175
|
+
diagnostics.errors.push({
|
|
8176
|
+
resource: resName,
|
|
8177
|
+
field: fieldName,
|
|
8178
|
+
issue: "Missing transaction resource",
|
|
8179
|
+
suggestion: "Ensure plugin is installed and resources are created after plugin installation"
|
|
8180
|
+
});
|
|
8181
|
+
}
|
|
8182
|
+
if (!handler.targetResource) {
|
|
8183
|
+
diagnostics.warnings.push({
|
|
8184
|
+
resource: resName,
|
|
8185
|
+
field: fieldName,
|
|
8186
|
+
issue: "Missing target resource",
|
|
8187
|
+
suggestion: "Target resource may not have been created yet"
|
|
8188
|
+
});
|
|
8189
|
+
}
|
|
8190
|
+
if (handler.analyticsResource && !handler.analyticsResource.name) {
|
|
8191
|
+
diagnostics.errors.push({
|
|
8192
|
+
resource: resName,
|
|
8193
|
+
field: fieldName,
|
|
8194
|
+
issue: "Invalid analytics resource",
|
|
8195
|
+
suggestion: "Analytics resource exists but has no name - possible initialization failure"
|
|
8196
|
+
});
|
|
8197
|
+
}
|
|
8198
|
+
if (includeStats && handler.transactionResource) {
|
|
8199
|
+
try {
|
|
8200
|
+
const [okPending, errPending, pendingTxns] = await handler.transactionResource.query({ applied: false }).catch(() => [false, null, []]);
|
|
8201
|
+
const [okApplied, errApplied, appliedTxns] = await handler.transactionResource.query({ applied: true }).catch(() => [false, null, []]);
|
|
8202
|
+
fieldDiag.stats = {
|
|
8203
|
+
pendingTransactions: okPending ? pendingTxns?.length || 0 : "error",
|
|
8204
|
+
appliedTransactions: okApplied ? appliedTxns?.length || 0 : "error",
|
|
8205
|
+
totalTransactions: okPending && okApplied ? (pendingTxns?.length || 0) + (appliedTxns?.length || 0) : "error"
|
|
8206
|
+
};
|
|
8207
|
+
if (handler.analyticsResource) {
|
|
8208
|
+
const [okAnalytics, errAnalytics, analyticsRecords] = await handler.analyticsResource.list().catch(() => [false, null, []]);
|
|
8209
|
+
fieldDiag.stats.analyticsRecords = okAnalytics ? analyticsRecords?.length || 0 : "error";
|
|
8210
|
+
}
|
|
8211
|
+
} catch (error) {
|
|
8212
|
+
diagnostics.warnings.push({
|
|
8213
|
+
resource: resName,
|
|
8214
|
+
field: fieldName,
|
|
8215
|
+
issue: "Failed to fetch statistics",
|
|
8216
|
+
error: error.message
|
|
8217
|
+
});
|
|
8218
|
+
}
|
|
8219
|
+
}
|
|
8220
|
+
resourceDiag.fields.push(fieldDiag);
|
|
8221
|
+
}
|
|
8222
|
+
if (resourceDiag.fields.length > 0) {
|
|
8223
|
+
diagnostics.resources.push(resourceDiag);
|
|
8224
|
+
}
|
|
8225
|
+
}
|
|
8226
|
+
diagnostics.health = {
|
|
8227
|
+
status: diagnostics.errors.length === 0 ? diagnostics.warnings.length === 0 ? "healthy" : "warning" : "error",
|
|
8228
|
+
totalResources: diagnostics.resources.length,
|
|
8229
|
+
totalFields: diagnostics.resources.reduce((sum, r) => sum + r.fields.length, 0),
|
|
8230
|
+
errorCount: diagnostics.errors.length,
|
|
8231
|
+
warningCount: diagnostics.warnings.length
|
|
8232
|
+
};
|
|
8233
|
+
return diagnostics;
|
|
8234
|
+
}
|
|
7779
8235
|
}
|
|
7780
8236
|
|
|
7781
8237
|
class FullTextPlugin extends Plugin {
|
|
@@ -11397,6 +11853,7 @@ ${errorDetails}`,
|
|
|
11397
11853
|
idGenerator: customIdGenerator,
|
|
11398
11854
|
idSize = 22,
|
|
11399
11855
|
versioningEnabled = false,
|
|
11856
|
+
strictValidation = true,
|
|
11400
11857
|
events = {},
|
|
11401
11858
|
asyncEvents = true,
|
|
11402
11859
|
asyncPartitions = true,
|
|
@@ -11410,6 +11867,7 @@ ${errorDetails}`,
|
|
|
11410
11867
|
this.parallelism = parallelism;
|
|
11411
11868
|
this.passphrase = passphrase ?? "secret";
|
|
11412
11869
|
this.versioningEnabled = versioningEnabled;
|
|
11870
|
+
this.strictValidation = strictValidation;
|
|
11413
11871
|
this.setAsyncMode(asyncEvents);
|
|
11414
11872
|
this.idGenerator = this.configureIdGenerator(customIdGenerator, idSize);
|
|
11415
11873
|
if (typeof customIdGenerator === "number" && customIdGenerator > 0) {
|
|
@@ -11657,9 +12115,12 @@ ${errorDetails}`,
|
|
|
11657
12115
|
}
|
|
11658
12116
|
/**
|
|
11659
12117
|
* Validate that all partition fields exist in current resource attributes
|
|
11660
|
-
* @throws {Error} If partition fields don't exist in current schema
|
|
12118
|
+
* @throws {Error} If partition fields don't exist in current schema (only when strictValidation is true)
|
|
11661
12119
|
*/
|
|
11662
12120
|
validatePartitions() {
|
|
12121
|
+
if (!this.strictValidation) {
|
|
12122
|
+
return;
|
|
12123
|
+
}
|
|
11663
12124
|
if (!this.config.partitions) {
|
|
11664
12125
|
return;
|
|
11665
12126
|
}
|
|
@@ -13794,7 +14255,7 @@ class Database extends EventEmitter {
|
|
|
13794
14255
|
this.id = idGenerator(7);
|
|
13795
14256
|
this.version = "1";
|
|
13796
14257
|
this.s3dbVersion = (() => {
|
|
13797
|
-
const [ok, err, version] = tryFn(() => true ? "11.2.
|
|
14258
|
+
const [ok, err, version] = tryFn(() => true ? "11.2.3" : "latest");
|
|
13798
14259
|
return ok ? version : "latest";
|
|
13799
14260
|
})();
|
|
13800
14261
|
this.resources = {};
|
|
@@ -13809,6 +14270,7 @@ class Database extends EventEmitter {
|
|
|
13809
14270
|
this.passphrase = options.passphrase || "secret";
|
|
13810
14271
|
this.versioningEnabled = options.versioningEnabled || false;
|
|
13811
14272
|
this.persistHooks = options.persistHooks || false;
|
|
14273
|
+
this.strictValidation = options.strictValidation !== false;
|
|
13812
14274
|
this._initHooks();
|
|
13813
14275
|
let connectionString = options.connectionString;
|
|
13814
14276
|
if (!connectionString && (options.bucket || options.accessKeyId || options.secretAccessKey)) {
|
|
@@ -13930,6 +14392,7 @@ class Database extends EventEmitter {
|
|
|
13930
14392
|
asyncEvents: versionData.asyncEvents !== void 0 ? versionData.asyncEvents : true,
|
|
13931
14393
|
hooks: this.persistHooks ? this._deserializeHooks(versionData.hooks || {}) : versionData.hooks || {},
|
|
13932
14394
|
versioningEnabled: this.versioningEnabled,
|
|
14395
|
+
strictValidation: this.strictValidation,
|
|
13933
14396
|
map: versionData.map,
|
|
13934
14397
|
idGenerator: restoredIdGenerator,
|
|
13935
14398
|
idSize: restoredIdSize
|
|
@@ -14591,6 +15054,7 @@ class Database extends EventEmitter {
|
|
|
14591
15054
|
autoDecrypt: config.autoDecrypt !== void 0 ? config.autoDecrypt : true,
|
|
14592
15055
|
hooks: hooks || {},
|
|
14593
15056
|
versioningEnabled: this.versioningEnabled,
|
|
15057
|
+
strictValidation: this.strictValidation,
|
|
14594
15058
|
map: config.map,
|
|
14595
15059
|
idGenerator: config.idGenerator,
|
|
14596
15060
|
idSize: config.idSize,
|
|
@@ -17425,5 +17889,5 @@ class StateMachinePlugin extends Plugin {
|
|
|
17425
17889
|
}
|
|
17426
17890
|
}
|
|
17427
17891
|
|
|
17428
|
-
export { AVAILABLE_BEHAVIORS, AuditPlugin, AuthenticationError, BackupPlugin, BaseError, CachePlugin, Client, ConnectionString, ConnectionStringError, CostsPlugin, CryptoError, DEFAULT_BEHAVIOR, Database, DatabaseError, EncryptionError, ErrorMap, EventualConsistencyPlugin, FullTextPlugin, InvalidResourceItem, MetricsPlugin, MissingMetadata, NoSuchBucket, NoSuchKey, NotFound, PartitionError, PermissionError, Plugin, PluginObject, QueueConsumerPlugin, ReplicatorPlugin, Resource, ResourceError, ResourceIdsPageReader, ResourceIdsReader, ResourceNotFound, ResourceReader, ResourceWriter, S3QueuePlugin, Database as S3db, S3dbError, SchedulerPlugin, Schema, SchemaError, StateMachinePlugin, UnknownError, ValidationError, Validator, behaviors, calculateAttributeNamesSize, calculateAttributeSizes, calculateEffectiveLimit, calculateSystemOverhead, calculateTotalSize, calculateUTF8Bytes, clearUTF8Cache, clearUTF8Memo, clearUTF8Memory, decode, decodeDecimal, decrypt, S3db as default, encode, encodeDecimal, encrypt, getBehavior, getSizeBreakdown, idGenerator, mapAwsError, md5, passwordGenerator, sha256, streamToString, transformValue, tryFn, tryFnSync };
|
|
17892
|
+
export { AVAILABLE_BEHAVIORS, AnalyticsNotEnabledError, AuditPlugin, AuthenticationError, BackupPlugin, BaseError, CachePlugin, Client, ConnectionString, ConnectionStringError, CostsPlugin, CryptoError, DEFAULT_BEHAVIOR, Database, DatabaseError, EncryptionError, ErrorMap, EventualConsistencyPlugin, FullTextPlugin, InvalidResourceItem, MetricsPlugin, MissingMetadata, NoSuchBucket, NoSuchKey, NotFound, PartitionError, PermissionError, Plugin, PluginObject, QueueConsumerPlugin, ReplicatorPlugin, Resource, ResourceError, ResourceIdsPageReader, ResourceIdsReader, ResourceNotFound, ResourceReader, ResourceWriter, S3QueuePlugin, Database as S3db, S3dbError, SchedulerPlugin, Schema, SchemaError, StateMachinePlugin, UnknownError, ValidationError, Validator, behaviors, calculateAttributeNamesSize, calculateAttributeSizes, calculateEffectiveLimit, calculateSystemOverhead, calculateTotalSize, calculateUTF8Bytes, clearUTF8Cache, clearUTF8Memo, clearUTF8Memory, decode, decodeDecimal, decrypt, S3db as default, encode, encodeDecimal, encrypt, getBehavior, getSizeBreakdown, idGenerator, mapAwsError, md5, passwordGenerator, sha256, streamToString, transformValue, tryFn, tryFnSync };
|
|
17429
17893
|
//# sourceMappingURL=s3db.es.js.map
|