s3db.js 11.2.2 → 11.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/s3db.cjs.js +481 -16
- package/dist/s3db.cjs.js.map +1 -1
- package/dist/s3db.es.js +481 -17
- package/dist/s3db.es.js.map +1 -1
- package/package.json +1 -1
- package/src/database.class.js +3 -0
- package/src/errors.js +115 -4
- package/src/plugins/cache/memory-cache.class.js +216 -33
- package/src/plugins/cache.plugin.js +85 -1
- package/src/plugins/eventual-consistency/analytics.js +145 -0
- package/src/plugins/eventual-consistency/index.js +203 -1
- package/src/resource.class.js +8 -1
package/package.json
CHANGED
package/src/database.class.js
CHANGED
|
@@ -35,6 +35,7 @@ export class Database extends EventEmitter {
|
|
|
35
35
|
this.passphrase = options.passphrase || "secret";
|
|
36
36
|
this.versioningEnabled = options.versioningEnabled || false;
|
|
37
37
|
this.persistHooks = options.persistHooks || false; // New configuration for hook persistence
|
|
38
|
+
this.strictValidation = options.strictValidation !== false; // Enable strict validation by default
|
|
38
39
|
|
|
39
40
|
// Initialize hooks system
|
|
40
41
|
this._initHooks();
|
|
@@ -199,6 +200,7 @@ export class Database extends EventEmitter {
|
|
|
199
200
|
asyncEvents: versionData.asyncEvents !== undefined ? versionData.asyncEvents : true,
|
|
200
201
|
hooks: this.persistHooks ? this._deserializeHooks(versionData.hooks || {}) : (versionData.hooks || {}),
|
|
201
202
|
versioningEnabled: this.versioningEnabled,
|
|
203
|
+
strictValidation: this.strictValidation,
|
|
202
204
|
map: versionData.map,
|
|
203
205
|
idGenerator: restoredIdGenerator,
|
|
204
206
|
idSize: restoredIdSize
|
|
@@ -999,6 +1001,7 @@ export class Database extends EventEmitter {
|
|
|
999
1001
|
autoDecrypt: config.autoDecrypt !== undefined ? config.autoDecrypt : true,
|
|
1000
1002
|
hooks: hooks || {},
|
|
1001
1003
|
versioningEnabled: this.versioningEnabled,
|
|
1004
|
+
strictValidation: this.strictValidation,
|
|
1002
1005
|
map: config.map,
|
|
1003
1006
|
idGenerator: config.idGenerator,
|
|
1004
1007
|
idSize: config.idSize,
|
package/src/errors.js
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
export class BaseError extends Error {
|
|
2
|
-
constructor({ verbose, bucket, key, message, code, statusCode, requestId, awsMessage, original, commandName, commandInput, metadata, suggestion, ...rest }) {
|
|
2
|
+
constructor({ verbose, bucket, key, message, code, statusCode, requestId, awsMessage, original, commandName, commandInput, metadata, suggestion, description, ...rest }) {
|
|
3
3
|
if (verbose) message = message + `\n\nVerbose:\n\n${JSON.stringify(rest, null, 2)}`;
|
|
4
4
|
super(message);
|
|
5
5
|
|
|
6
6
|
if (typeof Error.captureStackTrace === 'function') {
|
|
7
7
|
Error.captureStackTrace(this, this.constructor);
|
|
8
|
-
} else {
|
|
9
|
-
this.stack = (new Error(message)).stack;
|
|
8
|
+
} else {
|
|
9
|
+
this.stack = (new Error(message)).stack;
|
|
10
10
|
}
|
|
11
11
|
|
|
12
12
|
super.name = this.constructor.name;
|
|
@@ -23,6 +23,7 @@ export class BaseError extends Error {
|
|
|
23
23
|
this.commandInput = commandInput;
|
|
24
24
|
this.metadata = metadata;
|
|
25
25
|
this.suggestion = suggestion;
|
|
26
|
+
this.description = description;
|
|
26
27
|
this.data = { bucket, key, ...rest, verbose, message };
|
|
27
28
|
}
|
|
28
29
|
|
|
@@ -41,6 +42,7 @@ export class BaseError extends Error {
|
|
|
41
42
|
commandInput: this.commandInput,
|
|
42
43
|
metadata: this.metadata,
|
|
43
44
|
suggestion: this.suggestion,
|
|
45
|
+
description: this.description,
|
|
44
46
|
data: this.data,
|
|
45
47
|
original: this.original,
|
|
46
48
|
stack: this.stack,
|
|
@@ -264,6 +266,115 @@ export class ResourceError extends S3dbError {
|
|
|
264
266
|
|
|
265
267
|
export class PartitionError extends S3dbError {
|
|
266
268
|
constructor(message, details = {}) {
|
|
267
|
-
|
|
269
|
+
// Generate description if not provided
|
|
270
|
+
let description = details.description;
|
|
271
|
+
if (!description && details.resourceName && details.partitionName && details.fieldName) {
|
|
272
|
+
const { resourceName, partitionName, fieldName, availableFields = [] } = details;
|
|
273
|
+
description = `
|
|
274
|
+
Partition Field Validation Error
|
|
275
|
+
|
|
276
|
+
Resource: ${resourceName}
|
|
277
|
+
Partition: ${partitionName}
|
|
278
|
+
Missing Field: ${fieldName}
|
|
279
|
+
|
|
280
|
+
Available fields in schema:
|
|
281
|
+
${availableFields.map(f => ` • ${f}`).join('\n') || ' (no fields defined)'}
|
|
282
|
+
|
|
283
|
+
Possible causes:
|
|
284
|
+
1. Field was removed from schema but partition still references it
|
|
285
|
+
2. Typo in partition field name
|
|
286
|
+
3. Nested field path is incorrect (use dot notation like 'utm.source')
|
|
287
|
+
|
|
288
|
+
Solution:
|
|
289
|
+
${details.strictValidation === false
|
|
290
|
+
? ' • Update partition definition to use existing fields'
|
|
291
|
+
: ` • Add missing field to schema, OR
|
|
292
|
+
• Update partition definition to use existing fields, OR
|
|
293
|
+
• Use strictValidation: false to skip this check during testing`}
|
|
294
|
+
|
|
295
|
+
Docs: https://docs.s3db.js.org/resources/partitions#validation
|
|
296
|
+
`.trim();
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
super(message, {
|
|
300
|
+
...details,
|
|
301
|
+
description,
|
|
302
|
+
suggestion: details.suggestion || 'Check partition definition, fields, and input values.'
|
|
303
|
+
});
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
export class AnalyticsNotEnabledError extends S3dbError {
|
|
308
|
+
constructor(details = {}) {
|
|
309
|
+
const {
|
|
310
|
+
pluginName = 'EventualConsistency',
|
|
311
|
+
resourceName = 'unknown',
|
|
312
|
+
field = 'unknown',
|
|
313
|
+
configuredResources = [],
|
|
314
|
+
registeredResources = [],
|
|
315
|
+
pluginInitialized = false,
|
|
316
|
+
...rest
|
|
317
|
+
} = details;
|
|
318
|
+
|
|
319
|
+
const message = `Analytics not enabled for ${resourceName}.${field}`;
|
|
320
|
+
|
|
321
|
+
// Generate diagnostic description
|
|
322
|
+
const description = `
|
|
323
|
+
Analytics Not Enabled
|
|
324
|
+
|
|
325
|
+
Plugin: ${pluginName}
|
|
326
|
+
Resource: ${resourceName}
|
|
327
|
+
Field: ${field}
|
|
328
|
+
|
|
329
|
+
Diagnostics:
|
|
330
|
+
• Plugin initialized: ${pluginInitialized ? '✓ Yes' : '✗ No'}
|
|
331
|
+
• Analytics resources created: ${registeredResources.length}/${configuredResources.length}
|
|
332
|
+
${configuredResources.map(r => {
|
|
333
|
+
const exists = registeredResources.includes(r);
|
|
334
|
+
return ` ${exists ? '✓' : '✗'} ${r}${!exists ? ' (missing)' : ''}`;
|
|
335
|
+
}).join('\n')}
|
|
336
|
+
|
|
337
|
+
Possible causes:
|
|
338
|
+
1. Resource not created yet - Analytics resources are created when db.createResource() is called
|
|
339
|
+
2. Resource created before plugin initialization - Plugin must be initialized before resources
|
|
340
|
+
3. Field not configured in analytics.resources config
|
|
341
|
+
|
|
342
|
+
Correct initialization order:
|
|
343
|
+
1. Create database: const db = new Database({ ... })
|
|
344
|
+
2. Install plugins: await db.connect() (triggers plugin.install())
|
|
345
|
+
3. Create resources: await db.createResource({ name: '${resourceName}', ... })
|
|
346
|
+
4. Analytics resources are auto-created by plugin
|
|
347
|
+
|
|
348
|
+
Example fix:
|
|
349
|
+
const db = new Database({
|
|
350
|
+
bucket: 'my-bucket',
|
|
351
|
+
plugins: [new EventualConsistencyPlugin({
|
|
352
|
+
resources: {
|
|
353
|
+
'${resourceName}': {
|
|
354
|
+
fields: {
|
|
355
|
+
'${field}': { type: 'counter', analytics: true }
|
|
356
|
+
}
|
|
357
|
+
}
|
|
358
|
+
}
|
|
359
|
+
})]
|
|
360
|
+
});
|
|
361
|
+
|
|
362
|
+
await db.connect(); // Plugin initialized here
|
|
363
|
+
await db.createResource({ name: '${resourceName}', ... }); // Analytics resource created here
|
|
364
|
+
|
|
365
|
+
Docs: https://docs.s3db.js.org/plugins/eventual-consistency#troubleshooting
|
|
366
|
+
`.trim();
|
|
367
|
+
|
|
368
|
+
super(message, {
|
|
369
|
+
...rest,
|
|
370
|
+
pluginName,
|
|
371
|
+
resourceName,
|
|
372
|
+
field,
|
|
373
|
+
configuredResources,
|
|
374
|
+
registeredResources,
|
|
375
|
+
pluginInitialized,
|
|
376
|
+
description,
|
|
377
|
+
suggestion: 'Ensure resources are created after plugin initialization. Check plugin configuration and resource creation order.'
|
|
378
|
+
});
|
|
268
379
|
}
|
|
269
380
|
}
|
|
@@ -1,12 +1,14 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Memory Cache Configuration Documentation
|
|
3
|
-
*
|
|
3
|
+
*
|
|
4
4
|
* This cache implementation stores data in memory using a Map-like structure.
|
|
5
5
|
* It provides fast access to frequently used data but is limited by available RAM
|
|
6
6
|
* and data is lost when the process restarts.
|
|
7
|
-
*
|
|
7
|
+
*
|
|
8
8
|
* @typedef {Object} MemoryCacheConfig
|
|
9
9
|
* @property {number} [maxSize=1000] - Maximum number of items to store in cache
|
|
10
|
+
* @property {number} [maxMemoryBytes=0] - Maximum memory usage in bytes (0 = unlimited). When set, cache will evict items to stay under this limit.
|
|
11
|
+
* @property {number} [maxMemoryPercent=0] - Maximum memory usage as decimal fraction of total system memory (0 = unlimited, 0.1 = 10%, 0.5 = 50%, 1.0 = 100%). Takes precedence over maxMemoryBytes if both are set.
|
|
10
12
|
* @property {number} [ttl=300000] - Time to live in milliseconds (5 minutes default)
|
|
11
13
|
* @property {boolean} [enableStats=false] - Whether to track cache statistics (hits, misses, etc.)
|
|
12
14
|
* @property {string} [evictionPolicy='lru'] - Cache eviction policy: 'lru' (Least Recently Used) or 'fifo' (First In First Out)
|
|
@@ -70,9 +72,54 @@
|
|
|
70
72
|
* maxSize: 1000,
|
|
71
73
|
* ttl: 300000 // 5 minutes
|
|
72
74
|
* }
|
|
73
|
-
*
|
|
75
|
+
*
|
|
76
|
+
* @example
|
|
77
|
+
* // Memory-limited configuration (prevents memory exhaustion)
|
|
78
|
+
* {
|
|
79
|
+
* maxMemoryBytes: 100 * 1024 * 1024, // 100MB hard limit
|
|
80
|
+
* ttl: 600000, // 10 minutes
|
|
81
|
+
* enableCompression: true, // Reduce memory usage
|
|
82
|
+
* compressionThreshold: 1024 // Compress items > 1KB
|
|
83
|
+
* }
|
|
84
|
+
*
|
|
85
|
+
* @example
|
|
86
|
+
* // Production configuration with memory monitoring (absolute bytes)
|
|
87
|
+
* {
|
|
88
|
+
* maxSize: 5000, // Limit number of items
|
|
89
|
+
* maxMemoryBytes: 512 * 1024 * 1024, // 512MB memory limit
|
|
90
|
+
* ttl: 1800000, // 30 minutes
|
|
91
|
+
* enableCompression: true,
|
|
92
|
+
* compressionThreshold: 512
|
|
93
|
+
* }
|
|
94
|
+
*
|
|
95
|
+
* // Check memory usage
|
|
96
|
+
* const stats = cache.getMemoryStats();
|
|
97
|
+
* console.log(`Memory: ${stats.memoryUsage.current} / ${stats.memoryUsage.max}`);
|
|
98
|
+
* console.log(`Usage: ${stats.memoryUsagePercent}%`);
|
|
99
|
+
* console.log(`Evicted due to memory: ${stats.evictedDueToMemory}`);
|
|
100
|
+
*
|
|
101
|
+
* @example
|
|
102
|
+
* // Production configuration with percentage of system memory
|
|
103
|
+
* {
|
|
104
|
+
* maxMemoryPercent: 0.1, // Use max 10% of system memory (0.1 = 10%)
|
|
105
|
+
* ttl: 1800000, // 30 minutes
|
|
106
|
+
* enableCompression: true
|
|
107
|
+
* }
|
|
108
|
+
*
|
|
109
|
+
* // On a 16GB system, this sets maxMemoryBytes to ~1.6GB
|
|
110
|
+
* // On a 32GB system, this sets maxMemoryBytes to ~3.2GB
|
|
111
|
+
*
|
|
112
|
+
* // Check system memory stats
|
|
113
|
+
* const stats = cache.getMemoryStats();
|
|
114
|
+
* console.log(`System Memory: ${stats.systemMemory.total}`);
|
|
115
|
+
* console.log(`Cache using: ${stats.systemMemory.cachePercent} of system memory`);
|
|
116
|
+
* console.log(`Max allowed: ${(stats.maxMemoryPercent * 100).toFixed(1)}%`);
|
|
117
|
+
*
|
|
74
118
|
* @notes
|
|
75
|
-
* - Memory usage is limited by available RAM and
|
|
119
|
+
* - Memory usage is limited by available RAM, maxSize setting, and optionally maxMemoryBytes or maxMemoryPercent
|
|
120
|
+
* - maxMemoryPercent takes precedence over maxMemoryBytes if both are set
|
|
121
|
+
* - maxMemoryPercent is calculated based on total system memory at cache creation time
|
|
122
|
+
* - Useful for containerized/cloud environments where system memory varies
|
|
76
123
|
* - TTL is checked on access, not automatically in background
|
|
77
124
|
* - LRU eviction removes least recently accessed items when cache is full
|
|
78
125
|
* - FIFO eviction removes oldest items when cache is full
|
|
@@ -83,8 +130,12 @@
|
|
|
83
130
|
* - Cleanup interval helps prevent memory leaks from expired items
|
|
84
131
|
* - Tags are useful for cache invalidation and monitoring
|
|
85
132
|
* - Case sensitivity affects key matching and storage efficiency
|
|
133
|
+
* - maxMemoryBytes prevents memory exhaustion by enforcing byte-level limits
|
|
134
|
+
* - Memory tracking includes serialized data size (compressed or uncompressed)
|
|
135
|
+
* - getMemoryStats() includes systemMemory info for monitoring
|
|
86
136
|
*/
|
|
87
137
|
import zlib from 'node:zlib';
|
|
138
|
+
import os from 'node:os';
|
|
88
139
|
import { Cache } from "./cache.class.js"
|
|
89
140
|
|
|
90
141
|
export class MemoryCache extends Cache {
|
|
@@ -93,12 +144,39 @@ export class MemoryCache extends Cache {
|
|
|
93
144
|
this.cache = {};
|
|
94
145
|
this.meta = {};
|
|
95
146
|
this.maxSize = config.maxSize !== undefined ? config.maxSize : 1000;
|
|
147
|
+
|
|
148
|
+
// Validate that only one memory limit option is used
|
|
149
|
+
if (config.maxMemoryBytes && config.maxMemoryBytes > 0 &&
|
|
150
|
+
config.maxMemoryPercent && config.maxMemoryPercent > 0) {
|
|
151
|
+
throw new Error(
|
|
152
|
+
'[MemoryCache] Cannot use both maxMemoryBytes and maxMemoryPercent. ' +
|
|
153
|
+
'Choose one: maxMemoryBytes (absolute) or maxMemoryPercent (0...1 fraction).'
|
|
154
|
+
);
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
// Calculate maxMemoryBytes from percentage if provided
|
|
158
|
+
if (config.maxMemoryPercent && config.maxMemoryPercent > 0) {
|
|
159
|
+
if (config.maxMemoryPercent > 1) {
|
|
160
|
+
throw new Error(
|
|
161
|
+
'[MemoryCache] maxMemoryPercent must be between 0 and 1 (e.g., 0.1 for 10%). ' +
|
|
162
|
+
`Received: ${config.maxMemoryPercent}`
|
|
163
|
+
);
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
const totalMemory = os.totalmem();
|
|
167
|
+
this.maxMemoryBytes = Math.floor(totalMemory * config.maxMemoryPercent);
|
|
168
|
+
this.maxMemoryPercent = config.maxMemoryPercent;
|
|
169
|
+
} else {
|
|
170
|
+
this.maxMemoryBytes = config.maxMemoryBytes !== undefined ? config.maxMemoryBytes : 0; // 0 = unlimited
|
|
171
|
+
this.maxMemoryPercent = 0;
|
|
172
|
+
}
|
|
173
|
+
|
|
96
174
|
this.ttl = config.ttl !== undefined ? config.ttl : 300000;
|
|
97
|
-
|
|
175
|
+
|
|
98
176
|
// Compression configuration
|
|
99
177
|
this.enableCompression = config.enableCompression !== undefined ? config.enableCompression : false;
|
|
100
178
|
this.compressionThreshold = config.compressionThreshold !== undefined ? config.compressionThreshold : 1024;
|
|
101
|
-
|
|
179
|
+
|
|
102
180
|
// Stats for compression
|
|
103
181
|
this.compressionStats = {
|
|
104
182
|
totalCompressed: 0,
|
|
@@ -106,33 +184,26 @@ export class MemoryCache extends Cache {
|
|
|
106
184
|
totalCompressedSize: 0,
|
|
107
185
|
compressionRatio: 0
|
|
108
186
|
};
|
|
187
|
+
|
|
188
|
+
// Memory tracking
|
|
189
|
+
this.currentMemoryBytes = 0;
|
|
190
|
+
this.evictedDueToMemory = 0;
|
|
109
191
|
}
|
|
110
192
|
|
|
111
193
|
async _set(key, data) {
|
|
112
|
-
// Limpar se exceder maxSize
|
|
113
|
-
if (this.maxSize > 0 && Object.keys(this.cache).length >= this.maxSize) {
|
|
114
|
-
// Remove o item mais antigo
|
|
115
|
-
const oldestKey = Object.entries(this.meta)
|
|
116
|
-
.sort((a, b) => a[1].ts - b[1].ts)[0]?.[0];
|
|
117
|
-
if (oldestKey) {
|
|
118
|
-
delete this.cache[oldestKey];
|
|
119
|
-
delete this.meta[oldestKey];
|
|
120
|
-
}
|
|
121
|
-
}
|
|
122
|
-
|
|
123
194
|
// Prepare data for storage
|
|
124
195
|
let finalData = data;
|
|
125
196
|
let compressed = false;
|
|
126
197
|
let originalSize = 0;
|
|
127
198
|
let compressedSize = 0;
|
|
128
|
-
|
|
199
|
+
|
|
200
|
+
// Calculate size first (needed for both compression and memory limit checks)
|
|
201
|
+
const serialized = JSON.stringify(data);
|
|
202
|
+
originalSize = Buffer.byteLength(serialized, 'utf8');
|
|
203
|
+
|
|
129
204
|
// Apply compression if enabled
|
|
130
205
|
if (this.enableCompression) {
|
|
131
206
|
try {
|
|
132
|
-
// Serialize data to measure size
|
|
133
|
-
const serialized = JSON.stringify(data);
|
|
134
|
-
originalSize = Buffer.byteLength(serialized, 'utf8');
|
|
135
|
-
|
|
136
207
|
// Compress only if over threshold
|
|
137
208
|
if (originalSize >= this.compressionThreshold) {
|
|
138
209
|
const compressedBuffer = zlib.gzipSync(Buffer.from(serialized, 'utf8'));
|
|
@@ -143,12 +214,12 @@ export class MemoryCache extends Cache {
|
|
|
143
214
|
};
|
|
144
215
|
compressedSize = Buffer.byteLength(finalData.__data, 'utf8');
|
|
145
216
|
compressed = true;
|
|
146
|
-
|
|
217
|
+
|
|
147
218
|
// Update compression stats
|
|
148
219
|
this.compressionStats.totalCompressed++;
|
|
149
220
|
this.compressionStats.totalOriginalSize += originalSize;
|
|
150
221
|
this.compressionStats.totalCompressedSize += compressedSize;
|
|
151
|
-
this.compressionStats.compressionRatio =
|
|
222
|
+
this.compressionStats.compressionRatio =
|
|
152
223
|
(this.compressionStats.totalCompressedSize / this.compressionStats.totalOriginalSize).toFixed(2);
|
|
153
224
|
}
|
|
154
225
|
} catch (error) {
|
|
@@ -156,33 +227,79 @@ export class MemoryCache extends Cache {
|
|
|
156
227
|
console.warn(`[MemoryCache] Compression failed for key '${key}':`, error.message);
|
|
157
228
|
}
|
|
158
229
|
}
|
|
159
|
-
|
|
230
|
+
|
|
231
|
+
// Calculate actual storage size (compressed or original)
|
|
232
|
+
const itemSize = compressed ? compressedSize : originalSize;
|
|
233
|
+
|
|
234
|
+
// If replacing existing key, subtract its old size from current memory
|
|
235
|
+
if (Object.prototype.hasOwnProperty.call(this.cache, key)) {
|
|
236
|
+
const oldSize = this.meta[key]?.compressedSize || 0;
|
|
237
|
+
this.currentMemoryBytes -= oldSize;
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
// Memory-aware eviction: Remove items until we have space
|
|
241
|
+
if (this.maxMemoryBytes > 0) {
|
|
242
|
+
while (this.currentMemoryBytes + itemSize > this.maxMemoryBytes && Object.keys(this.cache).length > 0) {
|
|
243
|
+
// Remove the oldest item
|
|
244
|
+
const oldestKey = Object.entries(this.meta)
|
|
245
|
+
.sort((a, b) => a[1].ts - b[1].ts)[0]?.[0];
|
|
246
|
+
if (oldestKey) {
|
|
247
|
+
const evictedSize = this.meta[oldestKey]?.compressedSize || 0;
|
|
248
|
+
delete this.cache[oldestKey];
|
|
249
|
+
delete this.meta[oldestKey];
|
|
250
|
+
this.currentMemoryBytes -= evictedSize;
|
|
251
|
+
this.evictedDueToMemory++;
|
|
252
|
+
} else {
|
|
253
|
+
break; // No more items to evict
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
// Item count eviction (original logic)
|
|
259
|
+
if (this.maxSize > 0 && Object.keys(this.cache).length >= this.maxSize) {
|
|
260
|
+
// Remove o item mais antigo
|
|
261
|
+
const oldestKey = Object.entries(this.meta)
|
|
262
|
+
.sort((a, b) => a[1].ts - b[1].ts)[0]?.[0];
|
|
263
|
+
if (oldestKey) {
|
|
264
|
+
const evictedSize = this.meta[oldestKey]?.compressedSize || 0;
|
|
265
|
+
delete this.cache[oldestKey];
|
|
266
|
+
delete this.meta[oldestKey];
|
|
267
|
+
this.currentMemoryBytes -= evictedSize;
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
// Store the item
|
|
160
272
|
this.cache[key] = finalData;
|
|
161
|
-
this.meta[key] = {
|
|
273
|
+
this.meta[key] = {
|
|
162
274
|
ts: Date.now(),
|
|
163
275
|
compressed,
|
|
164
276
|
originalSize,
|
|
165
|
-
compressedSize:
|
|
277
|
+
compressedSize: itemSize
|
|
166
278
|
};
|
|
167
|
-
|
|
279
|
+
|
|
280
|
+
// Update current memory usage
|
|
281
|
+
this.currentMemoryBytes += itemSize;
|
|
282
|
+
|
|
168
283
|
return data;
|
|
169
284
|
}
|
|
170
285
|
|
|
171
286
|
async _get(key) {
|
|
172
287
|
if (!Object.prototype.hasOwnProperty.call(this.cache, key)) return null;
|
|
173
|
-
|
|
288
|
+
|
|
174
289
|
// Check TTL expiration
|
|
175
290
|
if (this.ttl > 0) {
|
|
176
291
|
const now = Date.now();
|
|
177
292
|
const meta = this.meta[key];
|
|
178
|
-
if (meta && now - meta.ts > this.ttl
|
|
179
|
-
//
|
|
293
|
+
if (meta && now - meta.ts > this.ttl) {
|
|
294
|
+
// Expired - decrement memory before deleting
|
|
295
|
+
const itemSize = meta.compressedSize || 0;
|
|
296
|
+
this.currentMemoryBytes -= itemSize;
|
|
180
297
|
delete this.cache[key];
|
|
181
298
|
delete this.meta[key];
|
|
182
299
|
return null;
|
|
183
300
|
}
|
|
184
301
|
}
|
|
185
|
-
|
|
302
|
+
|
|
186
303
|
const rawData = this.cache[key];
|
|
187
304
|
|
|
188
305
|
// Check if data is compressed
|
|
@@ -206,6 +323,12 @@ export class MemoryCache extends Cache {
|
|
|
206
323
|
}
|
|
207
324
|
|
|
208
325
|
async _del(key) {
|
|
326
|
+
// Decrement memory usage
|
|
327
|
+
if (Object.prototype.hasOwnProperty.call(this.cache, key)) {
|
|
328
|
+
const itemSize = this.meta[key]?.compressedSize || 0;
|
|
329
|
+
this.currentMemoryBytes -= itemSize;
|
|
330
|
+
}
|
|
331
|
+
|
|
209
332
|
delete this.cache[key];
|
|
210
333
|
delete this.meta[key];
|
|
211
334
|
return true;
|
|
@@ -215,6 +338,7 @@ export class MemoryCache extends Cache {
|
|
|
215
338
|
if (!prefix) {
|
|
216
339
|
this.cache = {};
|
|
217
340
|
this.meta = {};
|
|
341
|
+
this.currentMemoryBytes = 0; // Reset memory counter
|
|
218
342
|
return true;
|
|
219
343
|
}
|
|
220
344
|
// Remove only keys that start with the prefix
|
|
@@ -222,6 +346,9 @@ export class MemoryCache extends Cache {
|
|
|
222
346
|
for (const key of Object.keys(this.cache)) {
|
|
223
347
|
if (key.startsWith(prefix)) {
|
|
224
348
|
removed.push(key);
|
|
349
|
+
// Decrement memory usage
|
|
350
|
+
const itemSize = this.meta[key]?.compressedSize || 0;
|
|
351
|
+
this.currentMemoryBytes -= itemSize;
|
|
225
352
|
delete this.cache[key];
|
|
226
353
|
delete this.meta[key];
|
|
227
354
|
}
|
|
@@ -248,7 +375,7 @@ export class MemoryCache extends Cache {
|
|
|
248
375
|
return { enabled: false, message: 'Compression is disabled' };
|
|
249
376
|
}
|
|
250
377
|
|
|
251
|
-
const spaceSavings = this.compressionStats.totalOriginalSize > 0
|
|
378
|
+
const spaceSavings = this.compressionStats.totalOriginalSize > 0
|
|
252
379
|
? ((this.compressionStats.totalOriginalSize - this.compressionStats.totalCompressedSize) / this.compressionStats.totalOriginalSize * 100).toFixed(2)
|
|
253
380
|
: 0;
|
|
254
381
|
|
|
@@ -268,6 +395,62 @@ export class MemoryCache extends Cache {
|
|
|
268
395
|
}
|
|
269
396
|
};
|
|
270
397
|
}
|
|
398
|
+
|
|
399
|
+
/**
|
|
400
|
+
* Get memory usage statistics
|
|
401
|
+
* @returns {Object} Memory stats including current usage, limits, and eviction counts
|
|
402
|
+
*/
|
|
403
|
+
getMemoryStats() {
|
|
404
|
+
const totalItems = Object.keys(this.cache).length;
|
|
405
|
+
const memoryUsagePercent = this.maxMemoryBytes > 0
|
|
406
|
+
? ((this.currentMemoryBytes / this.maxMemoryBytes) * 100).toFixed(2)
|
|
407
|
+
: 0;
|
|
408
|
+
|
|
409
|
+
const systemMemory = {
|
|
410
|
+
total: os.totalmem(),
|
|
411
|
+
free: os.freemem(),
|
|
412
|
+
used: os.totalmem() - os.freemem()
|
|
413
|
+
};
|
|
414
|
+
|
|
415
|
+
const cachePercentOfTotal = systemMemory.total > 0
|
|
416
|
+
? ((this.currentMemoryBytes / systemMemory.total) * 100).toFixed(2)
|
|
417
|
+
: 0;
|
|
418
|
+
|
|
419
|
+
return {
|
|
420
|
+
currentMemoryBytes: this.currentMemoryBytes,
|
|
421
|
+
maxMemoryBytes: this.maxMemoryBytes,
|
|
422
|
+
maxMemoryPercent: this.maxMemoryPercent,
|
|
423
|
+
memoryUsagePercent: parseFloat(memoryUsagePercent),
|
|
424
|
+
cachePercentOfSystemMemory: parseFloat(cachePercentOfTotal),
|
|
425
|
+
totalItems,
|
|
426
|
+
maxSize: this.maxSize,
|
|
427
|
+
evictedDueToMemory: this.evictedDueToMemory,
|
|
428
|
+
averageItemSize: totalItems > 0 ? Math.round(this.currentMemoryBytes / totalItems) : 0,
|
|
429
|
+
memoryUsage: {
|
|
430
|
+
current: this._formatBytes(this.currentMemoryBytes),
|
|
431
|
+
max: this.maxMemoryBytes > 0 ? this._formatBytes(this.maxMemoryBytes) : 'unlimited',
|
|
432
|
+
available: this.maxMemoryBytes > 0 ? this._formatBytes(this.maxMemoryBytes - this.currentMemoryBytes) : 'unlimited'
|
|
433
|
+
},
|
|
434
|
+
systemMemory: {
|
|
435
|
+
total: this._formatBytes(systemMemory.total),
|
|
436
|
+
free: this._formatBytes(systemMemory.free),
|
|
437
|
+
used: this._formatBytes(systemMemory.used),
|
|
438
|
+
cachePercent: `${cachePercentOfTotal}%`
|
|
439
|
+
}
|
|
440
|
+
};
|
|
441
|
+
}
|
|
442
|
+
|
|
443
|
+
/**
|
|
444
|
+
* Format bytes to human-readable format
|
|
445
|
+
* @private
|
|
446
|
+
*/
|
|
447
|
+
_formatBytes(bytes) {
|
|
448
|
+
if (bytes === 0) return '0 B';
|
|
449
|
+
const k = 1024;
|
|
450
|
+
const sizes = ['B', 'KB', 'MB', 'GB'];
|
|
451
|
+
const i = Math.floor(Math.log(bytes) / Math.log(k));
|
|
452
|
+
return `${(bytes / Math.pow(k, i)).toFixed(2)} ${sizes[i]}`;
|
|
453
|
+
}
|
|
271
454
|
}
|
|
272
455
|
|
|
273
456
|
export default MemoryCache
|
|
@@ -9,6 +9,88 @@ import { FilesystemCache } from "./cache/filesystem-cache.class.js";
|
|
|
9
9
|
import { PartitionAwareFilesystemCache } from "./cache/partition-aware-filesystem-cache.class.js";
|
|
10
10
|
import tryFn from "../concerns/try-fn.js";
|
|
11
11
|
|
|
12
|
+
/**
|
|
13
|
+
* Cache Plugin Configuration
|
|
14
|
+
*
|
|
15
|
+
* Provides caching layer for S3DB resources with multiple backend options.
|
|
16
|
+
* Automatically caches read operations and invalidates on writes.
|
|
17
|
+
*
|
|
18
|
+
* @typedef {Object} CachePluginOptions
|
|
19
|
+
* @property {string} [driver='s3'] - Cache driver: 'memory', 'filesystem', 's3', or custom driver instance
|
|
20
|
+
* @property {number} [ttl] - Time to live in milliseconds for cached items (shortcut for config.ttl)
|
|
21
|
+
* @property {number} [maxSize] - Maximum number of items to cache (shortcut for config.maxSize)
|
|
22
|
+
* @property {number} [maxMemoryBytes] - (MemoryCache only) Maximum memory in bytes (shortcut for config.maxMemoryBytes). Cannot be used with maxMemoryPercent.
|
|
23
|
+
* @property {number} [maxMemoryPercent] - (MemoryCache only) Maximum memory as fraction 0...1 (shortcut for config.maxMemoryPercent). Cannot be used with maxMemoryBytes.
|
|
24
|
+
*
|
|
25
|
+
* @property {Array<string>} [include] - Only cache these resource names (null = cache all)
|
|
26
|
+
* @property {Array<string>} [exclude=[]] - Never cache these resource names
|
|
27
|
+
*
|
|
28
|
+
* @property {boolean} [includePartitions=true] - Whether to cache partition queries
|
|
29
|
+
* @property {string} [partitionStrategy='hierarchical'] - Partition caching strategy
|
|
30
|
+
* @property {boolean} [partitionAware=true] - Use partition-aware filesystem cache
|
|
31
|
+
* @property {boolean} [trackUsage=true] - Track cache usage statistics
|
|
32
|
+
* @property {boolean} [preloadRelated=true] - Preload related partitions
|
|
33
|
+
*
|
|
34
|
+
* @property {number} [retryAttempts=3] - Number of retry attempts for cache operations
|
|
35
|
+
* @property {number} [retryDelay=100] - Delay between retries in milliseconds
|
|
36
|
+
* @property {boolean} [verbose=false] - Enable verbose logging
|
|
37
|
+
*
|
|
38
|
+
* @property {Object} [config] - Driver-specific configuration (can override top-level ttl, maxSize, maxMemoryBytes, maxMemoryPercent)
|
|
39
|
+
* @property {number} [config.ttl] - Override TTL for this driver
|
|
40
|
+
* @property {number} [config.maxSize] - Override max number of items
|
|
41
|
+
* @property {number} [config.maxMemoryBytes] - (MemoryCache only) Maximum memory in bytes. Cannot be used with config.maxMemoryPercent.
|
|
42
|
+
* @property {number} [config.maxMemoryPercent] - (MemoryCache only) Maximum memory as fraction 0...1 (e.g., 0.1 = 10%). Cannot be used with config.maxMemoryBytes.
|
|
43
|
+
* @property {boolean} [config.enableCompression] - (MemoryCache only) Enable gzip compression
|
|
44
|
+
* @property {number} [config.compressionThreshold=1024] - (MemoryCache only) Minimum size in bytes to trigger compression
|
|
45
|
+
*
|
|
46
|
+
* @example
|
|
47
|
+
* // Memory cache with absolute byte limit
|
|
48
|
+
* new CachePlugin({
|
|
49
|
+
* driver: 'memory',
|
|
50
|
+
* maxMemoryBytes: 512 * 1024 * 1024, // 512MB
|
|
51
|
+
* ttl: 600000 // 10 minutes
|
|
52
|
+
* })
|
|
53
|
+
*
|
|
54
|
+
* @example
|
|
55
|
+
* // Memory cache with percentage limit (cloud-native)
|
|
56
|
+
* new CachePlugin({
|
|
57
|
+
* driver: 'memory',
|
|
58
|
+
* maxMemoryPercent: 0.1, // 10% of system memory
|
|
59
|
+
* ttl: 1800000 // 30 minutes
|
|
60
|
+
* })
|
|
61
|
+
*
|
|
62
|
+
* @example
|
|
63
|
+
* // Filesystem cache with partition awareness
|
|
64
|
+
* new CachePlugin({
|
|
65
|
+
* driver: 'filesystem',
|
|
66
|
+
* partitionAware: true,
|
|
67
|
+
* includePartitions: true,
|
|
68
|
+
* ttl: 3600000 // 1 hour
|
|
69
|
+
* })
|
|
70
|
+
*
|
|
71
|
+
* @example
|
|
72
|
+
* // S3 cache (default)
|
|
73
|
+
* new CachePlugin({
|
|
74
|
+
* driver: 's3',
|
|
75
|
+
* ttl: 7200000 // 2 hours
|
|
76
|
+
* })
|
|
77
|
+
*
|
|
78
|
+
* @example
|
|
79
|
+
* // Selective caching
|
|
80
|
+
* new CachePlugin({
|
|
81
|
+
* driver: 'memory',
|
|
82
|
+
* include: ['users', 'products'], // Only cache these
|
|
83
|
+
* exclude: ['audit_logs'], // Never cache these
|
|
84
|
+
* maxMemoryPercent: 0.15
|
|
85
|
+
* })
|
|
86
|
+
*
|
|
87
|
+
* @notes
|
|
88
|
+
* - maxMemoryBytes and maxMemoryPercent are mutually exclusive (throws error if both set)
|
|
89
|
+
* - maxMemoryPercent is recommended for containerized/cloud environments
|
|
90
|
+
* - Plugin-created resources (createdBy !== 'user') are skipped by default
|
|
91
|
+
* - Cache is automatically invalidated on insert/update/delete operations
|
|
92
|
+
* - Use skipCache: true option on queries to bypass cache for specific calls
|
|
93
|
+
*/
|
|
12
94
|
export class CachePlugin extends Plugin {
|
|
13
95
|
constructor(options = {}) {
|
|
14
96
|
super(options);
|
|
@@ -20,7 +102,9 @@ export class CachePlugin extends Plugin {
|
|
|
20
102
|
config: {
|
|
21
103
|
ttl: options.ttl,
|
|
22
104
|
maxSize: options.maxSize,
|
|
23
|
-
|
|
105
|
+
maxMemoryBytes: options.maxMemoryBytes,
|
|
106
|
+
maxMemoryPercent: options.maxMemoryPercent,
|
|
107
|
+
...options.config // Driver-specific config (can override ttl/maxSize/maxMemoryBytes/maxMemoryPercent)
|
|
24
108
|
},
|
|
25
109
|
|
|
26
110
|
// Resource filtering
|