s3db.js 11.2.2 → 11.2.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/s3db.cjs.js +1650 -136
- package/dist/s3db.cjs.js.map +1 -1
- package/dist/s3db.es.js +1644 -137
- package/dist/s3db.es.js.map +1 -1
- package/package.json +1 -1
- package/src/behaviors/enforce-limits.js +28 -4
- package/src/behaviors/index.js +6 -1
- package/src/client.class.js +11 -1
- package/src/concerns/partition-queue.js +7 -1
- package/src/concerns/plugin-storage.js +75 -13
- package/src/database.class.js +22 -4
- package/src/errors.js +414 -24
- package/src/partition-drivers/base-partition-driver.js +12 -2
- package/src/partition-drivers/index.js +7 -1
- package/src/partition-drivers/memory-partition-driver.js +20 -5
- package/src/partition-drivers/sqs-partition-driver.js +6 -1
- package/src/plugins/audit.errors.js +46 -0
- package/src/plugins/backup/base-backup-driver.class.js +36 -6
- package/src/plugins/backup/filesystem-backup-driver.class.js +55 -7
- package/src/plugins/backup/index.js +40 -9
- package/src/plugins/backup/multi-backup-driver.class.js +69 -9
- package/src/plugins/backup/s3-backup-driver.class.js +48 -6
- package/src/plugins/backup.errors.js +45 -0
- package/src/plugins/cache/cache.class.js +8 -1
- package/src/plugins/cache/memory-cache.class.js +216 -33
- package/src/plugins/cache.errors.js +47 -0
- package/src/plugins/cache.plugin.js +94 -3
- package/src/plugins/eventual-consistency/analytics.js +145 -0
- package/src/plugins/eventual-consistency/index.js +203 -1
- package/src/plugins/fulltext.errors.js +46 -0
- package/src/plugins/fulltext.plugin.js +15 -3
- package/src/plugins/metrics.errors.js +46 -0
- package/src/plugins/queue-consumer.plugin.js +31 -4
- package/src/plugins/queue.errors.js +46 -0
- package/src/plugins/replicator.errors.js +46 -0
- package/src/plugins/replicator.plugin.js +40 -5
- package/src/plugins/replicators/base-replicator.class.js +19 -3
- package/src/plugins/replicators/index.js +9 -3
- package/src/plugins/replicators/s3db-replicator.class.js +38 -8
- package/src/plugins/scheduler.errors.js +46 -0
- package/src/plugins/scheduler.plugin.js +79 -19
- package/src/plugins/state-machine.errors.js +47 -0
- package/src/plugins/state-machine.plugin.js +86 -17
- package/src/resource.class.js +8 -1
- package/src/stream/index.js +6 -1
- package/src/stream/resource-reader.class.js +6 -1
|
@@ -1,12 +1,14 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Memory Cache Configuration Documentation
|
|
3
|
-
*
|
|
3
|
+
*
|
|
4
4
|
* This cache implementation stores data in memory using a Map-like structure.
|
|
5
5
|
* It provides fast access to frequently used data but is limited by available RAM
|
|
6
6
|
* and data is lost when the process restarts.
|
|
7
|
-
*
|
|
7
|
+
*
|
|
8
8
|
* @typedef {Object} MemoryCacheConfig
|
|
9
9
|
* @property {number} [maxSize=1000] - Maximum number of items to store in cache
|
|
10
|
+
* @property {number} [maxMemoryBytes=0] - Maximum memory usage in bytes (0 = unlimited). When set, cache will evict items to stay under this limit.
|
|
11
|
+
* @property {number} [maxMemoryPercent=0] - Maximum memory usage as decimal fraction of total system memory (0 = unlimited, 0.1 = 10%, 0.5 = 50%, 1.0 = 100%). Takes precedence over maxMemoryBytes if both are set.
|
|
10
12
|
* @property {number} [ttl=300000] - Time to live in milliseconds (5 minutes default)
|
|
11
13
|
* @property {boolean} [enableStats=false] - Whether to track cache statistics (hits, misses, etc.)
|
|
12
14
|
* @property {string} [evictionPolicy='lru'] - Cache eviction policy: 'lru' (Least Recently Used) or 'fifo' (First In First Out)
|
|
@@ -70,9 +72,54 @@
|
|
|
70
72
|
* maxSize: 1000,
|
|
71
73
|
* ttl: 300000 // 5 minutes
|
|
72
74
|
* }
|
|
73
|
-
*
|
|
75
|
+
*
|
|
76
|
+
* @example
|
|
77
|
+
* // Memory-limited configuration (prevents memory exhaustion)
|
|
78
|
+
* {
|
|
79
|
+
* maxMemoryBytes: 100 * 1024 * 1024, // 100MB hard limit
|
|
80
|
+
* ttl: 600000, // 10 minutes
|
|
81
|
+
* enableCompression: true, // Reduce memory usage
|
|
82
|
+
* compressionThreshold: 1024 // Compress items > 1KB
|
|
83
|
+
* }
|
|
84
|
+
*
|
|
85
|
+
* @example
|
|
86
|
+
* // Production configuration with memory monitoring (absolute bytes)
|
|
87
|
+
* {
|
|
88
|
+
* maxSize: 5000, // Limit number of items
|
|
89
|
+
* maxMemoryBytes: 512 * 1024 * 1024, // 512MB memory limit
|
|
90
|
+
* ttl: 1800000, // 30 minutes
|
|
91
|
+
* enableCompression: true,
|
|
92
|
+
* compressionThreshold: 512
|
|
93
|
+
* }
|
|
94
|
+
*
|
|
95
|
+
* // Check memory usage
|
|
96
|
+
* const stats = cache.getMemoryStats();
|
|
97
|
+
* console.log(`Memory: ${stats.memoryUsage.current} / ${stats.memoryUsage.max}`);
|
|
98
|
+
* console.log(`Usage: ${stats.memoryUsagePercent}%`);
|
|
99
|
+
* console.log(`Evicted due to memory: ${stats.evictedDueToMemory}`);
|
|
100
|
+
*
|
|
101
|
+
* @example
|
|
102
|
+
* // Production configuration with percentage of system memory
|
|
103
|
+
* {
|
|
104
|
+
* maxMemoryPercent: 0.1, // Use max 10% of system memory (0.1 = 10%)
|
|
105
|
+
* ttl: 1800000, // 30 minutes
|
|
106
|
+
* enableCompression: true
|
|
107
|
+
* }
|
|
108
|
+
*
|
|
109
|
+
* // On a 16GB system, this sets maxMemoryBytes to ~1.6GB
|
|
110
|
+
* // On a 32GB system, this sets maxMemoryBytes to ~3.2GB
|
|
111
|
+
*
|
|
112
|
+
* // Check system memory stats
|
|
113
|
+
* const stats = cache.getMemoryStats();
|
|
114
|
+
* console.log(`System Memory: ${stats.systemMemory.total}`);
|
|
115
|
+
* console.log(`Cache using: ${stats.systemMemory.cachePercent} of system memory`);
|
|
116
|
+
* console.log(`Max allowed: ${(stats.maxMemoryPercent * 100).toFixed(1)}%`);
|
|
117
|
+
*
|
|
74
118
|
* @notes
|
|
75
|
-
* - Memory usage is limited by available RAM and
|
|
119
|
+
* - Memory usage is limited by available RAM, maxSize setting, and optionally maxMemoryBytes or maxMemoryPercent
|
|
120
|
+
* - maxMemoryPercent takes precedence over maxMemoryBytes if both are set
|
|
121
|
+
* - maxMemoryPercent is calculated based on total system memory at cache creation time
|
|
122
|
+
* - Useful for containerized/cloud environments where system memory varies
|
|
76
123
|
* - TTL is checked on access, not automatically in background
|
|
77
124
|
* - LRU eviction removes least recently accessed items when cache is full
|
|
78
125
|
* - FIFO eviction removes oldest items when cache is full
|
|
@@ -83,8 +130,12 @@
|
|
|
83
130
|
* - Cleanup interval helps prevent memory leaks from expired items
|
|
84
131
|
* - Tags are useful for cache invalidation and monitoring
|
|
85
132
|
* - Case sensitivity affects key matching and storage efficiency
|
|
133
|
+
* - maxMemoryBytes prevents memory exhaustion by enforcing byte-level limits
|
|
134
|
+
* - Memory tracking includes serialized data size (compressed or uncompressed)
|
|
135
|
+
* - getMemoryStats() includes systemMemory info for monitoring
|
|
86
136
|
*/
|
|
87
137
|
import zlib from 'node:zlib';
|
|
138
|
+
import os from 'node:os';
|
|
88
139
|
import { Cache } from "./cache.class.js"
|
|
89
140
|
|
|
90
141
|
export class MemoryCache extends Cache {
|
|
@@ -93,12 +144,39 @@ export class MemoryCache extends Cache {
|
|
|
93
144
|
this.cache = {};
|
|
94
145
|
this.meta = {};
|
|
95
146
|
this.maxSize = config.maxSize !== undefined ? config.maxSize : 1000;
|
|
147
|
+
|
|
148
|
+
// Validate that only one memory limit option is used
|
|
149
|
+
if (config.maxMemoryBytes && config.maxMemoryBytes > 0 &&
|
|
150
|
+
config.maxMemoryPercent && config.maxMemoryPercent > 0) {
|
|
151
|
+
throw new Error(
|
|
152
|
+
'[MemoryCache] Cannot use both maxMemoryBytes and maxMemoryPercent. ' +
|
|
153
|
+
'Choose one: maxMemoryBytes (absolute) or maxMemoryPercent (0...1 fraction).'
|
|
154
|
+
);
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
// Calculate maxMemoryBytes from percentage if provided
|
|
158
|
+
if (config.maxMemoryPercent && config.maxMemoryPercent > 0) {
|
|
159
|
+
if (config.maxMemoryPercent > 1) {
|
|
160
|
+
throw new Error(
|
|
161
|
+
'[MemoryCache] maxMemoryPercent must be between 0 and 1 (e.g., 0.1 for 10%). ' +
|
|
162
|
+
`Received: ${config.maxMemoryPercent}`
|
|
163
|
+
);
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
const totalMemory = os.totalmem();
|
|
167
|
+
this.maxMemoryBytes = Math.floor(totalMemory * config.maxMemoryPercent);
|
|
168
|
+
this.maxMemoryPercent = config.maxMemoryPercent;
|
|
169
|
+
} else {
|
|
170
|
+
this.maxMemoryBytes = config.maxMemoryBytes !== undefined ? config.maxMemoryBytes : 0; // 0 = unlimited
|
|
171
|
+
this.maxMemoryPercent = 0;
|
|
172
|
+
}
|
|
173
|
+
|
|
96
174
|
this.ttl = config.ttl !== undefined ? config.ttl : 300000;
|
|
97
|
-
|
|
175
|
+
|
|
98
176
|
// Compression configuration
|
|
99
177
|
this.enableCompression = config.enableCompression !== undefined ? config.enableCompression : false;
|
|
100
178
|
this.compressionThreshold = config.compressionThreshold !== undefined ? config.compressionThreshold : 1024;
|
|
101
|
-
|
|
179
|
+
|
|
102
180
|
// Stats for compression
|
|
103
181
|
this.compressionStats = {
|
|
104
182
|
totalCompressed: 0,
|
|
@@ -106,33 +184,26 @@ export class MemoryCache extends Cache {
|
|
|
106
184
|
totalCompressedSize: 0,
|
|
107
185
|
compressionRatio: 0
|
|
108
186
|
};
|
|
187
|
+
|
|
188
|
+
// Memory tracking
|
|
189
|
+
this.currentMemoryBytes = 0;
|
|
190
|
+
this.evictedDueToMemory = 0;
|
|
109
191
|
}
|
|
110
192
|
|
|
111
193
|
async _set(key, data) {
|
|
112
|
-
// Limpar se exceder maxSize
|
|
113
|
-
if (this.maxSize > 0 && Object.keys(this.cache).length >= this.maxSize) {
|
|
114
|
-
// Remove o item mais antigo
|
|
115
|
-
const oldestKey = Object.entries(this.meta)
|
|
116
|
-
.sort((a, b) => a[1].ts - b[1].ts)[0]?.[0];
|
|
117
|
-
if (oldestKey) {
|
|
118
|
-
delete this.cache[oldestKey];
|
|
119
|
-
delete this.meta[oldestKey];
|
|
120
|
-
}
|
|
121
|
-
}
|
|
122
|
-
|
|
123
194
|
// Prepare data for storage
|
|
124
195
|
let finalData = data;
|
|
125
196
|
let compressed = false;
|
|
126
197
|
let originalSize = 0;
|
|
127
198
|
let compressedSize = 0;
|
|
128
|
-
|
|
199
|
+
|
|
200
|
+
// Calculate size first (needed for both compression and memory limit checks)
|
|
201
|
+
const serialized = JSON.stringify(data);
|
|
202
|
+
originalSize = Buffer.byteLength(serialized, 'utf8');
|
|
203
|
+
|
|
129
204
|
// Apply compression if enabled
|
|
130
205
|
if (this.enableCompression) {
|
|
131
206
|
try {
|
|
132
|
-
// Serialize data to measure size
|
|
133
|
-
const serialized = JSON.stringify(data);
|
|
134
|
-
originalSize = Buffer.byteLength(serialized, 'utf8');
|
|
135
|
-
|
|
136
207
|
// Compress only if over threshold
|
|
137
208
|
if (originalSize >= this.compressionThreshold) {
|
|
138
209
|
const compressedBuffer = zlib.gzipSync(Buffer.from(serialized, 'utf8'));
|
|
@@ -143,12 +214,12 @@ export class MemoryCache extends Cache {
|
|
|
143
214
|
};
|
|
144
215
|
compressedSize = Buffer.byteLength(finalData.__data, 'utf8');
|
|
145
216
|
compressed = true;
|
|
146
|
-
|
|
217
|
+
|
|
147
218
|
// Update compression stats
|
|
148
219
|
this.compressionStats.totalCompressed++;
|
|
149
220
|
this.compressionStats.totalOriginalSize += originalSize;
|
|
150
221
|
this.compressionStats.totalCompressedSize += compressedSize;
|
|
151
|
-
this.compressionStats.compressionRatio =
|
|
222
|
+
this.compressionStats.compressionRatio =
|
|
152
223
|
(this.compressionStats.totalCompressedSize / this.compressionStats.totalOriginalSize).toFixed(2);
|
|
153
224
|
}
|
|
154
225
|
} catch (error) {
|
|
@@ -156,33 +227,79 @@ export class MemoryCache extends Cache {
|
|
|
156
227
|
console.warn(`[MemoryCache] Compression failed for key '${key}':`, error.message);
|
|
157
228
|
}
|
|
158
229
|
}
|
|
159
|
-
|
|
230
|
+
|
|
231
|
+
// Calculate actual storage size (compressed or original)
|
|
232
|
+
const itemSize = compressed ? compressedSize : originalSize;
|
|
233
|
+
|
|
234
|
+
// If replacing existing key, subtract its old size from current memory
|
|
235
|
+
if (Object.prototype.hasOwnProperty.call(this.cache, key)) {
|
|
236
|
+
const oldSize = this.meta[key]?.compressedSize || 0;
|
|
237
|
+
this.currentMemoryBytes -= oldSize;
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
// Memory-aware eviction: Remove items until we have space
|
|
241
|
+
if (this.maxMemoryBytes > 0) {
|
|
242
|
+
while (this.currentMemoryBytes + itemSize > this.maxMemoryBytes && Object.keys(this.cache).length > 0) {
|
|
243
|
+
// Remove the oldest item
|
|
244
|
+
const oldestKey = Object.entries(this.meta)
|
|
245
|
+
.sort((a, b) => a[1].ts - b[1].ts)[0]?.[0];
|
|
246
|
+
if (oldestKey) {
|
|
247
|
+
const evictedSize = this.meta[oldestKey]?.compressedSize || 0;
|
|
248
|
+
delete this.cache[oldestKey];
|
|
249
|
+
delete this.meta[oldestKey];
|
|
250
|
+
this.currentMemoryBytes -= evictedSize;
|
|
251
|
+
this.evictedDueToMemory++;
|
|
252
|
+
} else {
|
|
253
|
+
break; // No more items to evict
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
// Item count eviction (original logic)
|
|
259
|
+
if (this.maxSize > 0 && Object.keys(this.cache).length >= this.maxSize) {
|
|
260
|
+
// Remove o item mais antigo
|
|
261
|
+
const oldestKey = Object.entries(this.meta)
|
|
262
|
+
.sort((a, b) => a[1].ts - b[1].ts)[0]?.[0];
|
|
263
|
+
if (oldestKey) {
|
|
264
|
+
const evictedSize = this.meta[oldestKey]?.compressedSize || 0;
|
|
265
|
+
delete this.cache[oldestKey];
|
|
266
|
+
delete this.meta[oldestKey];
|
|
267
|
+
this.currentMemoryBytes -= evictedSize;
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
// Store the item
|
|
160
272
|
this.cache[key] = finalData;
|
|
161
|
-
this.meta[key] = {
|
|
273
|
+
this.meta[key] = {
|
|
162
274
|
ts: Date.now(),
|
|
163
275
|
compressed,
|
|
164
276
|
originalSize,
|
|
165
|
-
compressedSize:
|
|
277
|
+
compressedSize: itemSize
|
|
166
278
|
};
|
|
167
|
-
|
|
279
|
+
|
|
280
|
+
// Update current memory usage
|
|
281
|
+
this.currentMemoryBytes += itemSize;
|
|
282
|
+
|
|
168
283
|
return data;
|
|
169
284
|
}
|
|
170
285
|
|
|
171
286
|
async _get(key) {
|
|
172
287
|
if (!Object.prototype.hasOwnProperty.call(this.cache, key)) return null;
|
|
173
|
-
|
|
288
|
+
|
|
174
289
|
// Check TTL expiration
|
|
175
290
|
if (this.ttl > 0) {
|
|
176
291
|
const now = Date.now();
|
|
177
292
|
const meta = this.meta[key];
|
|
178
|
-
if (meta && now - meta.ts > this.ttl
|
|
179
|
-
//
|
|
293
|
+
if (meta && now - meta.ts > this.ttl) {
|
|
294
|
+
// Expired - decrement memory before deleting
|
|
295
|
+
const itemSize = meta.compressedSize || 0;
|
|
296
|
+
this.currentMemoryBytes -= itemSize;
|
|
180
297
|
delete this.cache[key];
|
|
181
298
|
delete this.meta[key];
|
|
182
299
|
return null;
|
|
183
300
|
}
|
|
184
301
|
}
|
|
185
|
-
|
|
302
|
+
|
|
186
303
|
const rawData = this.cache[key];
|
|
187
304
|
|
|
188
305
|
// Check if data is compressed
|
|
@@ -206,6 +323,12 @@ export class MemoryCache extends Cache {
|
|
|
206
323
|
}
|
|
207
324
|
|
|
208
325
|
async _del(key) {
|
|
326
|
+
// Decrement memory usage
|
|
327
|
+
if (Object.prototype.hasOwnProperty.call(this.cache, key)) {
|
|
328
|
+
const itemSize = this.meta[key]?.compressedSize || 0;
|
|
329
|
+
this.currentMemoryBytes -= itemSize;
|
|
330
|
+
}
|
|
331
|
+
|
|
209
332
|
delete this.cache[key];
|
|
210
333
|
delete this.meta[key];
|
|
211
334
|
return true;
|
|
@@ -215,6 +338,7 @@ export class MemoryCache extends Cache {
|
|
|
215
338
|
if (!prefix) {
|
|
216
339
|
this.cache = {};
|
|
217
340
|
this.meta = {};
|
|
341
|
+
this.currentMemoryBytes = 0; // Reset memory counter
|
|
218
342
|
return true;
|
|
219
343
|
}
|
|
220
344
|
// Remove only keys that start with the prefix
|
|
@@ -222,6 +346,9 @@ export class MemoryCache extends Cache {
|
|
|
222
346
|
for (const key of Object.keys(this.cache)) {
|
|
223
347
|
if (key.startsWith(prefix)) {
|
|
224
348
|
removed.push(key);
|
|
349
|
+
// Decrement memory usage
|
|
350
|
+
const itemSize = this.meta[key]?.compressedSize || 0;
|
|
351
|
+
this.currentMemoryBytes -= itemSize;
|
|
225
352
|
delete this.cache[key];
|
|
226
353
|
delete this.meta[key];
|
|
227
354
|
}
|
|
@@ -248,7 +375,7 @@ export class MemoryCache extends Cache {
|
|
|
248
375
|
return { enabled: false, message: 'Compression is disabled' };
|
|
249
376
|
}
|
|
250
377
|
|
|
251
|
-
const spaceSavings = this.compressionStats.totalOriginalSize > 0
|
|
378
|
+
const spaceSavings = this.compressionStats.totalOriginalSize > 0
|
|
252
379
|
? ((this.compressionStats.totalOriginalSize - this.compressionStats.totalCompressedSize) / this.compressionStats.totalOriginalSize * 100).toFixed(2)
|
|
253
380
|
: 0;
|
|
254
381
|
|
|
@@ -268,6 +395,62 @@ export class MemoryCache extends Cache {
|
|
|
268
395
|
}
|
|
269
396
|
};
|
|
270
397
|
}
|
|
398
|
+
|
|
399
|
+
/**
|
|
400
|
+
* Get memory usage statistics
|
|
401
|
+
* @returns {Object} Memory stats including current usage, limits, and eviction counts
|
|
402
|
+
*/
|
|
403
|
+
getMemoryStats() {
|
|
404
|
+
const totalItems = Object.keys(this.cache).length;
|
|
405
|
+
const memoryUsagePercent = this.maxMemoryBytes > 0
|
|
406
|
+
? ((this.currentMemoryBytes / this.maxMemoryBytes) * 100).toFixed(2)
|
|
407
|
+
: 0;
|
|
408
|
+
|
|
409
|
+
const systemMemory = {
|
|
410
|
+
total: os.totalmem(),
|
|
411
|
+
free: os.freemem(),
|
|
412
|
+
used: os.totalmem() - os.freemem()
|
|
413
|
+
};
|
|
414
|
+
|
|
415
|
+
const cachePercentOfTotal = systemMemory.total > 0
|
|
416
|
+
? ((this.currentMemoryBytes / systemMemory.total) * 100).toFixed(2)
|
|
417
|
+
: 0;
|
|
418
|
+
|
|
419
|
+
return {
|
|
420
|
+
currentMemoryBytes: this.currentMemoryBytes,
|
|
421
|
+
maxMemoryBytes: this.maxMemoryBytes,
|
|
422
|
+
maxMemoryPercent: this.maxMemoryPercent,
|
|
423
|
+
memoryUsagePercent: parseFloat(memoryUsagePercent),
|
|
424
|
+
cachePercentOfSystemMemory: parseFloat(cachePercentOfTotal),
|
|
425
|
+
totalItems,
|
|
426
|
+
maxSize: this.maxSize,
|
|
427
|
+
evictedDueToMemory: this.evictedDueToMemory,
|
|
428
|
+
averageItemSize: totalItems > 0 ? Math.round(this.currentMemoryBytes / totalItems) : 0,
|
|
429
|
+
memoryUsage: {
|
|
430
|
+
current: this._formatBytes(this.currentMemoryBytes),
|
|
431
|
+
max: this.maxMemoryBytes > 0 ? this._formatBytes(this.maxMemoryBytes) : 'unlimited',
|
|
432
|
+
available: this.maxMemoryBytes > 0 ? this._formatBytes(this.maxMemoryBytes - this.currentMemoryBytes) : 'unlimited'
|
|
433
|
+
},
|
|
434
|
+
systemMemory: {
|
|
435
|
+
total: this._formatBytes(systemMemory.total),
|
|
436
|
+
free: this._formatBytes(systemMemory.free),
|
|
437
|
+
used: this._formatBytes(systemMemory.used),
|
|
438
|
+
cachePercent: `${cachePercentOfTotal}%`
|
|
439
|
+
}
|
|
440
|
+
};
|
|
441
|
+
}
|
|
442
|
+
|
|
443
|
+
/**
|
|
444
|
+
* Format bytes to human-readable format
|
|
445
|
+
* @private
|
|
446
|
+
*/
|
|
447
|
+
_formatBytes(bytes) {
|
|
448
|
+
if (bytes === 0) return '0 B';
|
|
449
|
+
const k = 1024;
|
|
450
|
+
const sizes = ['B', 'KB', 'MB', 'GB'];
|
|
451
|
+
const i = Math.floor(Math.log(bytes) / Math.log(k));
|
|
452
|
+
return `${(bytes / Math.pow(k, i)).toFixed(2)} ${sizes[i]}`;
|
|
453
|
+
}
|
|
271
454
|
}
|
|
272
455
|
|
|
273
456
|
export default MemoryCache
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import { S3dbError } from '../errors.js';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* CacheError - Errors related to cache operations
|
|
5
|
+
*
|
|
6
|
+
* Used for cache operations including:
|
|
7
|
+
* - Cache driver initialization and setup
|
|
8
|
+
* - Cache get/set/delete operations
|
|
9
|
+
* - Cache invalidation and warming
|
|
10
|
+
* - Driver-specific operations (memory, filesystem, S3)
|
|
11
|
+
* - Resource-level caching
|
|
12
|
+
*
|
|
13
|
+
* @extends S3dbError
|
|
14
|
+
*/
|
|
15
|
+
export class CacheError extends S3dbError {
|
|
16
|
+
constructor(message, details = {}) {
|
|
17
|
+
const { driver = 'unknown', operation = 'unknown', resourceName, key, ...rest } = details;
|
|
18
|
+
|
|
19
|
+
let description = details.description;
|
|
20
|
+
if (!description) {
|
|
21
|
+
description = `
|
|
22
|
+
Cache Operation Error
|
|
23
|
+
|
|
24
|
+
Driver: ${driver}
|
|
25
|
+
Operation: ${operation}
|
|
26
|
+
${resourceName ? `Resource: ${resourceName}` : ''}
|
|
27
|
+
${key ? `Key: ${key}` : ''}
|
|
28
|
+
|
|
29
|
+
Common causes:
|
|
30
|
+
1. Invalid cache key format
|
|
31
|
+
2. Cache driver not properly initialized
|
|
32
|
+
3. Resource not found or not cached
|
|
33
|
+
4. Memory limits exceeded
|
|
34
|
+
5. Filesystem permissions issues
|
|
35
|
+
|
|
36
|
+
Solution:
|
|
37
|
+
Check cache configuration and ensure the cache driver is properly initialized.
|
|
38
|
+
|
|
39
|
+
Docs: https://github.com/forattini-dev/s3db.js/blob/main/docs/plugins/cache.md
|
|
40
|
+
`.trim();
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
super(message, { ...rest, driver, operation, resourceName, key, description });
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
export default CacheError;
|
|
@@ -8,7 +8,90 @@ import MemoryCache from "./cache/memory-cache.class.js";
|
|
|
8
8
|
import { FilesystemCache } from "./cache/filesystem-cache.class.js";
|
|
9
9
|
import { PartitionAwareFilesystemCache } from "./cache/partition-aware-filesystem-cache.class.js";
|
|
10
10
|
import tryFn from "../concerns/try-fn.js";
|
|
11
|
-
|
|
11
|
+
import { CacheError } from "./cache.errors.js";
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* Cache Plugin Configuration
|
|
15
|
+
*
|
|
16
|
+
* Provides caching layer for S3DB resources with multiple backend options.
|
|
17
|
+
* Automatically caches read operations and invalidates on writes.
|
|
18
|
+
*
|
|
19
|
+
* @typedef {Object} CachePluginOptions
|
|
20
|
+
* @property {string} [driver='s3'] - Cache driver: 'memory', 'filesystem', 's3', or custom driver instance
|
|
21
|
+
* @property {number} [ttl] - Time to live in milliseconds for cached items (shortcut for config.ttl)
|
|
22
|
+
* @property {number} [maxSize] - Maximum number of items to cache (shortcut for config.maxSize)
|
|
23
|
+
* @property {number} [maxMemoryBytes] - (MemoryCache only) Maximum memory in bytes (shortcut for config.maxMemoryBytes). Cannot be used with maxMemoryPercent.
|
|
24
|
+
* @property {number} [maxMemoryPercent] - (MemoryCache only) Maximum memory as fraction 0...1 (shortcut for config.maxMemoryPercent). Cannot be used with maxMemoryBytes.
|
|
25
|
+
*
|
|
26
|
+
* @property {Array<string>} [include] - Only cache these resource names (null = cache all)
|
|
27
|
+
* @property {Array<string>} [exclude=[]] - Never cache these resource names
|
|
28
|
+
*
|
|
29
|
+
* @property {boolean} [includePartitions=true] - Whether to cache partition queries
|
|
30
|
+
* @property {string} [partitionStrategy='hierarchical'] - Partition caching strategy
|
|
31
|
+
* @property {boolean} [partitionAware=true] - Use partition-aware filesystem cache
|
|
32
|
+
* @property {boolean} [trackUsage=true] - Track cache usage statistics
|
|
33
|
+
* @property {boolean} [preloadRelated=true] - Preload related partitions
|
|
34
|
+
*
|
|
35
|
+
* @property {number} [retryAttempts=3] - Number of retry attempts for cache operations
|
|
36
|
+
* @property {number} [retryDelay=100] - Delay between retries in milliseconds
|
|
37
|
+
* @property {boolean} [verbose=false] - Enable verbose logging
|
|
38
|
+
*
|
|
39
|
+
* @property {Object} [config] - Driver-specific configuration (can override top-level ttl, maxSize, maxMemoryBytes, maxMemoryPercent)
|
|
40
|
+
* @property {number} [config.ttl] - Override TTL for this driver
|
|
41
|
+
* @property {number} [config.maxSize] - Override max number of items
|
|
42
|
+
* @property {number} [config.maxMemoryBytes] - (MemoryCache only) Maximum memory in bytes. Cannot be used with config.maxMemoryPercent.
|
|
43
|
+
* @property {number} [config.maxMemoryPercent] - (MemoryCache only) Maximum memory as fraction 0...1 (e.g., 0.1 = 10%). Cannot be used with config.maxMemoryBytes.
|
|
44
|
+
* @property {boolean} [config.enableCompression] - (MemoryCache only) Enable gzip compression
|
|
45
|
+
* @property {number} [config.compressionThreshold=1024] - (MemoryCache only) Minimum size in bytes to trigger compression
|
|
46
|
+
*
|
|
47
|
+
* @example
|
|
48
|
+
* // Memory cache with absolute byte limit
|
|
49
|
+
* new CachePlugin({
|
|
50
|
+
* driver: 'memory',
|
|
51
|
+
* maxMemoryBytes: 512 * 1024 * 1024, // 512MB
|
|
52
|
+
* ttl: 600000 // 10 minutes
|
|
53
|
+
* })
|
|
54
|
+
*
|
|
55
|
+
* @example
|
|
56
|
+
* // Memory cache with percentage limit (cloud-native)
|
|
57
|
+
* new CachePlugin({
|
|
58
|
+
* driver: 'memory',
|
|
59
|
+
* maxMemoryPercent: 0.1, // 10% of system memory
|
|
60
|
+
* ttl: 1800000 // 30 minutes
|
|
61
|
+
* })
|
|
62
|
+
*
|
|
63
|
+
* @example
|
|
64
|
+
* // Filesystem cache with partition awareness
|
|
65
|
+
* new CachePlugin({
|
|
66
|
+
* driver: 'filesystem',
|
|
67
|
+
* partitionAware: true,
|
|
68
|
+
* includePartitions: true,
|
|
69
|
+
* ttl: 3600000 // 1 hour
|
|
70
|
+
* })
|
|
71
|
+
*
|
|
72
|
+
* @example
|
|
73
|
+
* // S3 cache (default)
|
|
74
|
+
* new CachePlugin({
|
|
75
|
+
* driver: 's3',
|
|
76
|
+
* ttl: 7200000 // 2 hours
|
|
77
|
+
* })
|
|
78
|
+
*
|
|
79
|
+
* @example
|
|
80
|
+
* // Selective caching
|
|
81
|
+
* new CachePlugin({
|
|
82
|
+
* driver: 'memory',
|
|
83
|
+
* include: ['users', 'products'], // Only cache these
|
|
84
|
+
* exclude: ['audit_logs'], // Never cache these
|
|
85
|
+
* maxMemoryPercent: 0.15
|
|
86
|
+
* })
|
|
87
|
+
*
|
|
88
|
+
* @notes
|
|
89
|
+
* - maxMemoryBytes and maxMemoryPercent are mutually exclusive (throws error if both set)
|
|
90
|
+
* - maxMemoryPercent is recommended for containerized/cloud environments
|
|
91
|
+
* - Plugin-created resources (createdBy !== 'user') are skipped by default
|
|
92
|
+
* - Cache is automatically invalidated on insert/update/delete operations
|
|
93
|
+
* - Use skipCache: true option on queries to bypass cache for specific calls
|
|
94
|
+
*/
|
|
12
95
|
export class CachePlugin extends Plugin {
|
|
13
96
|
constructor(options = {}) {
|
|
14
97
|
super(options);
|
|
@@ -20,7 +103,9 @@ export class CachePlugin extends Plugin {
|
|
|
20
103
|
config: {
|
|
21
104
|
ttl: options.ttl,
|
|
22
105
|
maxSize: options.maxSize,
|
|
23
|
-
|
|
106
|
+
maxMemoryBytes: options.maxMemoryBytes,
|
|
107
|
+
maxMemoryPercent: options.maxMemoryPercent,
|
|
108
|
+
...options.config // Driver-specific config (can override ttl/maxSize/maxMemoryBytes/maxMemoryPercent)
|
|
24
109
|
},
|
|
25
110
|
|
|
26
111
|
// Resource filtering
|
|
@@ -471,7 +556,13 @@ export class CachePlugin extends Plugin {
|
|
|
471
556
|
async warmCache(resourceName, options = {}) {
|
|
472
557
|
const resource = this.database.resources[resourceName];
|
|
473
558
|
if (!resource) {
|
|
474
|
-
throw new
|
|
559
|
+
throw new CacheError('Resource not found for cache warming', {
|
|
560
|
+
operation: 'warmCache',
|
|
561
|
+
driver: this.driver?.constructor.name,
|
|
562
|
+
resourceName,
|
|
563
|
+
availableResources: Object.keys(this.database.resources),
|
|
564
|
+
suggestion: 'Check resource name spelling or ensure resource has been created'
|
|
565
|
+
});
|
|
475
566
|
}
|
|
476
567
|
|
|
477
568
|
const { includePartitions = true, sampleSize = 100 } = options;
|
|
@@ -1209,3 +1209,148 @@ export async function getLastNMonths(resourceName, field, months = 12, options,
|
|
|
1209
1209
|
|
|
1210
1210
|
return data;
|
|
1211
1211
|
}
|
|
1212
|
+
|
|
1213
|
+
/**
|
|
1214
|
+
* Get raw transaction events for custom aggregation
|
|
1215
|
+
*
|
|
1216
|
+
* This method provides direct access to the underlying transaction events,
|
|
1217
|
+
* allowing developers to perform custom aggregations beyond the pre-built analytics.
|
|
1218
|
+
* Useful for complex queries, custom metrics, or when you need the raw event data.
|
|
1219
|
+
*
|
|
1220
|
+
* @param {string} resourceName - Resource name
|
|
1221
|
+
* @param {string} field - Field name
|
|
1222
|
+
* @param {Object} options - Query options
|
|
1223
|
+
* @param {string} options.recordId - Filter by specific record ID
|
|
1224
|
+
* @param {string} options.startDate - Start date filter (YYYY-MM-DD or YYYY-MM-DDTHH)
|
|
1225
|
+
* @param {string} options.endDate - End date filter (YYYY-MM-DD or YYYY-MM-DDTHH)
|
|
1226
|
+
* @param {string} options.cohortDate - Filter by cohort date (YYYY-MM-DD)
|
|
1227
|
+
* @param {string} options.cohortHour - Filter by cohort hour (YYYY-MM-DDTHH)
|
|
1228
|
+
* @param {string} options.cohortMonth - Filter by cohort month (YYYY-MM)
|
|
1229
|
+
* @param {boolean} options.applied - Filter by applied status (true/false/undefined for both)
|
|
1230
|
+
* @param {string} options.operation - Filter by operation type ('add', 'sub', 'set')
|
|
1231
|
+
* @param {number} options.limit - Maximum number of events to return
|
|
1232
|
+
* @param {Object} fieldHandlers - Field handlers map
|
|
1233
|
+
* @returns {Promise<Array>} Raw transaction events
|
|
1234
|
+
*
|
|
1235
|
+
* @example
|
|
1236
|
+
* // Get all events for a specific record
|
|
1237
|
+
* const events = await plugin.getRawEvents('wallets', 'balance', {
|
|
1238
|
+
* recordId: 'wallet1'
|
|
1239
|
+
* });
|
|
1240
|
+
*
|
|
1241
|
+
* @example
|
|
1242
|
+
* // Get events for a specific time range
|
|
1243
|
+
* const events = await plugin.getRawEvents('wallets', 'balance', {
|
|
1244
|
+
* startDate: '2025-10-01',
|
|
1245
|
+
* endDate: '2025-10-31'
|
|
1246
|
+
* });
|
|
1247
|
+
*
|
|
1248
|
+
* @example
|
|
1249
|
+
* // Get only pending (unapplied) transactions
|
|
1250
|
+
* const pending = await plugin.getRawEvents('wallets', 'balance', {
|
|
1251
|
+
* applied: false
|
|
1252
|
+
* });
|
|
1253
|
+
*/
|
|
1254
|
+
export async function getRawEvents(resourceName, field, options, fieldHandlers) {
|
|
1255
|
+
// Get handler for this resource/field combination
|
|
1256
|
+
const resourceHandlers = fieldHandlers.get(resourceName);
|
|
1257
|
+
if (!resourceHandlers) {
|
|
1258
|
+
throw new Error(`No eventual consistency configured for resource: ${resourceName}`);
|
|
1259
|
+
}
|
|
1260
|
+
|
|
1261
|
+
const handler = resourceHandlers.get(field);
|
|
1262
|
+
if (!handler) {
|
|
1263
|
+
throw new Error(`No eventual consistency configured for field: ${resourceName}.${field}`);
|
|
1264
|
+
}
|
|
1265
|
+
|
|
1266
|
+
if (!handler.transactionResource) {
|
|
1267
|
+
throw new Error('Transaction resource not initialized');
|
|
1268
|
+
}
|
|
1269
|
+
|
|
1270
|
+
const {
|
|
1271
|
+
recordId,
|
|
1272
|
+
startDate,
|
|
1273
|
+
endDate,
|
|
1274
|
+
cohortDate,
|
|
1275
|
+
cohortHour,
|
|
1276
|
+
cohortMonth,
|
|
1277
|
+
applied,
|
|
1278
|
+
operation,
|
|
1279
|
+
limit
|
|
1280
|
+
} = options;
|
|
1281
|
+
|
|
1282
|
+
// Build query object for partition-based filtering
|
|
1283
|
+
const query = {};
|
|
1284
|
+
|
|
1285
|
+
// Filter by recordId (uses partition if available)
|
|
1286
|
+
if (recordId !== undefined) {
|
|
1287
|
+
query.originalId = recordId;
|
|
1288
|
+
}
|
|
1289
|
+
|
|
1290
|
+
// Filter by applied status (uses partition)
|
|
1291
|
+
if (applied !== undefined) {
|
|
1292
|
+
query.applied = applied;
|
|
1293
|
+
}
|
|
1294
|
+
|
|
1295
|
+
// Fetch transactions using partition-aware query
|
|
1296
|
+
const [ok, err, allTransactions] = await tryFn(() =>
|
|
1297
|
+
handler.transactionResource.query(query)
|
|
1298
|
+
);
|
|
1299
|
+
|
|
1300
|
+
if (!ok || !allTransactions) {
|
|
1301
|
+
return [];
|
|
1302
|
+
}
|
|
1303
|
+
|
|
1304
|
+
// Ensure all transactions have cohort fields
|
|
1305
|
+
let filtered = allTransactions;
|
|
1306
|
+
|
|
1307
|
+
// Filter by operation type
|
|
1308
|
+
if (operation !== undefined) {
|
|
1309
|
+
filtered = filtered.filter(t => t.operation === operation);
|
|
1310
|
+
}
|
|
1311
|
+
|
|
1312
|
+
// Filter by temporal fields (these are in-memory filters after partition query)
|
|
1313
|
+
if (cohortDate) {
|
|
1314
|
+
filtered = filtered.filter(t => t.cohortDate === cohortDate);
|
|
1315
|
+
}
|
|
1316
|
+
|
|
1317
|
+
if (cohortHour) {
|
|
1318
|
+
filtered = filtered.filter(t => t.cohortHour === cohortHour);
|
|
1319
|
+
}
|
|
1320
|
+
|
|
1321
|
+
if (cohortMonth) {
|
|
1322
|
+
filtered = filtered.filter(t => t.cohortMonth === cohortMonth);
|
|
1323
|
+
}
|
|
1324
|
+
|
|
1325
|
+
if (startDate && endDate) {
|
|
1326
|
+
// Determine which cohort field to use based on format
|
|
1327
|
+
const isHourly = startDate.length > 10; // YYYY-MM-DDTHH vs YYYY-MM-DD
|
|
1328
|
+
const cohortField = isHourly ? 'cohortHour' : 'cohortDate';
|
|
1329
|
+
|
|
1330
|
+
filtered = filtered.filter(t =>
|
|
1331
|
+
t[cohortField] && t[cohortField] >= startDate && t[cohortField] <= endDate
|
|
1332
|
+
);
|
|
1333
|
+
} else if (startDate) {
|
|
1334
|
+
const isHourly = startDate.length > 10;
|
|
1335
|
+
const cohortField = isHourly ? 'cohortHour' : 'cohortDate';
|
|
1336
|
+
filtered = filtered.filter(t => t[cohortField] && t[cohortField] >= startDate);
|
|
1337
|
+
} else if (endDate) {
|
|
1338
|
+
const isHourly = endDate.length > 10;
|
|
1339
|
+
const cohortField = isHourly ? 'cohortHour' : 'cohortDate';
|
|
1340
|
+
filtered = filtered.filter(t => t[cohortField] && t[cohortField] <= endDate);
|
|
1341
|
+
}
|
|
1342
|
+
|
|
1343
|
+
// Sort by timestamp (newest first by default)
|
|
1344
|
+
filtered.sort((a, b) => {
|
|
1345
|
+
const aTime = new Date(a.timestamp || a.createdAt).getTime();
|
|
1346
|
+
const bTime = new Date(b.timestamp || b.createdAt).getTime();
|
|
1347
|
+
return bTime - aTime;
|
|
1348
|
+
});
|
|
1349
|
+
|
|
1350
|
+
// Apply limit
|
|
1351
|
+
if (limit && limit > 0) {
|
|
1352
|
+
filtered = filtered.slice(0, limit);
|
|
1353
|
+
}
|
|
1354
|
+
|
|
1355
|
+
return filtered;
|
|
1356
|
+
}
|