s3db.js 11.2.2 → 11.2.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/dist/s3db.cjs.js +1650 -136
  2. package/dist/s3db.cjs.js.map +1 -1
  3. package/dist/s3db.es.js +1644 -137
  4. package/dist/s3db.es.js.map +1 -1
  5. package/package.json +1 -1
  6. package/src/behaviors/enforce-limits.js +28 -4
  7. package/src/behaviors/index.js +6 -1
  8. package/src/client.class.js +11 -1
  9. package/src/concerns/partition-queue.js +7 -1
  10. package/src/concerns/plugin-storage.js +75 -13
  11. package/src/database.class.js +22 -4
  12. package/src/errors.js +414 -24
  13. package/src/partition-drivers/base-partition-driver.js +12 -2
  14. package/src/partition-drivers/index.js +7 -1
  15. package/src/partition-drivers/memory-partition-driver.js +20 -5
  16. package/src/partition-drivers/sqs-partition-driver.js +6 -1
  17. package/src/plugins/audit.errors.js +46 -0
  18. package/src/plugins/backup/base-backup-driver.class.js +36 -6
  19. package/src/plugins/backup/filesystem-backup-driver.class.js +55 -7
  20. package/src/plugins/backup/index.js +40 -9
  21. package/src/plugins/backup/multi-backup-driver.class.js +69 -9
  22. package/src/plugins/backup/s3-backup-driver.class.js +48 -6
  23. package/src/plugins/backup.errors.js +45 -0
  24. package/src/plugins/cache/cache.class.js +8 -1
  25. package/src/plugins/cache/memory-cache.class.js +216 -33
  26. package/src/plugins/cache.errors.js +47 -0
  27. package/src/plugins/cache.plugin.js +94 -3
  28. package/src/plugins/eventual-consistency/analytics.js +145 -0
  29. package/src/plugins/eventual-consistency/index.js +203 -1
  30. package/src/plugins/fulltext.errors.js +46 -0
  31. package/src/plugins/fulltext.plugin.js +15 -3
  32. package/src/plugins/metrics.errors.js +46 -0
  33. package/src/plugins/queue-consumer.plugin.js +31 -4
  34. package/src/plugins/queue.errors.js +46 -0
  35. package/src/plugins/replicator.errors.js +46 -0
  36. package/src/plugins/replicator.plugin.js +40 -5
  37. package/src/plugins/replicators/base-replicator.class.js +19 -3
  38. package/src/plugins/replicators/index.js +9 -3
  39. package/src/plugins/replicators/s3db-replicator.class.js +38 -8
  40. package/src/plugins/scheduler.errors.js +46 -0
  41. package/src/plugins/scheduler.plugin.js +79 -19
  42. package/src/plugins/state-machine.errors.js +47 -0
  43. package/src/plugins/state-machine.plugin.js +86 -17
  44. package/src/resource.class.js +8 -1
  45. package/src/stream/index.js +6 -1
  46. package/src/stream/resource-reader.class.js +6 -1
@@ -1,12 +1,14 @@
1
1
  /**
2
2
  * Memory Cache Configuration Documentation
3
- *
3
+ *
4
4
  * This cache implementation stores data in memory using a Map-like structure.
5
5
  * It provides fast access to frequently used data but is limited by available RAM
6
6
  * and data is lost when the process restarts.
7
- *
7
+ *
8
8
  * @typedef {Object} MemoryCacheConfig
9
9
  * @property {number} [maxSize=1000] - Maximum number of items to store in cache
10
+ * @property {number} [maxMemoryBytes=0] - Maximum memory usage in bytes (0 = unlimited). When set, cache will evict items to stay under this limit.
11
+ * @property {number} [maxMemoryPercent=0] - Maximum memory usage as decimal fraction of total system memory (0 = unlimited, 0.1 = 10%, 0.5 = 50%, 1.0 = 100%). Takes precedence over maxMemoryBytes if both are set.
10
12
  * @property {number} [ttl=300000] - Time to live in milliseconds (5 minutes default)
11
13
  * @property {boolean} [enableStats=false] - Whether to track cache statistics (hits, misses, etc.)
12
14
  * @property {string} [evictionPolicy='lru'] - Cache eviction policy: 'lru' (Least Recently Used) or 'fifo' (First In First Out)
@@ -70,9 +72,54 @@
70
72
  * maxSize: 1000,
71
73
  * ttl: 300000 // 5 minutes
72
74
  * }
73
- *
75
+ *
76
+ * @example
77
+ * // Memory-limited configuration (prevents memory exhaustion)
78
+ * {
79
+ * maxMemoryBytes: 100 * 1024 * 1024, // 100MB hard limit
80
+ * ttl: 600000, // 10 minutes
81
+ * enableCompression: true, // Reduce memory usage
82
+ * compressionThreshold: 1024 // Compress items > 1KB
83
+ * }
84
+ *
85
+ * @example
86
+ * // Production configuration with memory monitoring (absolute bytes)
87
+ * {
88
+ * maxSize: 5000, // Limit number of items
89
+ * maxMemoryBytes: 512 * 1024 * 1024, // 512MB memory limit
90
+ * ttl: 1800000, // 30 minutes
91
+ * enableCompression: true,
92
+ * compressionThreshold: 512
93
+ * }
94
+ *
95
+ * // Check memory usage
96
+ * const stats = cache.getMemoryStats();
97
+ * console.log(`Memory: ${stats.memoryUsage.current} / ${stats.memoryUsage.max}`);
98
+ * console.log(`Usage: ${stats.memoryUsagePercent}%`);
99
+ * console.log(`Evicted due to memory: ${stats.evictedDueToMemory}`);
100
+ *
101
+ * @example
102
+ * // Production configuration with percentage of system memory
103
+ * {
104
+ * maxMemoryPercent: 0.1, // Use max 10% of system memory (0.1 = 10%)
105
+ * ttl: 1800000, // 30 minutes
106
+ * enableCompression: true
107
+ * }
108
+ *
109
+ * // On a 16GB system, this sets maxMemoryBytes to ~1.6GB
110
+ * // On a 32GB system, this sets maxMemoryBytes to ~3.2GB
111
+ *
112
+ * // Check system memory stats
113
+ * const stats = cache.getMemoryStats();
114
+ * console.log(`System Memory: ${stats.systemMemory.total}`);
115
+ * console.log(`Cache using: ${stats.systemMemory.cachePercent} of system memory`);
116
+ * console.log(`Max allowed: ${(stats.maxMemoryPercent * 100).toFixed(1)}%`);
117
+ *
74
118
  * @notes
75
- * - Memory usage is limited by available RAM and maxSize setting
119
+ * - Memory usage is limited by available RAM, maxSize setting, and optionally maxMemoryBytes or maxMemoryPercent
120
+ * - maxMemoryPercent takes precedence over maxMemoryBytes if both are set
121
+ * - maxMemoryPercent is calculated based on total system memory at cache creation time
122
+ * - Useful for containerized/cloud environments where system memory varies
76
123
  * - TTL is checked on access, not automatically in background
77
124
  * - LRU eviction removes least recently accessed items when cache is full
78
125
  * - FIFO eviction removes oldest items when cache is full
@@ -83,8 +130,12 @@
83
130
  * - Cleanup interval helps prevent memory leaks from expired items
84
131
  * - Tags are useful for cache invalidation and monitoring
85
132
  * - Case sensitivity affects key matching and storage efficiency
133
+ * - maxMemoryBytes prevents memory exhaustion by enforcing byte-level limits
134
+ * - Memory tracking includes serialized data size (compressed or uncompressed)
135
+ * - getMemoryStats() includes systemMemory info for monitoring
86
136
  */
87
137
  import zlib from 'node:zlib';
138
+ import os from 'node:os';
88
139
  import { Cache } from "./cache.class.js"
89
140
 
90
141
  export class MemoryCache extends Cache {
@@ -93,12 +144,39 @@ export class MemoryCache extends Cache {
93
144
  this.cache = {};
94
145
  this.meta = {};
95
146
  this.maxSize = config.maxSize !== undefined ? config.maxSize : 1000;
147
+
148
+ // Validate that only one memory limit option is used
149
+ if (config.maxMemoryBytes && config.maxMemoryBytes > 0 &&
150
+ config.maxMemoryPercent && config.maxMemoryPercent > 0) {
151
+ throw new Error(
152
+ '[MemoryCache] Cannot use both maxMemoryBytes and maxMemoryPercent. ' +
153
+ 'Choose one: maxMemoryBytes (absolute) or maxMemoryPercent (0...1 fraction).'
154
+ );
155
+ }
156
+
157
+ // Calculate maxMemoryBytes from percentage if provided
158
+ if (config.maxMemoryPercent && config.maxMemoryPercent > 0) {
159
+ if (config.maxMemoryPercent > 1) {
160
+ throw new Error(
161
+ '[MemoryCache] maxMemoryPercent must be between 0 and 1 (e.g., 0.1 for 10%). ' +
162
+ `Received: ${config.maxMemoryPercent}`
163
+ );
164
+ }
165
+
166
+ const totalMemory = os.totalmem();
167
+ this.maxMemoryBytes = Math.floor(totalMemory * config.maxMemoryPercent);
168
+ this.maxMemoryPercent = config.maxMemoryPercent;
169
+ } else {
170
+ this.maxMemoryBytes = config.maxMemoryBytes !== undefined ? config.maxMemoryBytes : 0; // 0 = unlimited
171
+ this.maxMemoryPercent = 0;
172
+ }
173
+
96
174
  this.ttl = config.ttl !== undefined ? config.ttl : 300000;
97
-
175
+
98
176
  // Compression configuration
99
177
  this.enableCompression = config.enableCompression !== undefined ? config.enableCompression : false;
100
178
  this.compressionThreshold = config.compressionThreshold !== undefined ? config.compressionThreshold : 1024;
101
-
179
+
102
180
  // Stats for compression
103
181
  this.compressionStats = {
104
182
  totalCompressed: 0,
@@ -106,33 +184,26 @@ export class MemoryCache extends Cache {
106
184
  totalCompressedSize: 0,
107
185
  compressionRatio: 0
108
186
  };
187
+
188
+ // Memory tracking
189
+ this.currentMemoryBytes = 0;
190
+ this.evictedDueToMemory = 0;
109
191
  }
110
192
 
111
193
  async _set(key, data) {
112
- // Limpar se exceder maxSize
113
- if (this.maxSize > 0 && Object.keys(this.cache).length >= this.maxSize) {
114
- // Remove o item mais antigo
115
- const oldestKey = Object.entries(this.meta)
116
- .sort((a, b) => a[1].ts - b[1].ts)[0]?.[0];
117
- if (oldestKey) {
118
- delete this.cache[oldestKey];
119
- delete this.meta[oldestKey];
120
- }
121
- }
122
-
123
194
  // Prepare data for storage
124
195
  let finalData = data;
125
196
  let compressed = false;
126
197
  let originalSize = 0;
127
198
  let compressedSize = 0;
128
-
199
+
200
+ // Calculate size first (needed for both compression and memory limit checks)
201
+ const serialized = JSON.stringify(data);
202
+ originalSize = Buffer.byteLength(serialized, 'utf8');
203
+
129
204
  // Apply compression if enabled
130
205
  if (this.enableCompression) {
131
206
  try {
132
- // Serialize data to measure size
133
- const serialized = JSON.stringify(data);
134
- originalSize = Buffer.byteLength(serialized, 'utf8');
135
-
136
207
  // Compress only if over threshold
137
208
  if (originalSize >= this.compressionThreshold) {
138
209
  const compressedBuffer = zlib.gzipSync(Buffer.from(serialized, 'utf8'));
@@ -143,12 +214,12 @@ export class MemoryCache extends Cache {
143
214
  };
144
215
  compressedSize = Buffer.byteLength(finalData.__data, 'utf8');
145
216
  compressed = true;
146
-
217
+
147
218
  // Update compression stats
148
219
  this.compressionStats.totalCompressed++;
149
220
  this.compressionStats.totalOriginalSize += originalSize;
150
221
  this.compressionStats.totalCompressedSize += compressedSize;
151
- this.compressionStats.compressionRatio =
222
+ this.compressionStats.compressionRatio =
152
223
  (this.compressionStats.totalCompressedSize / this.compressionStats.totalOriginalSize).toFixed(2);
153
224
  }
154
225
  } catch (error) {
@@ -156,33 +227,79 @@ export class MemoryCache extends Cache {
156
227
  console.warn(`[MemoryCache] Compression failed for key '${key}':`, error.message);
157
228
  }
158
229
  }
159
-
230
+
231
+ // Calculate actual storage size (compressed or original)
232
+ const itemSize = compressed ? compressedSize : originalSize;
233
+
234
+ // If replacing existing key, subtract its old size from current memory
235
+ if (Object.prototype.hasOwnProperty.call(this.cache, key)) {
236
+ const oldSize = this.meta[key]?.compressedSize || 0;
237
+ this.currentMemoryBytes -= oldSize;
238
+ }
239
+
240
+ // Memory-aware eviction: Remove items until we have space
241
+ if (this.maxMemoryBytes > 0) {
242
+ while (this.currentMemoryBytes + itemSize > this.maxMemoryBytes && Object.keys(this.cache).length > 0) {
243
+ // Remove the oldest item
244
+ const oldestKey = Object.entries(this.meta)
245
+ .sort((a, b) => a[1].ts - b[1].ts)[0]?.[0];
246
+ if (oldestKey) {
247
+ const evictedSize = this.meta[oldestKey]?.compressedSize || 0;
248
+ delete this.cache[oldestKey];
249
+ delete this.meta[oldestKey];
250
+ this.currentMemoryBytes -= evictedSize;
251
+ this.evictedDueToMemory++;
252
+ } else {
253
+ break; // No more items to evict
254
+ }
255
+ }
256
+ }
257
+
258
+ // Item count eviction (original logic)
259
+ if (this.maxSize > 0 && Object.keys(this.cache).length >= this.maxSize) {
260
+ // Remove o item mais antigo
261
+ const oldestKey = Object.entries(this.meta)
262
+ .sort((a, b) => a[1].ts - b[1].ts)[0]?.[0];
263
+ if (oldestKey) {
264
+ const evictedSize = this.meta[oldestKey]?.compressedSize || 0;
265
+ delete this.cache[oldestKey];
266
+ delete this.meta[oldestKey];
267
+ this.currentMemoryBytes -= evictedSize;
268
+ }
269
+ }
270
+
271
+ // Store the item
160
272
  this.cache[key] = finalData;
161
- this.meta[key] = {
273
+ this.meta[key] = {
162
274
  ts: Date.now(),
163
275
  compressed,
164
276
  originalSize,
165
- compressedSize: compressed ? compressedSize : originalSize
277
+ compressedSize: itemSize
166
278
  };
167
-
279
+
280
+ // Update current memory usage
281
+ this.currentMemoryBytes += itemSize;
282
+
168
283
  return data;
169
284
  }
170
285
 
171
286
  async _get(key) {
172
287
  if (!Object.prototype.hasOwnProperty.call(this.cache, key)) return null;
173
-
288
+
174
289
  // Check TTL expiration
175
290
  if (this.ttl > 0) {
176
291
  const now = Date.now();
177
292
  const meta = this.meta[key];
178
- if (meta && now - meta.ts > this.ttl * 1000) {
179
- // Expirado
293
+ if (meta && now - meta.ts > this.ttl) {
294
+ // Expired - decrement memory before deleting
295
+ const itemSize = meta.compressedSize || 0;
296
+ this.currentMemoryBytes -= itemSize;
180
297
  delete this.cache[key];
181
298
  delete this.meta[key];
182
299
  return null;
183
300
  }
184
301
  }
185
-
302
+
186
303
  const rawData = this.cache[key];
187
304
 
188
305
  // Check if data is compressed
@@ -206,6 +323,12 @@ export class MemoryCache extends Cache {
206
323
  }
207
324
 
208
325
  async _del(key) {
326
+ // Decrement memory usage
327
+ if (Object.prototype.hasOwnProperty.call(this.cache, key)) {
328
+ const itemSize = this.meta[key]?.compressedSize || 0;
329
+ this.currentMemoryBytes -= itemSize;
330
+ }
331
+
209
332
  delete this.cache[key];
210
333
  delete this.meta[key];
211
334
  return true;
@@ -215,6 +338,7 @@ export class MemoryCache extends Cache {
215
338
  if (!prefix) {
216
339
  this.cache = {};
217
340
  this.meta = {};
341
+ this.currentMemoryBytes = 0; // Reset memory counter
218
342
  return true;
219
343
  }
220
344
  // Remove only keys that start with the prefix
@@ -222,6 +346,9 @@ export class MemoryCache extends Cache {
222
346
  for (const key of Object.keys(this.cache)) {
223
347
  if (key.startsWith(prefix)) {
224
348
  removed.push(key);
349
+ // Decrement memory usage
350
+ const itemSize = this.meta[key]?.compressedSize || 0;
351
+ this.currentMemoryBytes -= itemSize;
225
352
  delete this.cache[key];
226
353
  delete this.meta[key];
227
354
  }
@@ -248,7 +375,7 @@ export class MemoryCache extends Cache {
248
375
  return { enabled: false, message: 'Compression is disabled' };
249
376
  }
250
377
 
251
- const spaceSavings = this.compressionStats.totalOriginalSize > 0
378
+ const spaceSavings = this.compressionStats.totalOriginalSize > 0
252
379
  ? ((this.compressionStats.totalOriginalSize - this.compressionStats.totalCompressedSize) / this.compressionStats.totalOriginalSize * 100).toFixed(2)
253
380
  : 0;
254
381
 
@@ -268,6 +395,62 @@ export class MemoryCache extends Cache {
268
395
  }
269
396
  };
270
397
  }
398
+
399
+ /**
400
+ * Get memory usage statistics
401
+ * @returns {Object} Memory stats including current usage, limits, and eviction counts
402
+ */
403
+ getMemoryStats() {
404
+ const totalItems = Object.keys(this.cache).length;
405
+ const memoryUsagePercent = this.maxMemoryBytes > 0
406
+ ? ((this.currentMemoryBytes / this.maxMemoryBytes) * 100).toFixed(2)
407
+ : 0;
408
+
409
+ const systemMemory = {
410
+ total: os.totalmem(),
411
+ free: os.freemem(),
412
+ used: os.totalmem() - os.freemem()
413
+ };
414
+
415
+ const cachePercentOfTotal = systemMemory.total > 0
416
+ ? ((this.currentMemoryBytes / systemMemory.total) * 100).toFixed(2)
417
+ : 0;
418
+
419
+ return {
420
+ currentMemoryBytes: this.currentMemoryBytes,
421
+ maxMemoryBytes: this.maxMemoryBytes,
422
+ maxMemoryPercent: this.maxMemoryPercent,
423
+ memoryUsagePercent: parseFloat(memoryUsagePercent),
424
+ cachePercentOfSystemMemory: parseFloat(cachePercentOfTotal),
425
+ totalItems,
426
+ maxSize: this.maxSize,
427
+ evictedDueToMemory: this.evictedDueToMemory,
428
+ averageItemSize: totalItems > 0 ? Math.round(this.currentMemoryBytes / totalItems) : 0,
429
+ memoryUsage: {
430
+ current: this._formatBytes(this.currentMemoryBytes),
431
+ max: this.maxMemoryBytes > 0 ? this._formatBytes(this.maxMemoryBytes) : 'unlimited',
432
+ available: this.maxMemoryBytes > 0 ? this._formatBytes(this.maxMemoryBytes - this.currentMemoryBytes) : 'unlimited'
433
+ },
434
+ systemMemory: {
435
+ total: this._formatBytes(systemMemory.total),
436
+ free: this._formatBytes(systemMemory.free),
437
+ used: this._formatBytes(systemMemory.used),
438
+ cachePercent: `${cachePercentOfTotal}%`
439
+ }
440
+ };
441
+ }
442
+
443
+ /**
444
+ * Format bytes to human-readable format
445
+ * @private
446
+ */
447
+ _formatBytes(bytes) {
448
+ if (bytes === 0) return '0 B';
449
+ const k = 1024;
450
+ const sizes = ['B', 'KB', 'MB', 'GB'];
451
+ const i = Math.floor(Math.log(bytes) / Math.log(k));
452
+ return `${(bytes / Math.pow(k, i)).toFixed(2)} ${sizes[i]}`;
453
+ }
271
454
  }
272
455
 
273
456
  export default MemoryCache
@@ -0,0 +1,47 @@
1
+ import { S3dbError } from '../errors.js';
2
+
3
+ /**
4
+ * CacheError - Errors related to cache operations
5
+ *
6
+ * Used for cache operations including:
7
+ * - Cache driver initialization and setup
8
+ * - Cache get/set/delete operations
9
+ * - Cache invalidation and warming
10
+ * - Driver-specific operations (memory, filesystem, S3)
11
+ * - Resource-level caching
12
+ *
13
+ * @extends S3dbError
14
+ */
15
+ export class CacheError extends S3dbError {
16
+ constructor(message, details = {}) {
17
+ const { driver = 'unknown', operation = 'unknown', resourceName, key, ...rest } = details;
18
+
19
+ let description = details.description;
20
+ if (!description) {
21
+ description = `
22
+ Cache Operation Error
23
+
24
+ Driver: ${driver}
25
+ Operation: ${operation}
26
+ ${resourceName ? `Resource: ${resourceName}` : ''}
27
+ ${key ? `Key: ${key}` : ''}
28
+
29
+ Common causes:
30
+ 1. Invalid cache key format
31
+ 2. Cache driver not properly initialized
32
+ 3. Resource not found or not cached
33
+ 4. Memory limits exceeded
34
+ 5. Filesystem permissions issues
35
+
36
+ Solution:
37
+ Check cache configuration and ensure the cache driver is properly initialized.
38
+
39
+ Docs: https://github.com/forattini-dev/s3db.js/blob/main/docs/plugins/cache.md
40
+ `.trim();
41
+ }
42
+
43
+ super(message, { ...rest, driver, operation, resourceName, key, description });
44
+ }
45
+ }
46
+
47
+ export default CacheError;
@@ -8,7 +8,90 @@ import MemoryCache from "./cache/memory-cache.class.js";
8
8
  import { FilesystemCache } from "./cache/filesystem-cache.class.js";
9
9
  import { PartitionAwareFilesystemCache } from "./cache/partition-aware-filesystem-cache.class.js";
10
10
  import tryFn from "../concerns/try-fn.js";
11
-
11
+ import { CacheError } from "./cache.errors.js";
12
+
13
+ /**
14
+ * Cache Plugin Configuration
15
+ *
16
+ * Provides caching layer for S3DB resources with multiple backend options.
17
+ * Automatically caches read operations and invalidates on writes.
18
+ *
19
+ * @typedef {Object} CachePluginOptions
20
+ * @property {string} [driver='s3'] - Cache driver: 'memory', 'filesystem', 's3', or custom driver instance
21
+ * @property {number} [ttl] - Time to live in milliseconds for cached items (shortcut for config.ttl)
22
+ * @property {number} [maxSize] - Maximum number of items to cache (shortcut for config.maxSize)
23
+ * @property {number} [maxMemoryBytes] - (MemoryCache only) Maximum memory in bytes (shortcut for config.maxMemoryBytes). Cannot be used with maxMemoryPercent.
24
+ * @property {number} [maxMemoryPercent] - (MemoryCache only) Maximum memory as fraction 0...1 (shortcut for config.maxMemoryPercent). Cannot be used with maxMemoryBytes.
25
+ *
26
+ * @property {Array<string>} [include] - Only cache these resource names (null = cache all)
27
+ * @property {Array<string>} [exclude=[]] - Never cache these resource names
28
+ *
29
+ * @property {boolean} [includePartitions=true] - Whether to cache partition queries
30
+ * @property {string} [partitionStrategy='hierarchical'] - Partition caching strategy
31
+ * @property {boolean} [partitionAware=true] - Use partition-aware filesystem cache
32
+ * @property {boolean} [trackUsage=true] - Track cache usage statistics
33
+ * @property {boolean} [preloadRelated=true] - Preload related partitions
34
+ *
35
+ * @property {number} [retryAttempts=3] - Number of retry attempts for cache operations
36
+ * @property {number} [retryDelay=100] - Delay between retries in milliseconds
37
+ * @property {boolean} [verbose=false] - Enable verbose logging
38
+ *
39
+ * @property {Object} [config] - Driver-specific configuration (can override top-level ttl, maxSize, maxMemoryBytes, maxMemoryPercent)
40
+ * @property {number} [config.ttl] - Override TTL for this driver
41
+ * @property {number} [config.maxSize] - Override max number of items
42
+ * @property {number} [config.maxMemoryBytes] - (MemoryCache only) Maximum memory in bytes. Cannot be used with config.maxMemoryPercent.
43
+ * @property {number} [config.maxMemoryPercent] - (MemoryCache only) Maximum memory as fraction 0...1 (e.g., 0.1 = 10%). Cannot be used with config.maxMemoryBytes.
44
+ * @property {boolean} [config.enableCompression] - (MemoryCache only) Enable gzip compression
45
+ * @property {number} [config.compressionThreshold=1024] - (MemoryCache only) Minimum size in bytes to trigger compression
46
+ *
47
+ * @example
48
+ * // Memory cache with absolute byte limit
49
+ * new CachePlugin({
50
+ * driver: 'memory',
51
+ * maxMemoryBytes: 512 * 1024 * 1024, // 512MB
52
+ * ttl: 600000 // 10 minutes
53
+ * })
54
+ *
55
+ * @example
56
+ * // Memory cache with percentage limit (cloud-native)
57
+ * new CachePlugin({
58
+ * driver: 'memory',
59
+ * maxMemoryPercent: 0.1, // 10% of system memory
60
+ * ttl: 1800000 // 30 minutes
61
+ * })
62
+ *
63
+ * @example
64
+ * // Filesystem cache with partition awareness
65
+ * new CachePlugin({
66
+ * driver: 'filesystem',
67
+ * partitionAware: true,
68
+ * includePartitions: true,
69
+ * ttl: 3600000 // 1 hour
70
+ * })
71
+ *
72
+ * @example
73
+ * // S3 cache (default)
74
+ * new CachePlugin({
75
+ * driver: 's3',
76
+ * ttl: 7200000 // 2 hours
77
+ * })
78
+ *
79
+ * @example
80
+ * // Selective caching
81
+ * new CachePlugin({
82
+ * driver: 'memory',
83
+ * include: ['users', 'products'], // Only cache these
84
+ * exclude: ['audit_logs'], // Never cache these
85
+ * maxMemoryPercent: 0.15
86
+ * })
87
+ *
88
+ * @notes
89
+ * - maxMemoryBytes and maxMemoryPercent are mutually exclusive (throws error if both set)
90
+ * - maxMemoryPercent is recommended for containerized/cloud environments
91
+ * - Plugin-created resources (createdBy !== 'user') are skipped by default
92
+ * - Cache is automatically invalidated on insert/update/delete operations
93
+ * - Use skipCache: true option on queries to bypass cache for specific calls
94
+ */
12
95
  export class CachePlugin extends Plugin {
13
96
  constructor(options = {}) {
14
97
  super(options);
@@ -20,7 +103,9 @@ export class CachePlugin extends Plugin {
20
103
  config: {
21
104
  ttl: options.ttl,
22
105
  maxSize: options.maxSize,
23
- ...options.config // Driver-specific config (can override ttl/maxSize)
106
+ maxMemoryBytes: options.maxMemoryBytes,
107
+ maxMemoryPercent: options.maxMemoryPercent,
108
+ ...options.config // Driver-specific config (can override ttl/maxSize/maxMemoryBytes/maxMemoryPercent)
24
109
  },
25
110
 
26
111
  // Resource filtering
@@ -471,7 +556,13 @@ export class CachePlugin extends Plugin {
471
556
  async warmCache(resourceName, options = {}) {
472
557
  const resource = this.database.resources[resourceName];
473
558
  if (!resource) {
474
- throw new Error(`Resource '${resourceName}' not found`);
559
+ throw new CacheError('Resource not found for cache warming', {
560
+ operation: 'warmCache',
561
+ driver: this.driver?.constructor.name,
562
+ resourceName,
563
+ availableResources: Object.keys(this.database.resources),
564
+ suggestion: 'Check resource name spelling or ensure resource has been created'
565
+ });
475
566
  }
476
567
 
477
568
  const { includePartitions = true, sampleSize = 100 } = options;
@@ -1209,3 +1209,148 @@ export async function getLastNMonths(resourceName, field, months = 12, options,
1209
1209
 
1210
1210
  return data;
1211
1211
  }
1212
+
1213
+ /**
1214
+ * Get raw transaction events for custom aggregation
1215
+ *
1216
+ * This method provides direct access to the underlying transaction events,
1217
+ * allowing developers to perform custom aggregations beyond the pre-built analytics.
1218
+ * Useful for complex queries, custom metrics, or when you need the raw event data.
1219
+ *
1220
+ * @param {string} resourceName - Resource name
1221
+ * @param {string} field - Field name
1222
+ * @param {Object} options - Query options
1223
+ * @param {string} options.recordId - Filter by specific record ID
1224
+ * @param {string} options.startDate - Start date filter (YYYY-MM-DD or YYYY-MM-DDTHH)
1225
+ * @param {string} options.endDate - End date filter (YYYY-MM-DD or YYYY-MM-DDTHH)
1226
+ * @param {string} options.cohortDate - Filter by cohort date (YYYY-MM-DD)
1227
+ * @param {string} options.cohortHour - Filter by cohort hour (YYYY-MM-DDTHH)
1228
+ * @param {string} options.cohortMonth - Filter by cohort month (YYYY-MM)
1229
+ * @param {boolean} options.applied - Filter by applied status (true/false/undefined for both)
1230
+ * @param {string} options.operation - Filter by operation type ('add', 'sub', 'set')
1231
+ * @param {number} options.limit - Maximum number of events to return
1232
+ * @param {Object} fieldHandlers - Field handlers map
1233
+ * @returns {Promise<Array>} Raw transaction events
1234
+ *
1235
+ * @example
1236
+ * // Get all events for a specific record
1237
+ * const events = await plugin.getRawEvents('wallets', 'balance', {
1238
+ * recordId: 'wallet1'
1239
+ * });
1240
+ *
1241
+ * @example
1242
+ * // Get events for a specific time range
1243
+ * const events = await plugin.getRawEvents('wallets', 'balance', {
1244
+ * startDate: '2025-10-01',
1245
+ * endDate: '2025-10-31'
1246
+ * });
1247
+ *
1248
+ * @example
1249
+ * // Get only pending (unapplied) transactions
1250
+ * const pending = await plugin.getRawEvents('wallets', 'balance', {
1251
+ * applied: false
1252
+ * });
1253
+ */
1254
+ export async function getRawEvents(resourceName, field, options, fieldHandlers) {
1255
+ // Get handler for this resource/field combination
1256
+ const resourceHandlers = fieldHandlers.get(resourceName);
1257
+ if (!resourceHandlers) {
1258
+ throw new Error(`No eventual consistency configured for resource: ${resourceName}`);
1259
+ }
1260
+
1261
+ const handler = resourceHandlers.get(field);
1262
+ if (!handler) {
1263
+ throw new Error(`No eventual consistency configured for field: ${resourceName}.${field}`);
1264
+ }
1265
+
1266
+ if (!handler.transactionResource) {
1267
+ throw new Error('Transaction resource not initialized');
1268
+ }
1269
+
1270
+ const {
1271
+ recordId,
1272
+ startDate,
1273
+ endDate,
1274
+ cohortDate,
1275
+ cohortHour,
1276
+ cohortMonth,
1277
+ applied,
1278
+ operation,
1279
+ limit
1280
+ } = options;
1281
+
1282
+ // Build query object for partition-based filtering
1283
+ const query = {};
1284
+
1285
+ // Filter by recordId (uses partition if available)
1286
+ if (recordId !== undefined) {
1287
+ query.originalId = recordId;
1288
+ }
1289
+
1290
+ // Filter by applied status (uses partition)
1291
+ if (applied !== undefined) {
1292
+ query.applied = applied;
1293
+ }
1294
+
1295
+ // Fetch transactions using partition-aware query
1296
+ const [ok, err, allTransactions] = await tryFn(() =>
1297
+ handler.transactionResource.query(query)
1298
+ );
1299
+
1300
+ if (!ok || !allTransactions) {
1301
+ return [];
1302
+ }
1303
+
1304
+ // Ensure all transactions have cohort fields
1305
+ let filtered = allTransactions;
1306
+
1307
+ // Filter by operation type
1308
+ if (operation !== undefined) {
1309
+ filtered = filtered.filter(t => t.operation === operation);
1310
+ }
1311
+
1312
+ // Filter by temporal fields (these are in-memory filters after partition query)
1313
+ if (cohortDate) {
1314
+ filtered = filtered.filter(t => t.cohortDate === cohortDate);
1315
+ }
1316
+
1317
+ if (cohortHour) {
1318
+ filtered = filtered.filter(t => t.cohortHour === cohortHour);
1319
+ }
1320
+
1321
+ if (cohortMonth) {
1322
+ filtered = filtered.filter(t => t.cohortMonth === cohortMonth);
1323
+ }
1324
+
1325
+ if (startDate && endDate) {
1326
+ // Determine which cohort field to use based on format
1327
+ const isHourly = startDate.length > 10; // YYYY-MM-DDTHH vs YYYY-MM-DD
1328
+ const cohortField = isHourly ? 'cohortHour' : 'cohortDate';
1329
+
1330
+ filtered = filtered.filter(t =>
1331
+ t[cohortField] && t[cohortField] >= startDate && t[cohortField] <= endDate
1332
+ );
1333
+ } else if (startDate) {
1334
+ const isHourly = startDate.length > 10;
1335
+ const cohortField = isHourly ? 'cohortHour' : 'cohortDate';
1336
+ filtered = filtered.filter(t => t[cohortField] && t[cohortField] >= startDate);
1337
+ } else if (endDate) {
1338
+ const isHourly = endDate.length > 10;
1339
+ const cohortField = isHourly ? 'cohortHour' : 'cohortDate';
1340
+ filtered = filtered.filter(t => t[cohortField] && t[cohortField] <= endDate);
1341
+ }
1342
+
1343
+ // Sort by timestamp (newest first by default)
1344
+ filtered.sort((a, b) => {
1345
+ const aTime = new Date(a.timestamp || a.createdAt).getTime();
1346
+ const bTime = new Date(b.timestamp || b.createdAt).getTime();
1347
+ return bTime - aTime;
1348
+ });
1349
+
1350
+ // Apply limit
1351
+ if (limit && limit > 0) {
1352
+ filtered = filtered.slice(0, limit);
1353
+ }
1354
+
1355
+ return filtered;
1356
+ }