s3db.js 6.1.0 → 7.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. package/PLUGINS.md +2724 -0
  2. package/README.md +377 -492
  3. package/UNLICENSE +24 -0
  4. package/dist/s3db.cjs.js +30054 -18189
  5. package/dist/s3db.cjs.min.js +1 -1
  6. package/dist/s3db.d.ts +373 -72
  7. package/dist/s3db.es.js +30040 -18186
  8. package/dist/s3db.es.min.js +1 -1
  9. package/dist/s3db.iife.js +29727 -17863
  10. package/dist/s3db.iife.min.js +1 -1
  11. package/package.json +44 -69
  12. package/src/behaviors/body-only.js +110 -0
  13. package/src/behaviors/body-overflow.js +153 -0
  14. package/src/behaviors/enforce-limits.js +195 -0
  15. package/src/behaviors/index.js +39 -0
  16. package/src/behaviors/truncate-data.js +204 -0
  17. package/src/behaviors/user-managed.js +147 -0
  18. package/src/client.class.js +515 -0
  19. package/src/concerns/base62.js +61 -0
  20. package/src/concerns/calculator.js +204 -0
  21. package/src/concerns/crypto.js +142 -0
  22. package/src/concerns/id.js +8 -0
  23. package/src/concerns/index.js +5 -0
  24. package/src/concerns/try-fn.js +151 -0
  25. package/src/connection-string.class.js +75 -0
  26. package/src/database.class.js +599 -0
  27. package/src/errors.js +261 -0
  28. package/src/index.js +17 -0
  29. package/src/plugins/audit.plugin.js +442 -0
  30. package/src/plugins/cache/cache.class.js +53 -0
  31. package/src/plugins/cache/index.js +6 -0
  32. package/src/plugins/cache/memory-cache.class.js +164 -0
  33. package/src/plugins/cache/s3-cache.class.js +189 -0
  34. package/src/plugins/cache.plugin.js +275 -0
  35. package/src/plugins/consumers/index.js +24 -0
  36. package/src/plugins/consumers/rabbitmq-consumer.js +56 -0
  37. package/src/plugins/consumers/sqs-consumer.js +102 -0
  38. package/src/plugins/costs.plugin.js +81 -0
  39. package/src/plugins/fulltext.plugin.js +473 -0
  40. package/src/plugins/index.js +12 -0
  41. package/src/plugins/metrics.plugin.js +603 -0
  42. package/src/plugins/plugin.class.js +210 -0
  43. package/src/plugins/plugin.obj.js +13 -0
  44. package/src/plugins/queue-consumer.plugin.js +134 -0
  45. package/src/plugins/replicator.plugin.js +769 -0
  46. package/src/plugins/replicators/base-replicator.class.js +85 -0
  47. package/src/plugins/replicators/bigquery-replicator.class.js +328 -0
  48. package/src/plugins/replicators/index.js +44 -0
  49. package/src/plugins/replicators/postgres-replicator.class.js +427 -0
  50. package/src/plugins/replicators/s3db-replicator.class.js +352 -0
  51. package/src/plugins/replicators/sqs-replicator.class.js +427 -0
  52. package/src/resource.class.js +2626 -0
  53. package/src/s3db.d.ts +1263 -0
  54. package/src/schema.class.js +706 -0
  55. package/src/stream/index.js +16 -0
  56. package/src/stream/resource-ids-page-reader.class.js +10 -0
  57. package/src/stream/resource-ids-reader.class.js +63 -0
  58. package/src/stream/resource-reader.class.js +81 -0
  59. package/src/stream/resource-writer.class.js +92 -0
  60. package/src/validator.class.js +97 -0
package/PLUGINS.md ADDED
@@ -0,0 +1,2724 @@
1
+ # 🔌 s3db.js Plugins Documentation
2
+
3
+ <p align="center">
4
+ <strong>Comprehensive guide to all s3db.js plugins</strong><br>
5
+ <em>Extend your database with powerful features</em>
6
+ </p>
7
+
8
+ ---
9
+
10
+ ## 📋 Table of Contents
11
+
12
+ - [🚀 Getting Started](#-getting-started-with-plugins)
13
+ - [🧩 Available Plugins](#-available-plugins)
14
+ - [💾 Cache Plugin](#-cache-plugin)
15
+ - [💰 Costs Plugin](#-costs-plugin)
16
+ - [📝 Audit Plugin](#-audit-plugin)
17
+ - [🔍 FullText Plugin](#-fulltext-plugin)
18
+ - [📊 Metrics Plugin](#-metrics-plugin)
19
+ - [🔄 Replicator Plugin](#-replicator-plugin)
20
+ - [📬 Queue Consumer Plugin](#-queue-consumer-plugin)
21
+ - [🔧 Plugin Development](#-plugin-development)
22
+ - [💡 Plugin Combinations](#-plugin-combinations)
23
+ - [🎯 Best Practices](#-best-practices)
24
+
25
+ ---
26
+
27
+ ## 🚀 Getting Started with Plugins
28
+
29
+ Plugins extend s3db.js with additional functionality. They can be used individually or combined for powerful workflows.
30
+
31
+ ### Basic Plugin Usage
32
+
33
+ ```javascript
34
+ import { S3db, CachePlugin, CostsPlugin } from 's3db.js';
35
+
36
+ const s3db = new S3db({
37
+ connectionString: "s3://ACCESS_KEY:SECRET_KEY@BUCKET_NAME/databases/myapp",
38
+ plugins: [
39
+ new CachePlugin(),
40
+ CostsPlugin // Some plugins are static objects
41
+ ]
42
+ });
43
+
44
+ await s3db.connect();
45
+ ```
46
+
47
+ ### Plugin Types
48
+
49
+ - **Instance Plugins**: Require `new` - `new CachePlugin(config)`
50
+ - **Static Plugins**: Used directly - `CostsPlugin`
51
+ - **Configurable**: Accept options for customization
52
+ - **Event-Driven**: Emit events for monitoring and integration
53
+
54
+ ---
55
+
56
+ ## 🧩 Available Plugins
57
+
58
+ ## 💾 Cache Plugin
59
+
60
+ Intelligent caching system that reduces S3 API calls and improves performance by storing frequently accessed data in memory or S3.
61
+
62
+ ### ⚡ Quick Start
63
+
64
+ ```javascript
65
+ import { S3db, CachePlugin } from 's3db.js';
66
+
67
+ const s3db = new S3db({
68
+ connectionString: "s3://ACCESS_KEY:SECRET_KEY@BUCKET_NAME/databases/myapp",
69
+ plugins: [new CachePlugin()]
70
+ });
71
+
72
+ await s3db.connect();
73
+
74
+ // Cache is automatically used for read operations
75
+ const users = s3db.resource('users');
76
+ await users.count(); // Cached for default TTL
77
+ await users.list(); // Cached result
78
+ ```
79
+
80
+ ### ⚙️ Configuration Parameters
81
+
82
+ | Parameter | Type | Default | Description |
83
+ |-----------|------|---------|-------------|
84
+ | `driverType` | string | `'s3'` | Cache driver: `'memory'` or `'s3'` |
85
+ | `ttl` | number | `300000` | Time-to-live in milliseconds (5 minutes) |
86
+ | `maxSize` | number | `1000` | Maximum number of items in cache (memory driver) |
87
+ | `includePartitions` | boolean | `true` | Include partition values in cache keys |
88
+ | `driver` | object | `null` | Custom cache driver instance |
89
+ | `memoryOptions` | object | `{}` | Options for memory cache driver |
90
+ | `s3Options` | object | `{}` | Options for S3 cache driver |
91
+
92
+ ### Memory Driver Options (`memoryOptions`)
93
+
94
+ | Parameter | Type | Default | Description |
95
+ |-----------|------|---------|-------------|
96
+ | `maxSize` | number | `1000` | Maximum items in memory |
97
+ | `ttl` | number | `300000` | Default TTL in milliseconds |
98
+ | `checkPeriod` | number | `600000` | Cleanup interval in milliseconds |
99
+
100
+ ### S3 Driver Options (`s3Options`)
101
+
102
+ | Parameter | Type | Default | Description |
103
+ |-----------|------|---------|-------------|
104
+ | `bucket` | string | Same as database | S3 bucket for cache storage |
105
+ | `prefix` | string | `'cache/'` | S3 key prefix for cache objects |
106
+ | `client` | object | Database client | Custom S3 client instance |
107
+
108
+ ### 🔧 Easy Example
109
+
110
+ ```javascript
111
+ import { S3db, CachePlugin } from 's3db.js';
112
+
113
+ const s3db = new S3db({
114
+ connectionString: "s3://ACCESS_KEY:SECRET_KEY@BUCKET_NAME/databases/myapp",
115
+ plugins: [new CachePlugin({
116
+ driverType: 'memory',
117
+ ttl: 600000, // 10 minutes
118
+ maxSize: 500
119
+ })]
120
+ });
121
+
122
+ await s3db.connect();
123
+
124
+ const products = s3db.resource('products');
125
+
126
+ // First call hits the database
127
+ console.time('First call');
128
+ const result1 = await products.count();
129
+ console.timeEnd('First call'); // ~200ms
130
+
131
+ // Second call uses cache
132
+ console.time('Cached call');
133
+ const result2 = await products.count();
134
+ console.timeEnd('Cached call'); // ~2ms
135
+
136
+ // Cache is automatically cleared on write operations
137
+ await products.insert({ name: 'New Product', price: 29.99 });
138
+
139
+ // Next call will hit database again (cache cleared)
140
+ const result3 = await products.count(); // Fresh data
141
+ ```
142
+
143
+ ### 🚀 Advanced Configuration Example
144
+
145
+ ```javascript
146
+ import { S3db, CachePlugin, MemoryCache, S3Cache } from 's3db.js';
147
+
148
+ // Custom cache driver with advanced configuration
149
+ const customCache = new MemoryCache({
150
+ maxSize: 2000,
151
+ ttl: 900000, // 15 minutes
152
+ checkPeriod: 300000, // 5 minutes cleanup
153
+ algorithm: 'lru' // Least Recently Used
154
+ });
155
+
156
+ const s3db = new S3db({
157
+ connectionString: "s3://ACCESS_KEY:SECRET_KEY@BUCKET_NAME/databases/myapp",
158
+ plugins: [new CachePlugin({
159
+ driver: customCache,
160
+ includePartitions: true,
161
+
162
+ // S3 cache fallback for persistence
163
+ s3Options: {
164
+ bucket: 'my-cache-bucket',
165
+ prefix: 'app-cache/',
166
+ ttl: 3600000, // 1 hour S3 cache
167
+ compression: true,
168
+ encryption: true
169
+ },
170
+
171
+ // Memory cache for speed
172
+ memoryOptions: {
173
+ maxSize: 5000,
174
+ ttl: 600000, // 10 minutes memory cache
175
+ checkPeriod: 120000, // 2 minutes cleanup
176
+ evictionPolicy: 'lru',
177
+ stats: true // Enable cache statistics
178
+ }
179
+ })]
180
+ });
181
+
182
+ await s3db.connect();
183
+
184
+ // Access cache methods on resources
185
+ const users = s3db.resource('users');
186
+
187
+ // Generate custom cache keys
188
+ const cacheKey = await users.cacheKeyFor({
189
+ action: 'list',
190
+ params: { limit: 10 },
191
+ partition: 'byStatus',
192
+ partitionValues: { status: 'active' }
193
+ });
194
+
195
+ // Manual cache operations
196
+ await users.cache.set(cacheKey, data, 1800000); // 30 minutes
197
+ const cached = await users.cache.get(cacheKey);
198
+ await users.cache.delete(cacheKey);
199
+ await users.cache.clear(); // Clear all cache
200
+
201
+ // Cache statistics (if enabled)
202
+ const stats = users.cache.stats();
203
+ console.log('Cache hit rate:', stats.hitRate);
204
+ console.log('Total hits:', stats.hits);
205
+ console.log('Total misses:', stats.misses);
206
+ ```
207
+
208
+ ---
209
+
210
+ ## 💰 Costs Plugin
211
+
212
+ Track and monitor AWS S3 costs in real-time by calculating expenses for each API operation. Essential for cost optimization and budget management.
213
+
214
+ ### ⚡ Quick Start
215
+
216
+ ```javascript
217
+ import { S3db, CostsPlugin } from 's3db.js';
218
+
219
+ const s3db = new S3db({
220
+ connectionString: "s3://ACCESS_KEY:SECRET_KEY@BUCKET_NAME/databases/myapp",
221
+ plugins: [CostsPlugin] // Static plugin - no 'new' required
222
+ });
223
+
224
+ await s3db.connect();
225
+
226
+ // Use your database normally
227
+ const users = s3db.resource('users');
228
+ await users.insert({ name: 'John', email: 'john@example.com' });
229
+ await users.list();
230
+
231
+ // Check costs
232
+ console.log('Total cost:', s3db.client.costs.total);
233
+ console.log('Request breakdown:', s3db.client.costs.requests);
234
+ ```
235
+
236
+ ### ⚙️ Configuration Parameters
237
+
238
+ **Note**: CostsPlugin is a static plugin with no configuration options. It automatically tracks all S3 operations.
239
+
240
+ ### Cost Tracking Details
241
+
242
+ | Operation | Cost per 1000 requests | Tracked Commands |
243
+ |-----------|------------------------|------------------|
244
+ | PUT operations | $0.005 | PutObjectCommand |
245
+ | GET operations | $0.0004 | GetObjectCommand |
246
+ | HEAD operations | $0.0004 | HeadObjectCommand |
247
+ | DELETE operations | $0.0004 | DeleteObjectCommand, DeleteObjectsCommand |
248
+ | LIST operations | $0.005 | ListObjectsV2Command |
249
+
250
+ ### Cost Data Structure
251
+
252
+ ```javascript
253
+ {
254
+ total: 0.000123, // Total cost in USD
255
+ prices: { // Cost per 1000 requests
256
+ put: 0.000005,
257
+ get: 0.0000004,
258
+ head: 0.0000004,
259
+ delete: 0.0000004,
260
+ list: 0.000005
261
+ },
262
+ requests: { // Request counters
263
+ total: 15,
264
+ put: 3,
265
+ get: 8,
266
+ head: 2,
267
+ delete: 1,
268
+ list: 1
269
+ },
270
+ events: { // Command-specific counters
271
+ total: 15,
272
+ PutObjectCommand: 3,
273
+ GetObjectCommand: 8,
274
+ HeadObjectCommand: 2,
275
+ DeleteObjectCommand: 1,
276
+ ListObjectsV2Command: 1
277
+ }
278
+ }
279
+ ```
280
+
281
+ ### 🔧 Easy Example
282
+
283
+ ```javascript
284
+ import { S3db, CostsPlugin } from 's3db.js';
285
+
286
+ const s3db = new S3db({
287
+ connectionString: "s3://ACCESS_KEY:SECRET_KEY@BUCKET_NAME/databases/myapp",
288
+ plugins: [CostsPlugin]
289
+ });
290
+
291
+ await s3db.connect();
292
+
293
+ const products = s3db.resource('products');
294
+
295
+ // Perform operations and track costs
296
+ await products.insert({ name: 'Widget A', price: 19.99 });
297
+ await products.insert({ name: 'Widget B', price: 29.99 });
298
+ await products.list();
299
+ await products.count();
300
+
301
+ // Analyze costs
302
+ const costs = s3db.client.costs;
303
+ console.log(`Operations performed: ${costs.requests.total}`);
304
+ console.log(`Total cost: $${costs.total.toFixed(6)}`);
305
+ console.log(`Most expensive operation: PUT (${costs.requests.put} requests)`);
306
+
307
+ // Cost breakdown
308
+ console.log('\nCost breakdown:');
309
+ Object.entries(costs.requests).forEach(([operation, count]) => {
310
+ if (operation !== 'total' && count > 0) {
311
+ const operationCost = count * costs.prices[operation];
312
+ console.log(` ${operation.toUpperCase()}: ${count} requests = $${operationCost.toFixed(6)}`);
313
+ }
314
+ });
315
+ ```
316
+
317
+ ### 🚀 Advanced Monitoring Example
318
+
319
+ ```javascript
320
+ import { S3db, CostsPlugin } from 's3db.js';
321
+
322
+ class CostMonitor {
323
+ constructor(s3db) {
324
+ this.s3db = s3db;
325
+ this.startTime = Date.now();
326
+ this.checkpoints = [];
327
+ }
328
+
329
+ checkpoint(label) {
330
+ const costs = { ...this.s3db.client.costs };
331
+ const timestamp = Date.now();
332
+
333
+ this.checkpoints.push({
334
+ label,
335
+ timestamp,
336
+ costs,
337
+ duration: timestamp - this.startTime
338
+ });
339
+
340
+ return costs;
341
+ }
342
+
343
+ report() {
344
+ console.log('\n=== Cost Analysis Report ===');
345
+
346
+ for (let i = 0; i < this.checkpoints.length; i++) {
347
+ const checkpoint = this.checkpoints[i];
348
+ const prevCheckpoint = i > 0 ? this.checkpoints[i - 1] : null;
349
+
350
+ console.log(`\n${checkpoint.label}:`);
351
+ console.log(` Time: ${checkpoint.duration}ms`);
352
+ console.log(` Total cost: $${checkpoint.costs.total.toFixed(6)}`);
353
+
354
+ if (prevCheckpoint) {
355
+ const costDiff = checkpoint.costs.total - prevCheckpoint.costs.total;
356
+ const requestDiff = checkpoint.costs.requests.total - prevCheckpoint.costs.requests.total;
357
+ console.log(` Cost increase: $${costDiff.toFixed(6)}`);
358
+ console.log(` New requests: ${requestDiff}`);
359
+ }
360
+ }
361
+
362
+ // Efficiency metrics
363
+ const finalCosts = this.checkpoints[this.checkpoints.length - 1].costs;
364
+ const totalTime = this.checkpoints[this.checkpoints.length - 1].duration;
365
+
366
+ console.log('\n=== Efficiency Metrics ===');
367
+ console.log(`Total execution time: ${totalTime}ms`);
368
+ console.log(`Total requests: ${finalCosts.requests.total}`);
369
+ console.log(`Requests per second: ${(finalCosts.requests.total / (totalTime / 1000)).toFixed(2)}`);
370
+ console.log(`Cost per request: $${(finalCosts.total / finalCosts.requests.total).toFixed(8)}`);
371
+ console.log(`Monthly projection (1M ops): $${(finalCosts.total * 1000000).toFixed(2)}`);
372
+ }
373
+ }
374
+
375
+ // Usage
376
+ const s3db = new S3db({
377
+ connectionString: "s3://ACCESS_KEY:SECRET_KEY@BUCKET_NAME/databases/myapp",
378
+ plugins: [CostsPlugin]
379
+ });
380
+
381
+ await s3db.connect();
382
+
383
+ const monitor = new CostMonitor(s3db);
384
+ const users = s3db.resource('users');
385
+
386
+ // Bulk operations with cost tracking
387
+ monitor.checkpoint('Initial state');
388
+
389
+ // Bulk insert
390
+ const userData = Array.from({ length: 100 }, (_, i) => ({
391
+ name: `User ${i}`,
392
+ email: `user${i}@example.com`,
393
+ role: i % 3 === 0 ? 'admin' : 'user'
394
+ }));
395
+
396
+ await users.insertMany(userData);
397
+ monitor.checkpoint('After bulk insert');
398
+
399
+ // Query operations
400
+ await users.count();
401
+ await users.list({ limit: 50 });
402
+ await users.list({ limit: 25, offset: 25 });
403
+ monitor.checkpoint('After queries');
404
+
405
+ // Update operations
406
+ const userList = await users.list({ limit: 10 });
407
+ for (const user of userList) {
408
+ await users.update(user.id, { lastLogin: new Date().toISOString() });
409
+ }
410
+ monitor.checkpoint('After updates');
411
+
412
+ // Generate detailed report
413
+ monitor.report();
414
+
415
+ // Set cost alerts
416
+ const currentCost = s3db.client.costs.total;
417
+ if (currentCost > 0.01) { // $0.01 threshold
418
+ console.warn(`⚠️ Cost threshold exceeded: $${currentCost.toFixed(6)}`);
419
+ }
420
+
421
+ // Export cost data for external analysis
422
+ const costData = {
423
+ timestamp: new Date().toISOString(),
424
+ sessionCosts: s3db.client.costs,
425
+ checkpoints: monitor.checkpoints,
426
+ summary: {
427
+ totalCost: s3db.client.costs.total,
428
+ totalRequests: s3db.client.costs.requests.total,
429
+ avgCostPerRequest: s3db.client.costs.total / s3db.client.costs.requests.total,
430
+ mostExpensiveOperation: Object.entries(s3db.client.costs.requests)
431
+ .filter(([key]) => key !== 'total')
432
+ .sort(([,a], [,b]) => b - a)[0]
433
+ }
434
+ };
435
+
436
+ console.log('\nExportable cost data:', JSON.stringify(costData, null, 2));
437
+ ```
438
+
439
+ ---
440
+
441
+ ## 📝 Audit Plugin
442
+
443
+ Comprehensive audit logging system that tracks all database operations for compliance, security monitoring, and debugging purposes.
444
+
445
+ ### ⚡ Quick Start
446
+
447
+ ```javascript
448
+ import { S3db, AuditPlugin } from 's3db.js';
449
+
450
+ const s3db = new S3db({
451
+ connectionString: "s3://ACCESS_KEY:SECRET_KEY@BUCKET_NAME/databases/myapp",
452
+ plugins: [new AuditPlugin({ enabled: true })]
453
+ });
454
+
455
+ await s3db.connect();
456
+
457
+ // All operations are automatically logged
458
+ const users = s3db.resource('users');
459
+ await users.insert({ name: 'John', email: 'john@example.com' });
460
+ await users.update(userId, { name: 'John Doe' });
461
+
462
+ // Access audit logs
463
+ const auditResource = s3db.resource('audits');
464
+ const logs = await auditResource.list();
465
+ console.log('Audit trail:', logs);
466
+ ```
467
+
468
+ ### ⚙️ Configuration Parameters
469
+
470
+ | Parameter | Type | Default | Description |
471
+ |-----------|------|---------|-------------|
472
+ | `enabled` | boolean | `true` | Enable/disable audit logging |
473
+ | `includeData` | boolean | `true` | Include data payloads in audit logs |
474
+ | `includePartitions` | boolean | `true` | Include partition information in logs |
475
+ | `maxDataSize` | number | `10000` | Maximum data size to log (bytes) |
476
+ | `trackOperations` | array | `['insert', 'update', 'delete']` | Operations to audit |
477
+ | `excludeResources` | array | `[]` | Resources to exclude from auditing |
478
+ | `userId` | function | `null` | Function to extract user ID from context |
479
+ | `metadata` | function | `null` | Function to add custom metadata |
480
+
481
+ ### Audit Log Structure
482
+
483
+ ```javascript
484
+ {
485
+ id: 'audit-abc123',
486
+ resourceName: 'users',
487
+ operation: 'insert',
488
+ recordId: 'user-123',
489
+ userId: 'admin-456',
490
+ timestamp: '2024-01-15T10:30:00.000Z',
491
+ oldData: '{"name":"John"}', // For updates
492
+ newData: '{"name":"John Doe"}', // JSON string of data
493
+ partition: 'byStatus', // If using partitions
494
+ partitionValues: '{"status":"active"}',
495
+ metadata: '{"ip":"192.168.1.1"}', // Custom metadata
496
+ _v: 0 // Audit record version
497
+ }
498
+ ```
499
+
500
+ ### 🔧 Easy Example
501
+
502
+ ```javascript
503
+ import { S3db, AuditPlugin } from 's3db.js';
504
+
505
+ const s3db = new S3db({
506
+ connectionString: "s3://ACCESS_KEY:SECRET_KEY@BUCKET_NAME/databases/myapp",
507
+ plugins: [new AuditPlugin({
508
+ enabled: true,
509
+ includeData: true,
510
+ trackOperations: ['insert', 'update', 'delete', 'get'],
511
+ maxDataSize: 5000
512
+ })]
513
+ });
514
+
515
+ await s3db.connect();
516
+
517
+ const products = s3db.resource('products');
518
+ const audits = s3db.resource('audits');
519
+
520
+ // Perform operations (automatically audited)
521
+ const product = await products.insert({
522
+ name: 'Gaming Laptop',
523
+ price: 1299.99,
524
+ category: 'electronics'
525
+ });
526
+
527
+ await products.update(product.id, { price: 1199.99 });
528
+ await products.get(product.id);
529
+ await products.delete(product.id);
530
+
531
+ // Review audit trail
532
+ const auditLogs = await audits.list();
533
+
534
+ console.log('\n=== Audit Trail ===');
535
+ auditLogs.forEach(log => {
536
+ console.log(`${log.timestamp} | ${log.operation.toUpperCase()} | ${log.resourceName} | ${log.recordId}`);
537
+
538
+ if (log.operation === 'update') {
539
+ const oldData = JSON.parse(log.oldData);
540
+ const newData = JSON.parse(log.newData);
541
+ console.log(` Price changed: $${oldData.price} → $${newData.price}`);
542
+ }
543
+ });
544
+
545
+ // Query specific audit logs
546
+ const updateLogs = await audits.list({
547
+ filter: log => log.operation === 'update'
548
+ });
549
+
550
+ console.log(`\nFound ${updateLogs.length} update operations`);
551
+ ```
552
+
553
+ ### 🚀 Advanced Configuration Example
554
+
555
+ ```javascript
556
+ import { S3db, AuditPlugin } from 's3db.js';
557
+
558
+ const s3db = new S3db({
559
+ connectionString: "s3://ACCESS_KEY:SECRET_KEY@BUCKET_NAME/databases/myapp",
560
+ plugins: [new AuditPlugin({
561
+ enabled: true,
562
+ includeData: true,
563
+ includePartitions: true,
564
+ maxDataSize: 20000, // 20KB limit
565
+
566
+ // Track all operations including reads
567
+ trackOperations: ['insert', 'update', 'delete', 'get', 'list'],
568
+
569
+ // Exclude sensitive resources from auditing
570
+ excludeResources: ['sessions', 'temp_data'],
571
+
572
+ // Extract user ID from request context
573
+ userId: (context) => {
574
+ return context?.user?.id ||
575
+ context?.headers?.['x-user-id'] ||
576
+ 'anonymous';
577
+ },
578
+
579
+ // Add custom metadata to audit logs
580
+ metadata: (operation, resourceName, data, context) => {
581
+ return {
582
+ ip: context?.ip,
583
+ userAgent: context?.userAgent,
584
+ sessionId: context?.sessionId,
585
+ apiVersion: '1.0',
586
+ environment: process.env.NODE_ENV,
587
+ requestId: context?.requestId,
588
+
589
+ // Operation-specific metadata
590
+ ...(operation === 'insert' && {
591
+ createdVia: 'api',
592
+ validationPassed: true
593
+ }),
594
+
595
+ ...(operation === 'update' && {
596
+ fieldsChanged: Object.keys(data || {}),
597
+ automaticUpdate: false
598
+ }),
599
+
600
+ ...(operation === 'delete' && {
601
+ softDelete: false,
602
+ cascadeDelete: false
603
+ })
604
+ };
605
+ }
606
+ })]
607
+ });
608
+
609
+ await s3db.connect();
610
+
611
+ // Custom audit query functions
612
+ class AuditAnalyzer {
613
+ constructor(auditResource) {
614
+ this.audits = auditResource;
615
+ }
616
+
617
+ async getUserActivity(userId, timeRange = 24) {
618
+ const since = new Date(Date.now() - timeRange * 60 * 60 * 1000);
619
+ const logs = await this.audits.list();
620
+
621
+ return logs.filter(log =>
622
+ log.userId === userId &&
623
+ new Date(log.timestamp) > since
624
+ );
625
+ }
626
+
627
+ async getResourceActivity(resourceName, operation = null) {
628
+ const logs = await this.audits.list();
629
+
630
+ return logs.filter(log =>
631
+ log.resourceName === resourceName &&
632
+ (!operation || log.operation === operation)
633
+ );
634
+ }
635
+
636
+ async getDataChanges(resourceName, recordId) {
637
+ const logs = await this.audits.list();
638
+
639
+ return logs
640
+ .filter(log =>
641
+ log.resourceName === resourceName &&
642
+ log.recordId === recordId &&
643
+ log.operation === 'update'
644
+ )
645
+ .sort((a, b) => new Date(a.timestamp) - new Date(b.timestamp))
646
+ .map(log => ({
647
+ timestamp: log.timestamp,
648
+ oldData: JSON.parse(log.oldData || '{}'),
649
+ newData: JSON.parse(log.newData || '{}'),
650
+ userId: log.userId,
651
+ metadata: JSON.parse(log.metadata || '{}')
652
+ }));
653
+ }
654
+
655
+ async generateComplianceReport(startDate, endDate) {
656
+ const logs = await this.audits.list();
657
+
658
+ const filteredLogs = logs.filter(log => {
659
+ const logDate = new Date(log.timestamp);
660
+ return logDate >= startDate && logDate <= endDate;
661
+ });
662
+
663
+ const summary = {
664
+ totalOperations: filteredLogs.length,
665
+ operationBreakdown: {},
666
+ resourceActivity: {},
667
+ userActivity: {},
668
+ timeRange: { startDate, endDate }
669
+ };
670
+
671
+ filteredLogs.forEach(log => {
672
+ // Operation breakdown
673
+ summary.operationBreakdown[log.operation] =
674
+ (summary.operationBreakdown[log.operation] || 0) + 1;
675
+
676
+ // Resource activity
677
+ summary.resourceActivity[log.resourceName] =
678
+ (summary.resourceActivity[log.resourceName] || 0) + 1;
679
+
680
+ // User activity
681
+ summary.userActivity[log.userId] =
682
+ (summary.userActivity[log.userId] || 0) + 1;
683
+ });
684
+
685
+ return summary;
686
+ }
687
+ }
688
+
689
+ // Usage with context
690
+ const users = s3db.resource('users');
691
+ const audits = s3db.resource('audits');
692
+ const analyzer = new AuditAnalyzer(audits);
693
+
694
+ // Simulate operations with user context
695
+ const userContext = {
696
+ user: { id: 'admin-123', role: 'admin' },
697
+ ip: '192.168.1.100',
698
+ userAgent: 'Mozilla/5.0...',
699
+ sessionId: 'sess-789',
700
+ requestId: 'req-456'
701
+ };
702
+
703
+ // Operations with context (would be passed through middleware in real app)
704
+ await users.insert({
705
+ name: 'Alice Johnson',
706
+ email: 'alice@example.com'
707
+ }, userContext);
708
+
709
+ // Analyze audit data
710
+ const userActivity = await analyzer.getUserActivity('admin-123');
711
+ console.log('Recent user activity:', userActivity);
712
+
713
+ const complianceReport = await analyzer.generateComplianceReport(
714
+ new Date(Date.now() - 7 * 24 * 60 * 60 * 1000), // Last 7 days
715
+ new Date()
716
+ );
717
+
718
+ console.log('\n=== Compliance Report ===');
719
+ console.log(`Total operations: ${complianceReport.totalOperations}`);
720
+ console.log('Operation breakdown:', complianceReport.operationBreakdown);
721
+ console.log('Most active resource:',
722
+ Object.entries(complianceReport.resourceActivity)
723
+ .sort(([,a], [,b]) => b - a)[0]
724
+ );
725
+
726
+ // Real-time audit monitoring
727
+ audits.on('insert', (auditLog) => {
728
+ console.log(`🔍 New audit log: ${auditLog.operation} on ${auditLog.resourceName}`);
729
+
730
+ // Security alerts
731
+ if (auditLog.operation === 'delete' && auditLog.userId === 'anonymous') {
732
+ console.warn('🚨 SECURITY ALERT: Anonymous user performed delete operation');
733
+ }
734
+
735
+ if (auditLog.operation === 'get' && auditLog.resourceName === 'sensitive_data') {
736
+ console.warn('🔒 PRIVACY ALERT: Sensitive data accessed');
737
+ }
738
+ });
739
+
740
+ // Audit log retention and cleanup
741
+ setInterval(async () => {
742
+ const thirtyDaysAgo = new Date(Date.now() - 30 * 24 * 60 * 60 * 1000);
743
+ const oldLogs = await audits.list({
744
+ filter: log => new Date(log.timestamp) < thirtyDaysAgo
745
+ });
746
+
747
+ console.log(`Cleaning up ${oldLogs.length} old audit logs`);
748
+
749
+ for (const log of oldLogs) {
750
+ await audits.delete(log.id);
751
+ }
752
+ }, 24 * 60 * 60 * 1000); // Daily cleanup
753
+ ```
754
+
755
+ ---
756
+
757
+ ## 🔍 FullText Plugin
758
+
759
+ Powerful full-text search engine with automatic indexing, scoring, and advanced search capabilities for your s3db resources.
760
+
761
+ ### ⚡ Quick Start
762
+
763
+ ```javascript
764
+ import { S3db, FullTextPlugin } from 's3db.js';
765
+
766
+ const s3db = new S3db({
767
+ connectionString: "s3://ACCESS_KEY:SECRET_KEY@BUCKET_NAME/databases/myapp",
768
+ plugins: [new FullTextPlugin({
769
+ enabled: true,
770
+ fields: ['title', 'description', 'content']
771
+ })]
772
+ });
773
+
774
+ await s3db.connect();
775
+
776
+ const articles = s3db.resource('articles');
777
+
778
+ // Insert data (automatically indexed)
779
+ await articles.insert({
780
+ title: 'Introduction to Machine Learning',
781
+ description: 'A comprehensive guide to ML basics',
782
+ content: 'Machine learning is a subset of artificial intelligence...'
783
+ });
784
+
785
+ // Search across indexed fields
786
+ const results = await s3db.plugins.fulltext.searchRecords('articles', 'machine learning');
787
+ console.log('Search results:', results);
788
+ ```
789
+
790
+ ### ⚙️ Configuration Parameters
791
+
792
+ | Parameter | Type | Default | Description |
793
+ |-----------|------|---------|-------------|
794
+ | `enabled` | boolean | `true` | Enable/disable full-text search |
795
+ | `fields` | array | `[]` | Fields to index for search |
796
+ | `minWordLength` | number | `3` | Minimum word length for indexing |
797
+ | `maxResults` | number | `100` | Maximum search results to return |
798
+ | `language` | string | `'en-US'` | Language for text processing |
799
+ | `stopWords` | array | `['the', 'a', 'an', ...]` | Words to exclude from indexing |
800
+ | `stemming` | boolean | `false` | Enable word stemming |
801
+ | `caseSensitive` | boolean | `false` | Case-sensitive search |
802
+ | `fuzzySearch` | boolean | `false` | Enable fuzzy matching |
803
+ | `indexName` | string | `'fulltext_indexes'` | Name of index resource |
804
+
805
+ ### Search Result Structure
806
+
807
+ ```javascript
808
+ {
809
+ id: 'article-123',
810
+ title: 'Introduction to Machine Learning',
811
+ description: 'A comprehensive guide to ML basics',
812
+ content: 'Machine learning is a subset...',
813
+ _searchScore: 0.85, // Relevance score (0-1)
814
+ _matchedFields: ['title', 'content'], // Fields with matches
815
+ _matchedWords: ['machine', 'learning'], // Matched search terms
816
+ _highlights: { // Highlighted snippets
817
+ title: 'Introduction to <mark>Machine Learning</mark>',
818
+ content: '<mark>Machine learning</mark> is a subset...'
819
+ }
820
+ }
821
+ ```
822
+
823
+ ### 🔧 Easy Example
824
+
825
+ ```javascript
826
+ import { S3db, FullTextPlugin } from 's3db.js';
827
+
828
+ const s3db = new S3db({
829
+ connectionString: "s3://ACCESS_KEY:SECRET_KEY@BUCKET_NAME/databases/myapp",
830
+ plugins: [new FullTextPlugin({
831
+ enabled: true,
832
+ fields: ['name', 'description', 'tags'],
833
+ minWordLength: 2,
834
+ maxResults: 50
835
+ })]
836
+ });
837
+
838
+ await s3db.connect();
839
+
840
+ const products = s3db.resource('products');
841
+
842
+ // Add products with searchable content
843
+ await products.insertMany([
844
+ {
845
+ name: 'Gaming Laptop Pro',
846
+ description: 'High-performance laptop for gaming and productivity',
847
+ tags: ['gaming', 'laptop', 'computer', 'electronics']
848
+ },
849
+ {
850
+ name: 'Wireless Gaming Mouse',
851
+ description: 'Precision wireless mouse designed for gamers',
852
+ tags: ['gaming', 'mouse', 'wireless', 'electronics']
853
+ },
854
+ {
855
+ name: 'Mechanical Keyboard',
856
+ description: 'Professional mechanical keyboard with RGB lighting',
857
+ tags: ['keyboard', 'mechanical', 'typing', 'electronics']
858
+ }
859
+ ]);
860
+
861
+ // Search for gaming products
862
+ const gamingProducts = await s3db.plugins.fulltext.searchRecords('products', 'gaming');
863
+
864
+ console.log('\n=== Gaming Products ===');
865
+ gamingProducts.forEach(product => {
866
+ console.log(`${product.name} (Score: ${product._searchScore.toFixed(2)})`);
867
+ console.log(` Matched fields: ${product._matchedFields.join(', ')}`);
868
+ console.log(` Description: ${product.description}`);
869
+ });
870
+
871
+ // Search for wireless devices
872
+ const wirelessProducts = await s3db.plugins.fulltext.searchRecords('products', 'wireless');
873
+
874
+ console.log('\n=== Wireless Products ===');
875
+ wirelessProducts.forEach(product => {
876
+ console.log(`${product.name} - ${product.description}`);
877
+ });
878
+
879
+ // Multi-word search
880
+ const laptopGaming = await s3db.plugins.fulltext.searchRecords('products', 'laptop gaming');
881
+
882
+ console.log('\n=== Laptop Gaming Search ===');
883
+ console.log(`Found ${laptopGaming.length} products matching "laptop gaming"`);
884
+ ```
885
+
886
+ ### 🚀 Advanced Configuration Example
887
+
888
+ ```javascript
889
+ import { S3db, FullTextPlugin } from 's3db.js';
890
+
891
+ const s3db = new S3db({
892
+ connectionString: "s3://ACCESS_KEY:SECRET_KEY@BUCKET_NAME/databases/myapp",
893
+ plugins: [new FullTextPlugin({
894
+ enabled: true,
895
+
896
+ // Comprehensive field indexing
897
+ fields: ['title', 'description', 'content', 'tags', 'category', 'author'],
898
+
899
+ // Advanced text processing
900
+ minWordLength: 2,
901
+ maxResults: 200,
902
+ language: 'en-US',
903
+ stemming: true, // Enable word stemming (run/running/ran)
904
+ caseSensitive: false,
905
+ fuzzySearch: true, // Enable typo tolerance
906
+
907
+ // Custom stop words (words to ignore)
908
+ stopWords: [
909
+ 'the', 'a', 'an', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for',
910
+ 'of', 'with', 'by', 'is', 'are', 'was', 'were', 'be', 'been', 'being',
911
+ 'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would', 'could',
912
+ 'should', 'may', 'might', 'must', 'can', 'this', 'that', 'these', 'those'
913
+ ],
914
+
915
+ // Advanced search options
916
+ highlightTags: {
917
+ start: '<mark class="highlight">',
918
+ end: '</mark>'
919
+ },
920
+
921
+ // Custom scoring weights per field
922
+ fieldWeights: {
923
+ title: 3.0, // Title matches score higher
924
+ description: 2.0, // Description is important
925
+ content: 1.0, // Content has normal weight
926
+ tags: 2.5, // Tags are highly relevant
927
+ category: 1.5, // Category is moderately important
928
+ author: 1.0 // Author has normal weight
929
+ },
930
+
931
+ // Indexing behavior
932
+ indexName: 'search_indexes',
933
+ autoReindex: true, // Automatically reindex on data changes
934
+ batchSize: 100, // Index batch size
935
+ maxIndexSize: 10000 // Maximum index entries
936
+ })]
937
+ });
938
+
939
+ await s3db.connect();
940
+
941
+ // Advanced search class with custom methods
942
+ class AdvancedSearch {
943
+ constructor(fulltextPlugin) {
944
+ this.plugin = fulltextPlugin;
945
+ }
946
+
947
+ async searchWithFilters(resourceName, query, filters = {}) {
948
+ let results = await this.plugin.searchRecords(resourceName, query);
949
+
950
+ // Apply additional filters
951
+ if (filters.category) {
952
+ results = results.filter(item => item.category === filters.category);
953
+ }
954
+
955
+ if (filters.minScore) {
956
+ results = results.filter(item => item._searchScore >= filters.minScore);
957
+ }
958
+
959
+ if (filters.dateRange) {
960
+ const { start, end } = filters.dateRange;
961
+ results = results.filter(item => {
962
+ const itemDate = new Date(item.createdAt);
963
+ return itemDate >= start && itemDate <= end;
964
+ });
965
+ }
966
+
967
+ return results;
968
+ }
969
+
970
+ async searchMultipleResources(resourceNames, query) {
971
+ const allResults = [];
972
+
973
+ for (const resourceName of resourceNames) {
974
+ const results = await this.plugin.searchRecords(resourceName, query);
975
+ allResults.push(...results.map(item => ({
976
+ ...item,
977
+ _resourceType: resourceName
978
+ })));
979
+ }
980
+
981
+ // Sort by relevance across all resources
982
+ return allResults.sort((a, b) => b._searchScore - a._searchScore);
983
+ }
984
+
985
+ async suggestWords(resourceName, partial) {
986
+ // Get all indexed words that start with partial
987
+ const allIndexes = await this.plugin.indexResource.list();
988
+
989
+ const suggestions = allIndexes
990
+ .filter(index =>
991
+ index.resourceName === resourceName &&
992
+ index.word.toLowerCase().startsWith(partial.toLowerCase())
993
+ )
994
+ .sort((a, b) => b.count - a.count) // Sort by frequency
995
+ .slice(0, 10)
996
+ .map(index => index.word);
997
+
998
+ return [...new Set(suggestions)]; // Remove duplicates
999
+ }
1000
+
1001
+ async getSearchAnalytics(resourceName) {
1002
+ const indexes = await this.plugin.indexResource.list();
1003
+ const resourceIndexes = indexes.filter(i => i.resourceName === resourceName);
1004
+
1005
+ const analytics = {
1006
+ totalWords: resourceIndexes.length,
1007
+ totalOccurrences: resourceIndexes.reduce((sum, i) => sum + i.count, 0),
1008
+ avgWordsPerDocument: 0,
1009
+ topWords: resourceIndexes
1010
+ .sort((a, b) => b.count - a.count)
1011
+ .slice(0, 20)
1012
+ .map(i => ({ word: i.word, count: i.count })),
1013
+ wordDistribution: {},
1014
+ lastIndexed: Math.max(...resourceIndexes.map(i => new Date(i.lastUpdated)))
1015
+ };
1016
+
1017
+ // Calculate word distribution by frequency ranges
1018
+ resourceIndexes.forEach(index => {
1019
+ const range = index.count < 5 ? 'rare' :
1020
+ index.count < 20 ? 'common' : 'frequent';
1021
+ analytics.wordDistribution[range] = (analytics.wordDistribution[range] || 0) + 1;
1022
+ });
1023
+
1024
+ return analytics;
1025
+ }
1026
+ }
1027
+
1028
+ // Setup sample data
1029
+ const articles = s3db.resource('articles');
1030
+ const products = s3db.resource('products');
1031
+
1032
+ await articles.insertMany([
1033
+ {
1034
+ title: 'Advanced JavaScript Techniques',
1035
+ description: 'Deep dive into modern JavaScript features',
1036
+ content: 'JavaScript has evolved significantly with ES6+ features...',
1037
+ tags: ['javascript', 'programming', 'web-development'],
1038
+ category: 'technology',
1039
+ author: 'John Smith'
1040
+ },
1041
+ {
1042
+ title: 'Machine Learning Fundamentals',
1043
+ description: 'Introduction to ML concepts and algorithms',
1044
+ content: 'Machine learning is revolutionizing how we process data...',
1045
+ tags: ['machine-learning', 'ai', 'data-science'],
1046
+ category: 'technology',
1047
+ author: 'Jane Doe'
1048
+ },
1049
+ {
1050
+ title: 'Sustainable Cooking Tips',
1051
+ description: 'Eco-friendly approaches to home cooking',
1052
+ content: 'Sustainable cooking practices can reduce your environmental impact...',
1053
+ tags: ['cooking', 'sustainability', 'environment'],
1054
+ category: 'lifestyle',
1055
+ author: 'Chef Maria'
1056
+ }
1057
+ ]);
1058
+
1059
+ // Initialize advanced search
1060
+ const search = new AdvancedSearch(s3db.plugins.fulltext);
1061
+
1062
+ // Complex search with filters
1063
+ const techArticles = await search.searchWithFilters('articles', 'javascript programming', {
1064
+ category: 'technology',
1065
+ minScore: 0.5
1066
+ });
1067
+
1068
+ console.log('\n=== Technology Articles ===');
1069
+ techArticles.forEach(article => {
1070
+ console.log(`${article.title} by ${article.author}`);
1071
+ console.log(` Score: ${article._searchScore.toFixed(3)}`);
1072
+ console.log(` Matches: ${article._matchedWords.join(', ')}`);
1073
+ console.log(` Highlighted: ${article._highlights?.title || article.title}`);
1074
+ });
1075
+
1076
+ // Multi-resource search
1077
+ const allContent = await search.searchMultipleResources(['articles', 'products'], 'technology');
1078
+
1079
+ console.log('\n=== Cross-Resource Search ===');
1080
+ allContent.forEach(item => {
1081
+ console.log(`[${item._resourceType.toUpperCase()}] ${item.title || item.name}`);
1082
+ console.log(` Score: ${item._searchScore.toFixed(3)}`);
1083
+ });
1084
+
1085
+ // Auto-complete suggestions
1086
+ const suggestions = await search.suggestWords('articles', 'java');
1087
+ console.log('\nSuggestions for "java":', suggestions);
1088
+
1089
+ // Search analytics
1090
+ const analytics = await search.getSearchAnalytics('articles');
1091
+ console.log('\n=== Search Analytics ===');
1092
+ console.log(`Total indexed words: ${analytics.totalWords}`);
1093
+ console.log(`Total word occurrences: ${analytics.totalOccurrences}`);
1094
+ console.log('Top words:', analytics.topWords.slice(0, 5));
1095
+ console.log('Word distribution:', analytics.wordDistribution);
1096
+
1097
+ // Real-time search monitoring
1098
+ s3db.plugins.fulltext.on('indexed', (data) => {
1099
+ console.log(`🔍 Indexed: ${data.resourceName} - ${data.recordId}`);
1100
+ });
1101
+
1102
+ s3db.plugins.fulltext.on('searched', (data) => {
1103
+ console.log(`🔎 Search: "${data.query}" in ${data.resourceName} (${data.results} results)`);
1104
+ });
1105
+
1106
+ // Performance monitoring
1107
+ console.time('Search Performance');
1108
+ const perfResults = await s3db.plugins.fulltext.searchRecords('articles', 'machine learning javascript');
1109
+ console.timeEnd('Search Performance');
1110
+ console.log(`Search returned ${perfResults.length} results`);
1111
+ ```
1112
+
1113
+ ---
1114
+
1115
+ ## 📊 Metrics Plugin
1116
+
1117
+ Comprehensive performance monitoring and usage analytics system that tracks operation timing, resource usage, errors, and provides detailed insights.
1118
+
1119
+ ### ⚡ Quick Start
1120
+
1121
+ ```javascript
1122
+ import { S3db, MetricsPlugin } from 's3db.js';
1123
+
1124
+ const s3db = new S3db({
1125
+ connectionString: "s3://ACCESS_KEY:SECRET_KEY@BUCKET_NAME/databases/myapp",
1126
+ plugins: [new MetricsPlugin({ enabled: true })]
1127
+ });
1128
+
1129
+ await s3db.connect();
1130
+
1131
+ // Use your database normally - metrics are collected automatically
1132
+ const users = s3db.resource('users');
1133
+ await users.insert({ name: 'John', email: 'john@example.com' });
1134
+ await users.list();
1135
+ await users.count();
1136
+
1137
+ // Get comprehensive metrics
1138
+ const metrics = await s3db.plugins.metrics.getMetrics();
1139
+ console.log('Performance metrics:', metrics);
1140
+ ```
1141
+
1142
+ ### ⚙️ Configuration Parameters
1143
+
1144
+ | Parameter | Type | Default | Description |
1145
+ |-----------|------|---------|-------------|
1146
+ | `enabled` | boolean | `true` | Enable/disable metrics collection |
1147
+ | `collectPerformance` | boolean | `true` | Track operation timing and performance |
1148
+ | `collectErrors` | boolean | `true` | Track errors and failures |
1149
+ | `collectUsage` | boolean | `true` | Track resource usage patterns |
1150
+ | `retentionDays` | number | `30` | Days to retain metric data |
1151
+ | `flushInterval` | number | `60000` | Interval to flush metrics (ms) |
1152
+ | `sampleRate` | number | `1.0` | Sampling rate for metrics (0.0-1.0) |
1153
+ | `trackSlowQueries` | boolean | `true` | Track slow operations |
1154
+ | `slowQueryThreshold` | number | `1000` | Threshold for slow queries (ms) |
1155
+ | `batchSize` | number | `100` | Batch size for metric storage |
1156
+
1157
+ ### Metrics Data Structure
1158
+
1159
+ ```javascript
1160
+ {
1161
+ performance: {
1162
+ averageResponseTime: 245, // milliseconds
1163
+ totalRequests: 1250,
1164
+ requestsPerSecond: 12.5,
1165
+ slowestOperations: [
1166
+ { operation: "list", resource: "users", avgTime: 450, count: 50 },
1167
+ { operation: "get", resource: "products", avgTime: 320, count: 200 }
1168
+ ],
1169
+ operationTiming: {
1170
+ insert: { avg: 180, min: 120, max: 350, total: 50 },
1171
+ update: { avg: 160, min: 90, max: 280, total: 30 },
1172
+ get: { avg: 95, min: 45, max: 180, total: 200 }
1173
+ }
1174
+ },
1175
+ usage: {
1176
+ resources: {
1177
+ users: { inserts: 150, updates: 75, deletes: 10, reads: 800 },
1178
+ products: { inserts: 300, updates: 120, deletes: 25, reads: 1200 }
1179
+ },
1180
+ totalOperations: 2680,
1181
+ mostActiveResource: "products",
1182
+ peakUsageHour: "14:00",
1183
+ dailyPatterns: { /* hourly usage data */ }
1184
+ },
1185
+ errors: {
1186
+ total: 15,
1187
+ byType: {
1188
+ "ValidationError": 8,
1189
+ "NotFoundError": 5,
1190
+ "PermissionError": 2
1191
+ },
1192
+ byResource: { users: 10, products: 5 },
1193
+ errorRate: 0.0056 // 0.56%
1194
+ },
1195
+ cache: {
1196
+ hitRate: 0.78,
1197
+ totalHits: 980,
1198
+ totalMisses: 270
1199
+ }
1200
+ }
1201
+ ```
1202
+
1203
+ ### 🔧 Easy Example
1204
+
1205
+ ```javascript
1206
+ import { S3db, MetricsPlugin } from 's3db.js';
1207
+
1208
+ const s3db = new S3db({
1209
+ connectionString: "s3://ACCESS_KEY:SECRET_KEY@BUCKET_NAME/databases/myapp",
1210
+ plugins: [new MetricsPlugin({
1211
+ enabled: true,
1212
+ collectPerformance: true,
1213
+ collectErrors: true,
1214
+ flushInterval: 30000 // 30 seconds
1215
+ })]
1216
+ });
1217
+
1218
+ await s3db.connect();
1219
+
1220
+ const orders = s3db.resource('orders');
1221
+
1222
+ // Simulate various operations
1223
+ console.log('Performing operations...');
1224
+
1225
+ // Fast operations
1226
+ for (let i = 0; i < 10; i++) {
1227
+ await orders.insert({
1228
+ customerId: `customer-${i}`,
1229
+ amount: Math.random() * 1000,
1230
+ status: 'pending'
1231
+ });
1232
+ }
1233
+
1234
+ // Query operations
1235
+ await orders.count();
1236
+ await orders.list({ limit: 5 });
1237
+
1238
+ // Some updates
1239
+ const orderList = await orders.list({ limit: 3 });
1240
+ for (const order of orderList) {
1241
+ await orders.update(order.id, { status: 'processing' });
1242
+ }
1243
+
1244
+ // Get performance metrics
1245
+ const metrics = await s3db.plugins.metrics.getMetrics();
1246
+
1247
+ console.log('\n=== Performance Report ===');
1248
+ console.log(`Average response time: ${metrics.performance.averageResponseTime}ms`);
1249
+ console.log(`Total operations: ${metrics.usage.totalOperations}`);
1250
+ console.log(`Error rate: ${(metrics.errors.errorRate * 100).toFixed(2)}%`);
1251
+
1252
+ console.log('\n=== Operation Breakdown ===');
1253
+ Object.entries(metrics.performance.operationTiming).forEach(([op, timing]) => {
1254
+ console.log(`${op.toUpperCase()}: avg ${timing.avg}ms (${timing.total} operations)`);
1255
+ });
1256
+
1257
+ console.log('\n=== Resource Usage ===');
1258
+ Object.entries(metrics.usage.resources).forEach(([resource, usage]) => {
1259
+ const total = Object.values(usage).reduce((sum, count) => sum + count, 0);
1260
+ console.log(`${resource}: ${total} total operations`);
1261
+ });
1262
+ ```
1263
+
1264
+ ### 🚀 Advanced Configuration Example
1265
+
1266
+ ```javascript
1267
+ import { S3db, MetricsPlugin } from 's3db.js';
1268
+
1269
+ const s3db = new S3db({
1270
+ connectionString: "s3://ACCESS_KEY:SECRET_KEY@BUCKET_NAME/databases/myapp",
1271
+ plugins: [new MetricsPlugin({
1272
+ enabled: true,
1273
+
1274
+ // Comprehensive monitoring
1275
+ collectPerformance: true,
1276
+ collectErrors: true,
1277
+ collectUsage: true,
1278
+
1279
+ // Advanced settings
1280
+ retentionDays: 90, // 3 months of data
1281
+ flushInterval: 10000, // 10 seconds
1282
+ sampleRate: 1.0, // 100% sampling
1283
+
1284
+ // Performance thresholds
1285
+ trackSlowQueries: true,
1286
+ slowQueryThreshold: 500, // 500ms threshold
1287
+
1288
+ // Storage optimization
1289
+ batchSize: 50,
1290
+ compressionEnabled: true,
1291
+
1292
+ // Custom alerting thresholds
1293
+ alertThresholds: {
1294
+ errorRate: 0.05, // 5% error rate
1295
+ avgResponseTime: 1000, // 1 second average
1296
+ memoryUsage: 0.9 // 90% memory usage
1297
+ },
1298
+
1299
+ // Event hooks
1300
+ onSlowQuery: (operation, resource, duration) => {
1301
+ console.warn(`🐌 Slow query: ${operation} on ${resource} took ${duration}ms`);
1302
+ },
1303
+
1304
+ onHighErrorRate: (resource, errorRate) => {
1305
+ console.error(`🚨 High error rate: ${resource} has ${(errorRate * 100).toFixed(1)}% errors`);
1306
+ },
1307
+
1308
+ onThresholdExceeded: (metric, value, threshold) => {
1309
+ console.warn(`⚠️ Threshold exceeded: ${metric} = ${value} (threshold: ${threshold})`);
1310
+ }
1311
+ })]
1312
+ });
1313
+
1314
+ await s3db.connect();
1315
+
1316
+ // Advanced metrics analysis class
1317
+ class MetricsAnalyzer {
1318
+ constructor(metricsPlugin) {
1319
+ this.plugin = metricsPlugin;
1320
+ this.alertHandlers = new Map();
1321
+ }
1322
+
1323
+ addAlertHandler(condition, handler) {
1324
+ this.alertHandlers.set(condition, handler);
1325
+ }
1326
+
1327
+ async analyzePerformance(timeRange = 3600000) { // 1 hour
1328
+ const metrics = await this.plugin.getMetrics();
1329
+ const analysis = {
1330
+ summary: {
1331
+ totalOperations: metrics.usage.totalOperations,
1332
+ avgResponseTime: metrics.performance.averageResponseTime,
1333
+ errorRate: metrics.errors.errorRate,
1334
+ slowQueries: metrics.performance.slowestOperations.length
1335
+ },
1336
+ recommendations: [],
1337
+ alerts: []
1338
+ };
1339
+
1340
+ // Performance analysis
1341
+ if (metrics.performance.averageResponseTime > 500) {
1342
+ analysis.recommendations.push({
1343
+ type: 'performance',
1344
+ message: 'Average response time is high. Consider adding caching or optimizing queries.',
1345
+ priority: 'high'
1346
+ });
1347
+ }
1348
+
1349
+ // Error rate analysis
1350
+ if (metrics.errors.errorRate > 0.02) { // 2%
1351
+ analysis.alerts.push({
1352
+ type: 'error_rate',
1353
+ message: `Error rate (${(metrics.errors.errorRate * 100).toFixed(2)}%) exceeds threshold`,
1354
+ severity: 'warning'
1355
+ });
1356
+ }
1357
+
1358
+ // Resource usage patterns
1359
+ const resourceUsage = Object.entries(metrics.usage.resources);
1360
+ const imbalancedResources = resourceUsage.filter(([name, usage]) => {
1361
+ const writes = usage.inserts + usage.updates + usage.deletes;
1362
+ const reads = usage.reads;
1363
+ return writes > 0 && (reads / writes) < 0.1; // Very low read/write ratio
1364
+ });
1365
+
1366
+ if (imbalancedResources.length > 0) {
1367
+ analysis.recommendations.push({
1368
+ type: 'usage_pattern',
1369
+ message: `Resources with low read/write ratio: ${imbalancedResources.map(([name]) => name).join(', ')}`,
1370
+ priority: 'medium'
1371
+ });
1372
+ }
1373
+
1374
+ return analysis;
1375
+ }
1376
+
1377
+ async generateReport(format = 'console') {
1378
+ const metrics = await this.plugin.getMetrics();
1379
+ const analysis = await this.analyzePerformance();
1380
+
1381
+ if (format === 'console') {
1382
+ console.log('\n=== 📊 COMPREHENSIVE METRICS REPORT ===');
1383
+
1384
+ // Performance Summary
1385
+ console.log('\n🚀 Performance Summary:');
1386
+ console.log(` Total Operations: ${analysis.summary.totalOperations.toLocaleString()}`);
1387
+ console.log(` Average Response Time: ${analysis.summary.avgResponseTime}ms`);
1388
+ console.log(` Error Rate: ${(analysis.summary.errorRate * 100).toFixed(2)}%`);
1389
+ console.log(` Slow Queries: ${analysis.summary.slowQueries}`);
1390
+
1391
+ // Operation Breakdown
1392
+ console.log('\n⏱️ Operation Timing:');
1393
+ Object.entries(metrics.performance.operationTiming).forEach(([op, timing]) => {
1394
+ console.log(` ${op.toUpperCase()}:`);
1395
+ console.log(` Average: ${timing.avg}ms`);
1396
+ console.log(` Range: ${timing.min}ms - ${timing.max}ms`);
1397
+ console.log(` Count: ${timing.total}`);
1398
+ });
1399
+
1400
+ // Resource Activity
1401
+ console.log('\n📈 Resource Activity:');
1402
+ Object.entries(metrics.usage.resources)
1403
+ .sort(([,a], [,b]) => {
1404
+ const totalA = Object.values(a).reduce((sum, val) => sum + val, 0);
1405
+ const totalB = Object.values(b).reduce((sum, val) => sum + val, 0);
1406
+ return totalB - totalA;
1407
+ })
1408
+ .forEach(([resource, usage]) => {
1409
+ const total = Object.values(usage).reduce((sum, val) => sum + val, 0);
1410
+ console.log(` ${resource}: ${total} operations`);
1411
+ console.log(` Reads: ${usage.reads}, Writes: ${usage.inserts + usage.updates + usage.deletes}`);
1412
+ });
1413
+
1414
+ // Error Analysis
1415
+ if (metrics.errors.total > 0) {
1416
+ console.log('\n🚨 Error Analysis:');
1417
+ console.log(` Total Errors: ${metrics.errors.total}`);
1418
+ console.log(' By Type:');
1419
+ Object.entries(metrics.errors.byType).forEach(([type, count]) => {
1420
+ console.log(` ${type}: ${count}`);
1421
+ });
1422
+ }
1423
+
1424
+ // Recommendations
1425
+ if (analysis.recommendations.length > 0) {
1426
+ console.log('\n💡 Recommendations:');
1427
+ analysis.recommendations.forEach(rec => {
1428
+ const emoji = rec.priority === 'high' ? '🔴' : rec.priority === 'medium' ? '🟡' : '🟢';
1429
+ console.log(` ${emoji} [${rec.priority.toUpperCase()}] ${rec.message}`);
1430
+ });
1431
+ }
1432
+
1433
+ // Alerts
1434
+ if (analysis.alerts.length > 0) {
1435
+ console.log('\n⚠️ Active Alerts:');
1436
+ analysis.alerts.forEach(alert => {
1437
+ console.log(` 🚨 ${alert.message}`);
1438
+ });
1439
+ }
1440
+ }
1441
+
1442
+ return { metrics, analysis };
1443
+ }
1444
+
1445
+ async exportMetrics(filename) {
1446
+ const metrics = await this.plugin.getMetrics();
1447
+ const data = {
1448
+ timestamp: new Date().toISOString(),
1449
+ metrics,
1450
+ analysis: await this.analyzePerformance()
1451
+ };
1452
+
1453
+ // In real implementation, save to file
1454
+ console.log(`📁 Metrics exported to ${filename}`);
1455
+ return data;
1456
+ }
1457
+
1458
+ startRealTimeMonitoring(interval = 5000) {
1459
+ const monitor = setInterval(async () => {
1460
+ const metrics = await this.plugin.getMetrics();
1461
+
1462
+ // Check alert conditions
1463
+ this.alertHandlers.forEach((handler, condition) => {
1464
+ if (condition(metrics)) {
1465
+ handler(metrics);
1466
+ }
1467
+ });
1468
+
1469
+ // Auto-optimization suggestions
1470
+ if (metrics.performance.averageResponseTime > 1000) {
1471
+ console.log('💡 Suggestion: Consider implementing caching for frequently accessed data');
1472
+ }
1473
+
1474
+ if (metrics.errors.errorRate > 0.05) {
1475
+ console.log('🚨 Alert: Error rate is above 5% - investigate immediately');
1476
+ }
1477
+
1478
+ }, interval);
1479
+
1480
+ return monitor;
1481
+ }
1482
+ }
1483
+
1484
+ // Simulate complex workload
1485
+ const users = s3db.resource('users');
1486
+ const products = s3db.resource('products');
1487
+ const orders = s3db.resource('orders');
1488
+
1489
+ // Setup metrics analyzer
1490
+ const analyzer = new MetricsAnalyzer(s3db.plugins.metrics);
1491
+
1492
+ // Add custom alert handlers
1493
+ analyzer.addAlertHandler(
1494
+ (metrics) => metrics.errors.errorRate > 0.03,
1495
+ (metrics) => console.log('🚨 Error rate alert triggered!')
1496
+ );
1497
+
1498
+ analyzer.addAlertHandler(
1499
+ (metrics) => metrics.performance.averageResponseTime > 800,
1500
+ (metrics) => console.log('⏰ Performance degradation detected!')
1501
+ );
1502
+
1503
+ // Simulate workload
1504
+ console.log('🔄 Simulating complex workload...');
1505
+
1506
+ // Bulk operations
1507
+ const userData = Array.from({ length: 50 }, (_, i) => ({
1508
+ name: `User ${i}`,
1509
+ email: `user${i}@example.com`,
1510
+ role: i % 3 === 0 ? 'admin' : 'user'
1511
+ }));
1512
+
1513
+ await users.insertMany(userData);
1514
+
1515
+ // Mixed operations with some errors
1516
+ for (let i = 0; i < 20; i++) {
1517
+ try {
1518
+ await products.insert({
1519
+ name: `Product ${i}`,
1520
+ price: Math.random() * 100,
1521
+ category: ['electronics', 'books', 'clothing'][i % 3]
1522
+ });
1523
+
1524
+ if (i % 5 === 0) {
1525
+ // Simulate some slow operations
1526
+ await new Promise(resolve => setTimeout(resolve, 600));
1527
+ await products.list({ limit: 20 });
1528
+ }
1529
+
1530
+ if (i % 10 === 0) {
1531
+ // Simulate some errors
1532
+ try {
1533
+ await products.get('non-existent-id');
1534
+ } catch (error) {
1535
+ // Expected error for testing
1536
+ }
1537
+ }
1538
+
1539
+ } catch (error) {
1540
+ // Handle errors
1541
+ }
1542
+ }
1543
+
1544
+ // Generate comprehensive report
1545
+ await analyzer.generateReport();
1546
+
1547
+ // Start real-time monitoring
1548
+ const monitor = analyzer.startRealTimeMonitoring(3000);
1549
+
1550
+ // Export metrics for external analysis
1551
+ await analyzer.exportMetrics('metrics-export.json');
1552
+
1553
+ // Stop monitoring after demo
1554
+ setTimeout(() => {
1555
+ clearInterval(monitor);
1556
+ console.log('\n✅ Metrics demonstration completed');
1557
+ }, 15000);
1558
+ ```
1559
+
1560
+ ---
1561
+
1562
+ ## 🔄 Replicator Plugin
1563
+
1564
+ Powerful data replication system that synchronizes your s3db data to multiple targets including other S3DB instances, SQS queues, BigQuery, and PostgreSQL databases.
1565
+
1566
+ ### ⚡ Quick Start
1567
+
1568
+ ```javascript
1569
+ import { S3db, ReplicatorPlugin } from 's3db.js';
1570
+
1571
+ const s3db = new S3db({
1572
+ connectionString: "s3://ACCESS_KEY:SECRET_KEY@BUCKET_NAME/databases/myapp",
1573
+ plugins: [new ReplicatorPlugin({
1574
+ replicators: [
1575
+ {
1576
+ driver: 's3db',
1577
+ resources: ['users'],
1578
+ config: {
1579
+ connectionString: "s3://BACKUP_KEY:BACKUP_SECRET@BACKUP_BUCKET/backup"
1580
+ }
1581
+ }
1582
+ ]
1583
+ })]
1584
+ });
1585
+
1586
+ await s3db.connect();
1587
+
1588
+ // Data is automatically replicated
1589
+ const users = s3db.resource('users');
1590
+ await users.insert({ name: 'John', email: 'john@example.com' });
1591
+ // This insert is automatically replicated to the backup database
1592
+ ```
1593
+
1594
+ ### ⚙️ Configuration Parameters
1595
+
1596
+ | Parameter | Type | Default | Description |
1597
+ |-----------|------|---------|-------------|
1598
+ | `enabled` | boolean | `true` | Enable/disable replication |
1599
+ | `replicators` | array | `[]` | Array of replicator configurations |
1600
+ | `persistReplicatorLog` | boolean | `false` | Store replication logs in database |
1601
+ | `replicatorLogResource` | string | `'replicator_logs'` | Name of log resource |
1602
+ | `batchSize` | number | `10` | Batch size for bulk operations |
1603
+ | `retryAttempts` | number | `3` | Retry failed replications |
1604
+ | `retryDelay` | number | `1000` | Delay between retries (ms) |
1605
+ | `syncInterval` | number | `0` | Auto-sync interval (0 = disabled) |
1606
+
1607
+ ### Replicator Drivers
1608
+
1609
+ #### S3DB Replicator
1610
+
1611
+ Replicate to another S3DB instance:
1612
+
1613
+ ```javascript
1614
+ {
1615
+ driver: 's3db',
1616
+ resources: ['users', 'products'],
1617
+ config: {
1618
+ connectionString: "s3://BACKUP_KEY:BACKUP_SECRET@BACKUP_BUCKET/backup"
1619
+ }
1620
+ }
1621
+ ```
1622
+
1623
+ #### SQS Replicator
1624
+
1625
+ Send changes to AWS SQS queues:
1626
+
1627
+ ```javascript
1628
+ {
1629
+ driver: 'sqs',
1630
+ resources: ['orders'],
1631
+ config: {
1632
+ queueUrl: 'https://sqs.us-east-1.amazonaws.com/123456789012/my-queue',
1633
+ region: 'us-east-1',
1634
+ messageGroupId: 's3db-replicator',
1635
+ deduplicationId: true
1636
+ }
1637
+ }
1638
+ ```
1639
+
1640
+ #### BigQuery Replicator
1641
+
1642
+ Replicate to Google BigQuery:
1643
+
1644
+ ```javascript
1645
+ {
1646
+ driver: 'bigquery',
1647
+ resources: {
1648
+ users: [{ actions: ['insert', 'update'], table: 'users_table' }],
1649
+ orders: 'orders_table'
1650
+ },
1651
+ config: {
1652
+ projectId: 'my-project',
1653
+ datasetId: 'analytics',
1654
+ credentials: { /* service account */ }
1655
+ }
1656
+ }
1657
+ ```
1658
+
1659
+ #### PostgreSQL Replicator
1660
+
1661
+ Replicate to PostgreSQL database:
1662
+
1663
+ ```javascript
1664
+ {
1665
+ driver: 'postgres',
1666
+ resources: {
1667
+ users: [{ actions: ['insert', 'update', 'delete'], table: 'users_table' }]
1668
+ },
1669
+ config: {
1670
+ connectionString: 'postgresql://user:pass@localhost:5432/analytics'
1671
+ }
1672
+ }
1673
+ ```
1674
+
1675
+ ### Resource Configuration Formats
1676
+
1677
+ Multiple formats supported for resource mapping:
1678
+
1679
+ ```javascript
1680
+ // 1. Simple array (replicate to same name)
1681
+ resources: ['users', 'products']
1682
+
1683
+ // 2. Object mapping (source → destination)
1684
+ resources: { users: 'people', products: 'items' }
1685
+
1686
+ // 3. Advanced mapping with transformers
1687
+ resources: {
1688
+ users: [
1689
+ {
1690
+ resource: 'people',
1691
+ transformer: (data) => ({ ...data, fullName: `${data.first} ${data.last}` })
1692
+ }
1693
+ ]
1694
+ }
1695
+
1696
+ // 4. Action-specific configuration (BigQuery/PostgreSQL)
1697
+ resources: {
1698
+ users: [
1699
+ { actions: ['insert', 'update'], table: 'users_table' },
1700
+ { actions: ['insert'], table: 'users_analytics' }
1701
+ ]
1702
+ }
1703
+ ```
1704
+
1705
+ ### 🔧 Easy Example
1706
+
1707
+ ```javascript
1708
+ import { S3db, ReplicatorPlugin } from 's3db.js';
1709
+
1710
+ const s3db = new S3db({
1711
+ connectionString: "s3://ACCESS_KEY:SECRET_KEY@BUCKET_NAME/databases/myapp",
1712
+ plugins: [new ReplicatorPlugin({
1713
+ persistReplicatorLog: true,
1714
+ replicators: [
1715
+ {
1716
+ driver: 's3db',
1717
+ resources: ['users', 'products'],
1718
+ config: {
1719
+ connectionString: "s3://BACKUP_KEY:BACKUP_SECRET@BACKUP_BUCKET/backup"
1720
+ }
1721
+ },
1722
+ {
1723
+ driver: 'sqs',
1724
+ resources: ['orders'],
1725
+ config: {
1726
+ queueUrl: 'https://sqs.us-east-1.amazonaws.com/123456789012/orders-queue.fifo',
1727
+ region: 'us-east-1',
1728
+ messageGroupId: 'order-updates'
1729
+ }
1730
+ }
1731
+ ]
1732
+ })]
1733
+ });
1734
+
1735
+ await s3db.connect();
1736
+
1737
+ const users = s3db.resource('users');
1738
+ const orders = s3db.resource('orders');
1739
+
1740
+ // Monitor replication events
1741
+ const replicatorPlugin = s3db.plugins.find(p => p.constructor.name === 'ReplicatorPlugin');
1742
+
1743
+ replicatorPlugin.on('replicator.success', (data) => {
1744
+ console.log(`✅ Replicated: ${data.action} on ${data.resource} to ${data.replicator}`);
1745
+ });
1746
+
1747
+ replicatorPlugin.on('replicator.failed', (data) => {
1748
+ console.error(`❌ Replication failed: ${data.error}`);
1749
+ });
1750
+
1751
+ // Insert data (automatically replicated)
1752
+ const user = await users.insert({
1753
+ name: 'Alice Johnson',
1754
+ email: 'alice@example.com',
1755
+ role: 'customer'
1756
+ });
1757
+
1758
+ const order = await orders.insert({
1759
+ userId: user.id,
1760
+ amount: 99.99,
1761
+ items: ['item1', 'item2']
1762
+ });
1763
+
1764
+ // Check replication logs
1765
+ const replicatorLogs = s3db.resource('replicator_logs');
1766
+ const logs = await replicatorLogs.list();
1767
+
1768
+ console.log('\n=== Replication History ===');
1769
+ logs.forEach(log => {
1770
+ console.log(`${log.timestamp}: ${log.action} ${log.resource} → ${log.replicator}`);
1771
+ });
1772
+ ```
1773
+
1774
+ ### 🚀 Advanced Multi-Driver Example
1775
+
1776
+ ```javascript
1777
+ import { S3db, ReplicatorPlugin } from 's3db.js';
1778
+
1779
+ const s3db = new S3db({
1780
+ connectionString: "s3://ACCESS_KEY:SECRET_KEY@BUCKET_NAME/databases/myapp",
1781
+ plugins: [new ReplicatorPlugin({
1782
+ enabled: true,
1783
+ persistReplicatorLog: true,
1784
+ replicatorLogResource: 'replication_audit',
1785
+ batchSize: 25,
1786
+ retryAttempts: 5,
1787
+ retryDelay: 2000,
1788
+
1789
+ replicators: [
1790
+ // Backup to another S3DB instance
1791
+ {
1792
+ driver: 's3db',
1793
+ resources: ['users', 'products', 'orders'],
1794
+ config: {
1795
+ connectionString: "s3://BACKUP_KEY:BACKUP_SECRET@BACKUP_BUCKET/backup",
1796
+ enabled: true,
1797
+ timeout: 30000
1798
+ }
1799
+ },
1800
+
1801
+ // Real-time events to SQS
1802
+ {
1803
+ driver: 'sqs',
1804
+ resources: ['orders', 'users'],
1805
+ config: {
1806
+ region: 'us-east-1',
1807
+ credentials: {
1808
+ accessKeyId: process.env.AWS_ACCESS_KEY_ID,
1809
+ secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY
1810
+ },
1811
+ // Resource-specific queues
1812
+ queues: {
1813
+ orders: 'https://sqs.us-east-1.amazonaws.com/123456789012/order-events.fifo',
1814
+ users: 'https://sqs.us-east-1.amazonaws.com/123456789012/user-events.fifo'
1815
+ },
1816
+ // Default queue for unspecified resources
1817
+ defaultQueueUrl: 'https://sqs.us-east-1.amazonaws.com/123456789012/default-events.fifo',
1818
+ messageGroupId: 's3db-replicator',
1819
+ deduplicationId: true,
1820
+ messageAttributes: {
1821
+ source: { StringValue: 'production-db', DataType: 'String' },
1822
+ version: { StringValue: '1.0', DataType: 'String' }
1823
+ }
1824
+ }
1825
+ },
1826
+
1827
+ // Analytics to BigQuery
1828
+ {
1829
+ driver: 'bigquery',
1830
+ config: {
1831
+ projectId: 'my-analytics-project',
1832
+ datasetId: 's3db_analytics',
1833
+ location: 'US',
1834
+ logTable: 'replication_log',
1835
+ credentials: {
1836
+ client_email: 'service-account@project.iam.gserviceaccount.com',
1837
+ private_key: process.env.BIGQUERY_PRIVATE_KEY,
1838
+ project_id: 'my-analytics-project'
1839
+ }
1840
+ },
1841
+ resources: {
1842
+ // Multiple destinations for users
1843
+ users: [
1844
+ { actions: ['insert', 'update'], table: 'dim_users' },
1845
+ { actions: ['insert'], table: 'fact_user_activity' }
1846
+ ],
1847
+
1848
+ // Orders to analytics tables
1849
+ orders: [
1850
+ { actions: ['insert'], table: 'fact_orders' },
1851
+ { actions: ['insert'], table: 'daily_revenue',
1852
+ transformer: (data) => ({
1853
+ date: data.createdAt?.split('T')[0],
1854
+ revenue: data.amount,
1855
+ customer_id: data.userId,
1856
+ order_count: 1
1857
+ })
1858
+ }
1859
+ ],
1860
+
1861
+ // Products with transformation
1862
+ products: {
1863
+ table: 'dim_products',
1864
+ actions: ['insert', 'update'],
1865
+ transformer: (data) => ({
1866
+ ...data,
1867
+ price_category: data.price > 100 ? 'premium' : 'standard',
1868
+ last_updated: new Date().toISOString()
1869
+ })
1870
+ }
1871
+ }
1872
+ },
1873
+
1874
+ // Operational database (PostgreSQL)
1875
+ {
1876
+ driver: 'postgres',
1877
+ config: {
1878
+ connectionString: 'postgresql://analytics:password@localhost:5432/operations',
1879
+ ssl: { rejectUnauthorized: false },
1880
+ logTable: 'replication_log',
1881
+ pool: {
1882
+ max: 20,
1883
+ idleTimeoutMillis: 30000,
1884
+ connectionTimeoutMillis: 2000
1885
+ }
1886
+ },
1887
+ resources: {
1888
+ users: [
1889
+ {
1890
+ actions: ['insert', 'update', 'delete'],
1891
+ table: 'operational_users',
1892
+ transformer: (data, action) => {
1893
+ if (action === 'delete') return { id: data.id, deleted_at: new Date() };
1894
+ return {
1895
+ ...data,
1896
+ sync_timestamp: new Date(),
1897
+ source_system: 's3db'
1898
+ };
1899
+ }
1900
+ }
1901
+ ],
1902
+
1903
+ orders: [
1904
+ { actions: ['insert'], table: 'order_events' },
1905
+ {
1906
+ actions: ['update'],
1907
+ table: 'order_updates',
1908
+ transformer: (data) => ({
1909
+ order_id: data.id,
1910
+ updated_fields: Object.keys(data),
1911
+ update_timestamp: new Date()
1912
+ })
1913
+ }
1914
+ ]
1915
+ }
1916
+ }
1917
+ ]
1918
+ })]
1919
+ });
1920
+
1921
+ await s3db.connect();
1922
+
1923
+ // Advanced replicator management
1924
+ class ReplicatorManager {
1925
+ constructor(replicatorPlugin) {
1926
+ this.plugin = replicatorPlugin;
1927
+ this.stats = {
1928
+ totalReplications: 0,
1929
+ successfulReplications: 0,
1930
+ failedReplications: 0,
1931
+ byReplicator: {},
1932
+ byResource: {}
1933
+ };
1934
+
1935
+ this.setupEventListeners();
1936
+ }
1937
+
1938
+ setupEventListeners() {
1939
+ this.plugin.on('replicator.queued', (data) => {
1940
+ this.stats.totalReplications++;
1941
+ this.updateResourceStats(data.resource, 'queued');
1942
+ });
1943
+
1944
+ this.plugin.on('replicator.success', (data) => {
1945
+ this.stats.successfulReplications++;
1946
+ this.updateReplicatorStats(data.replicator, 'success');
1947
+ this.updateResourceStats(data.resource, 'success');
1948
+ });
1949
+
1950
+ this.plugin.on('replicator.failed', (data) => {
1951
+ this.stats.failedReplications++;
1952
+ this.updateReplicatorStats(data.replicator, 'failed');
1953
+ this.updateResourceStats(data.resource, 'failed');
1954
+
1955
+ // Advanced error handling
1956
+ if (data.error.includes('BigQuery')) {
1957
+ console.log('🔧 BigQuery error detected - checking schema compatibility...');
1958
+ } else if (data.error.includes('SQS')) {
1959
+ console.log('📮 SQS error detected - checking queue permissions...');
1960
+ }
1961
+ });
1962
+ }
1963
+
1964
+ updateReplicatorStats(replicator, status) {
1965
+ if (!this.stats.byReplicator[replicator]) {
1966
+ this.stats.byReplicator[replicator] = { success: 0, failed: 0 };
1967
+ }
1968
+ this.stats.byReplicator[replicator][status]++;
1969
+ }
1970
+
1971
+ updateResourceStats(resource, status) {
1972
+ if (!this.stats.byResource[resource]) {
1973
+ this.stats.byResource[resource] = { queued: 0, success: 0, failed: 0 };
1974
+ }
1975
+ this.stats.byResource[resource][status]++;
1976
+ }
1977
+
1978
+ async getReplicationHealth() {
1979
+ const totalAttempts = this.stats.successfulReplications + this.stats.failedReplications;
1980
+ const successRate = totalAttempts > 0 ? this.stats.successfulReplications / totalAttempts : 1;
1981
+
1982
+ return {
1983
+ overall: {
1984
+ successRate: successRate,
1985
+ totalReplications: this.stats.totalReplications,
1986
+ pending: this.stats.totalReplications - totalAttempts,
1987
+ health: successRate > 0.95 ? 'excellent' :
1988
+ successRate > 0.85 ? 'good' :
1989
+ successRate > 0.7 ? 'warning' : 'critical'
1990
+ },
1991
+ byReplicator: this.stats.byReplicator,
1992
+ byResource: this.stats.byResource
1993
+ };
1994
+ }
1995
+
1996
+ async pauseReplicator(replicatorId) {
1997
+ const replicator = this.plugin.replicators.find(r => r.id === replicatorId);
1998
+ if (replicator) {
1999
+ replicator.enabled = false;
2000
+ console.log(`⏸️ Paused replicator: ${replicatorId}`);
2001
+ }
2002
+ }
2003
+
2004
+ async resumeReplicator(replicatorId) {
2005
+ const replicator = this.plugin.replicators.find(r => r.id === replicatorId);
2006
+ if (replicator) {
2007
+ replicator.enabled = true;
2008
+ console.log(`▶️ Resumed replicator: ${replicatorId}`);
2009
+ }
2010
+ }
2011
+
2012
+ async testReplicatorConnections() {
2013
+ console.log('🔍 Testing replicator connections...');
2014
+
2015
+ for (const replicator of this.plugin.replicators) {
2016
+ try {
2017
+ const result = await replicator.testConnection();
2018
+ console.log(`✅ ${replicator.driver}: ${result.status}`);
2019
+ } catch (error) {
2020
+ console.log(`❌ ${replicator.driver}: ${error.message}`);
2021
+ }
2022
+ }
2023
+ }
2024
+ }
2025
+
2026
+ // Setup sample data and test all replicators
2027
+ const users = s3db.resource('users');
2028
+ const products = s3db.resource('products');
2029
+ const orders = s3db.resource('orders');
2030
+
2031
+ const replicatorPlugin = s3db.plugins.find(p => p.constructor.name === 'ReplicatorPlugin');
2032
+ const manager = new ReplicatorManager(replicatorPlugin);
2033
+
2034
+ // Test connections
2035
+ await manager.testReplicatorConnections();
2036
+
2037
+ // Create sample data
2038
+ console.log('🔄 Creating sample data with multi-driver replication...');
2039
+
2040
+ const sampleUsers = await users.insertMany([
2041
+ { name: 'John Smith', email: 'john@example.com', role: 'admin' },
2042
+ { name: 'Jane Doe', email: 'jane@example.com', role: 'user' },
2043
+ { name: 'Bob Wilson', email: 'bob@example.com', role: 'user' }
2044
+ ]);
2045
+
2046
+ const sampleProducts = await products.insertMany([
2047
+ { name: 'Laptop Pro', price: 1299.99, category: 'electronics' },
2048
+ { name: 'Wireless Mouse', price: 29.99, category: 'electronics' },
2049
+ { name: 'Coffee Mug', price: 12.99, category: 'home' }
2050
+ ]);
2051
+
2052
+ const sampleOrders = await orders.insertMany([
2053
+ { userId: sampleUsers[0].id, amount: 1329.98, items: [sampleProducts[0].id, sampleProducts[1].id] },
2054
+ { userId: sampleUsers[1].id, amount: 29.99, items: [sampleProducts[1].id] },
2055
+ { userId: sampleUsers[2].id, amount: 12.99, items: [sampleProducts[2].id] }
2056
+ ]);
2057
+
2058
+ // Wait for replications to complete
2059
+ await new Promise(resolve => setTimeout(resolve, 3000));
2060
+
2061
+ // Get replication statistics
2062
+ const health = await manager.getReplicationHealth();
2063
+ console.log('\n=== Replication Health Report ===');
2064
+ console.log(`Overall success rate: ${(health.overall.successRate * 100).toFixed(1)}%`);
2065
+ console.log(`Health status: ${health.overall.health.toUpperCase()}`);
2066
+ console.log(`Total replications: ${health.overall.totalReplications}`);
2067
+ console.log(`Pending: ${health.overall.pending}`);
2068
+
2069
+ console.log('\n=== By Replicator ===');
2070
+ Object.entries(health.byReplicator).forEach(([replicator, stats]) => {
2071
+ const total = stats.success + stats.failed;
2072
+ const rate = total > 0 ? (stats.success / total * 100).toFixed(1) : 0;
2073
+ console.log(`${replicator}: ${rate}% success (${stats.success}/${total})`);
2074
+ });
2075
+
2076
+ console.log('\n=== By Resource ===');
2077
+ Object.entries(health.byResource).forEach(([resource, stats]) => {
2078
+ console.log(`${resource}: queued ${stats.queued}, success ${stats.success}, failed ${stats.failed}`);
2079
+ });
2080
+
2081
+ // Get detailed replication logs
2082
+ const replicationLogs = await replicatorPlugin.getReplicatorLogs({ limit: 10 });
2083
+ console.log('\n=== Recent Replication Logs ===');
2084
+ replicationLogs.forEach(log => {
2085
+ const status = log.success ? '✅' : '❌';
2086
+ console.log(`${status} ${log.timestamp} | ${log.action} ${log.resource} → ${log.replicator}`);
2087
+ if (!log.success && log.error) {
2088
+ console.log(` Error: ${log.error}`);
2089
+ }
2090
+ });
2091
+
2092
+ console.log('\n✅ Multi-driver replication demonstration completed');
2093
+ ```
2094
+
2095
+ ---
2096
+
2097
+ ## 📬 Queue Consumer Plugin
2098
+
2099
+ Consume messages from external queues (SQS, RabbitMQ) and automatically process them into your s3db resources.
2100
+
2101
+ ### ⚡ Quick Start
2102
+
2103
+ ```javascript
2104
+ import { S3db, QueueConsumerPlugin } from 's3db.js';
2105
+
2106
+ const s3db = new S3db({
2107
+ connectionString: "s3://ACCESS_KEY:SECRET_KEY@BUCKET_NAME/databases/myapp",
2108
+ plugins: [new QueueConsumerPlugin({
2109
+ consumers: [
2110
+ {
2111
+ driver: 'sqs',
2112
+ config: {
2113
+ queueUrl: 'https://sqs.us-east-1.amazonaws.com/123456789012/my-queue',
2114
+ region: 'us-east-1'
2115
+ },
2116
+ consumers: [
2117
+ { resources: 'users' }
2118
+ ]
2119
+ }
2120
+ ]
2121
+ })]
2122
+ });
2123
+
2124
+ await s3db.connect();
2125
+ // Queue messages are automatically processed into your resources
2126
+ ```
2127
+
2128
+ ### ⚙️ Configuration Parameters
2129
+
2130
+ | Parameter | Type | Default | Description |
2131
+ |-----------|------|---------|-------------|
2132
+ | `enabled` | boolean | `true` | Enable/disable queue consumption |
2133
+ | `consumers` | array | `[]` | Array of consumer configurations |
2134
+ | `batchSize` | number | `10` | Messages to process per batch |
2135
+ | `concurrency` | number | `5` | Concurrent message processing |
2136
+ | `retryAttempts` | number | `3` | Retry failed message processing |
2137
+ | `retryDelay` | number | `1000` | Delay between retries (ms) |
2138
+ | `deadLetterQueue` | string | `null` | DLQ for failed messages |
2139
+
2140
+ ### Supported Drivers
2141
+
2142
+ #### SQS Consumer
2143
+
2144
+ Consume from AWS SQS queues:
2145
+
2146
+ ```javascript
2147
+ {
2148
+ driver: 'sqs',
2149
+ config: {
2150
+ queueUrl: 'https://sqs.us-east-1.amazonaws.com/123456789012/my-queue',
2151
+ region: 'us-east-1',
2152
+ credentials: { accessKeyId: '...', secretAccessKey: '...' },
2153
+ pollingInterval: 1000,
2154
+ maxMessages: 10,
2155
+ visibilityTimeout: 300
2156
+ },
2157
+ consumers: [
2158
+ { resources: ['users'], queueUrl: 'specific-queue-url' }
2159
+ ]
2160
+ }
2161
+ ```
2162
+
2163
+ #### RabbitMQ Consumer
2164
+
2165
+ Consume from RabbitMQ queues:
2166
+
2167
+ ```javascript
2168
+ {
2169
+ driver: 'rabbitmq',
2170
+ config: {
2171
+ amqpUrl: 'amqp://user:pass@localhost:5672',
2172
+ exchange: 'my-exchange',
2173
+ prefetch: 10,
2174
+ reconnectInterval: 2000
2175
+ },
2176
+ consumers: [
2177
+ { resources: ['orders'], queue: 'orders-queue' }
2178
+ ]
2179
+ }
2180
+ ```
2181
+
2182
+ ### Message Format
2183
+
2184
+ Expected message structure:
2185
+
2186
+ ```javascript
2187
+ {
2188
+ resource: 'users', // Target resource name
2189
+ action: 'insert', // Operation: insert, update, delete
2190
+ data: { // Data payload
2191
+ name: 'John Doe',
2192
+ email: 'john@example.com'
2193
+ }
2194
+ }
2195
+ ```
2196
+
2197
+ ### 🔧 Easy Example
2198
+
2199
+ ```javascript
2200
+ import { S3db, QueueConsumerPlugin } from 's3db.js';
2201
+
2202
+ const s3db = new S3db({
2203
+ connectionString: "s3://ACCESS_KEY:SECRET_KEY@BUCKET_NAME/databases/myapp",
2204
+ plugins: [new QueueConsumerPlugin({
2205
+ enabled: true,
2206
+ consumers: [
2207
+ {
2208
+ driver: 'sqs',
2209
+ config: {
2210
+ queueUrl: 'https://sqs.us-east-1.amazonaws.com/123456789012/user-updates.fifo',
2211
+ region: 'us-east-1',
2212
+ pollingInterval: 2000,
2213
+ maxMessages: 5
2214
+ },
2215
+ consumers: [
2216
+ { resources: ['users', 'profiles'] }
2217
+ ]
2218
+ }
2219
+ ]
2220
+ })]
2221
+ });
2222
+
2223
+ await s3db.connect();
2224
+
2225
+ // Messages are automatically consumed and processed
2226
+ console.log('Queue consumer started - listening for messages...');
2227
+
2228
+ // Simulate sending a message (in real use, external systems send these)
2229
+ const testMessage = {
2230
+ resource: 'users',
2231
+ action: 'insert',
2232
+ data: {
2233
+ name: 'Queue User',
2234
+ email: 'queue@example.com',
2235
+ source: 'external-system'
2236
+ }
2237
+ };
2238
+
2239
+ console.log('Processing message:', testMessage);
2240
+ ```
2241
+
2242
+ ### 🚀 Advanced Multi-Driver Example
2243
+
2244
+ ```javascript
2245
+ import { S3db, QueueConsumerPlugin } from 's3db.js';
2246
+
2247
+ const s3db = new S3db({
2248
+ connectionString: "s3://ACCESS_KEY:SECRET_KEY@BUCKET_NAME/databases/myapp",
2249
+ plugins: [new QueueConsumerPlugin({
2250
+ enabled: true,
2251
+ batchSize: 20,
2252
+ concurrency: 10,
2253
+ retryAttempts: 5,
2254
+ retryDelay: 2000,
2255
+
2256
+ consumers: [
2257
+ // SQS Consumer for user events
2258
+ {
2259
+ driver: 'sqs',
2260
+ config: {
2261
+ region: 'us-east-1',
2262
+ credentials: {
2263
+ accessKeyId: process.env.AWS_ACCESS_KEY_ID,
2264
+ secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY
2265
+ },
2266
+ pollingInterval: 1000,
2267
+ maxMessages: 10,
2268
+ visibilityTimeout: 300,
2269
+ waitTimeSeconds: 20 // Long polling
2270
+ },
2271
+ consumers: [
2272
+ {
2273
+ resources: ['users'],
2274
+ queueUrl: 'https://sqs.us-east-1.amazonaws.com/123456789012/user-events.fifo',
2275
+ messageGroupId: 'user-processing'
2276
+ },
2277
+ {
2278
+ resources: ['orders'],
2279
+ queueUrl: 'https://sqs.us-east-1.amazonaws.com/123456789012/order-events.fifo',
2280
+ messageGroupId: 'order-processing'
2281
+ }
2282
+ ]
2283
+ },
2284
+
2285
+ // RabbitMQ Consumer for analytics events
2286
+ {
2287
+ driver: 'rabbitmq',
2288
+ config: {
2289
+ amqpUrl: 'amqp://analytics:password@localhost:5672',
2290
+ exchange: 'analytics-events',
2291
+ exchangeType: 'topic',
2292
+ prefetch: 15,
2293
+ reconnectInterval: 3000,
2294
+ heartbeat: 60
2295
+ },
2296
+ consumers: [
2297
+ {
2298
+ resources: ['analytics', 'metrics'],
2299
+ queue: 'analytics-queue',
2300
+ routingKey: 'analytics.*',
2301
+ durable: true
2302
+ },
2303
+ {
2304
+ resources: ['logs'],
2305
+ queue: 'logs-queue',
2306
+ routingKey: 'logs.*',
2307
+ durable: true
2308
+ }
2309
+ ]
2310
+ }
2311
+ ]
2312
+ })]
2313
+ });
2314
+
2315
+ await s3db.connect();
2316
+
2317
+ // Advanced message processing with custom handlers
2318
+ class QueueMessageProcessor {
2319
+ constructor(queuePlugin) {
2320
+ this.plugin = queuePlugin;
2321
+ this.stats = {
2322
+ processed: 0,
2323
+ errors: 0,
2324
+ byResource: {},
2325
+ byAction: {}
2326
+ };
2327
+
2328
+ this.setupEventListeners();
2329
+ }
2330
+
2331
+ setupEventListeners() {
2332
+ // Listen for message processing events
2333
+ this.plugin.on('message.received', (data) => {
2334
+ console.log(`📨 Received message: ${data.action} on ${data.resource}`);
2335
+ });
2336
+
2337
+ this.plugin.on('message.processed', (data) => {
2338
+ this.stats.processed++;
2339
+ this.updateStats(data.resource, data.action, 'success');
2340
+ console.log(`✅ Processed: ${data.action} on ${data.resource}`);
2341
+ });
2342
+
2343
+ this.plugin.on('message.failed', (data) => {
2344
+ this.stats.errors++;
2345
+ this.updateStats(data.resource, data.action, 'error');
2346
+ console.error(`❌ Failed: ${data.error}`);
2347
+
2348
+ // Custom error handling
2349
+ this.handleProcessingError(data);
2350
+ });
2351
+ }
2352
+
2353
+ updateStats(resource, action, status) {
2354
+ if (!this.stats.byResource[resource]) {
2355
+ this.stats.byResource[resource] = { success: 0, error: 0 };
2356
+ }
2357
+ if (!this.stats.byAction[action]) {
2358
+ this.stats.byAction[action] = { success: 0, error: 0 };
2359
+ }
2360
+
2361
+ this.stats.byResource[resource][status]++;
2362
+ this.stats.byAction[action][status]++;
2363
+ }
2364
+
2365
+ handleProcessingError(errorData) {
2366
+ const { resource, action, error, attempts } = errorData;
2367
+
2368
+ // Log to external monitoring system
2369
+ console.log(`🚨 Error processing ${action} on ${resource}: ${error}`);
2370
+
2371
+ // Custom retry logic
2372
+ if (attempts >= 3) {
2373
+ console.log(`💀 Moving to dead letter queue after ${attempts} attempts`);
2374
+ // In real implementation, move to DLQ
2375
+ }
2376
+
2377
+ // Resource-specific error handling
2378
+ if (resource === 'users' && error.includes('validation')) {
2379
+ console.log('👤 User validation error - checking schema compatibility');
2380
+ } else if (resource === 'orders' && error.includes('duplicate')) {
2381
+ console.log('🛒 Duplicate order detected - implementing idempotency check');
2382
+ }
2383
+ }
2384
+
2385
+ getProcessingStats() {
2386
+ const totalMessages = this.stats.processed + this.stats.errors;
2387
+ const successRate = totalMessages > 0 ? this.stats.processed / totalMessages : 1;
2388
+
2389
+ return {
2390
+ summary: {
2391
+ totalProcessed: this.stats.processed,
2392
+ totalErrors: this.stats.errors,
2393
+ successRate: successRate,
2394
+ health: successRate > 0.95 ? 'excellent' :
2395
+ successRate > 0.85 ? 'good' :
2396
+ successRate > 0.7 ? 'warning' : 'critical'
2397
+ },
2398
+ byResource: this.stats.byResource,
2399
+ byAction: this.stats.byAction
2400
+ };
2401
+ }
2402
+
2403
+ async pauseConsumption() {
2404
+ console.log('⏸️ Pausing queue consumption...');
2405
+ await this.plugin.pause();
2406
+ }
2407
+
2408
+ async resumeConsumption() {
2409
+ console.log('▶️ Resuming queue consumption...');
2410
+ await this.plugin.resume();
2411
+ }
2412
+ }
2413
+
2414
+ // Setup message processing
2415
+ const queuePlugin = s3db.plugins.find(p => p.constructor.name === 'QueueConsumerPlugin');
2416
+ const processor = new QueueMessageProcessor(queuePlugin);
2417
+
2418
+ // Simulate processing for demonstration
2419
+ console.log('🔄 Queue consumers started - processing messages...');
2420
+
2421
+ // In real scenario, messages come from external systems
2422
+ // Here we simulate the processing results
2423
+ setTimeout(async () => {
2424
+ const stats = processor.getProcessingStats();
2425
+
2426
+ console.log('\n=== Queue Processing Stats ===');
2427
+ console.log(`Total processed: ${stats.summary.totalProcessed}`);
2428
+ console.log(`Total errors: ${stats.summary.totalErrors}`);
2429
+ console.log(`Success rate: ${(stats.summary.successRate * 100).toFixed(1)}%`);
2430
+ console.log(`Health: ${stats.summary.health.toUpperCase()}`);
2431
+
2432
+ console.log('\n=== By Resource ===');
2433
+ Object.entries(stats.byResource).forEach(([resource, counts]) => {
2434
+ const total = counts.success + counts.error;
2435
+ console.log(`${resource}: ${counts.success}/${total} successful`);
2436
+ });
2437
+
2438
+ console.log('\n=== By Action ===');
2439
+ Object.entries(stats.byAction).forEach(([action, counts]) => {
2440
+ const total = counts.success + counts.error;
2441
+ console.log(`${action}: ${counts.success}/${total} successful`);
2442
+ });
2443
+
2444
+ }, 5000);
2445
+
2446
+ console.log('\n✅ Queue consumer demonstration completed');
2447
+ ```
2448
+
2449
+ ---
2450
+
2451
+ ## 🔧 Plugin Development
2452
+
2453
+ Create custom plugins to extend s3db.js with your specific requirements.
2454
+
2455
+ ### Plugin Base Class
2456
+
2457
+ ```javascript
2458
+ import { Plugin } from 's3db.js';
2459
+
2460
+ class MyCustomPlugin extends Plugin {
2461
+ constructor(options = {}) {
2462
+ super(options);
2463
+ this.config = {
2464
+ enabled: options.enabled !== false,
2465
+ ...options
2466
+ };
2467
+ }
2468
+
2469
+ async onSetup() {
2470
+ // Initialize plugin after database connection
2471
+ console.log('Setting up MyCustomPlugin');
2472
+ }
2473
+
2474
+ async onStart() {
2475
+ // Plugin is ready to operate
2476
+ console.log('MyCustomPlugin started');
2477
+ }
2478
+
2479
+ async onStop() {
2480
+ // Cleanup before shutdown
2481
+ console.log('MyCustomPlugin stopped');
2482
+ }
2483
+ }
2484
+ ```
2485
+
2486
+ ### Plugin Lifecycle
2487
+
2488
+ 1. **Constructor**: Configure plugin options
2489
+ 2. **setup()**: Called when database connects
2490
+ 3. **onSetup()**: Initialize plugin resources
2491
+ 4. **start()**: Called when database is ready
2492
+ 5. **onStart()**: Begin plugin operations
2493
+ 6. **stop()**: Called during shutdown
2494
+ 7. **onStop()**: Cleanup plugin resources
2495
+
2496
+ ### Custom Plugin Example
2497
+
2498
+ ```javascript
2499
+ class NotificationPlugin extends Plugin {
2500
+ constructor(options = {}) {
2501
+ super(options);
2502
+ this.config = {
2503
+ enabled: options.enabled !== false,
2504
+ webhookUrl: options.webhookUrl,
2505
+ events: options.events || ['insert', 'update', 'delete'],
2506
+ ...options
2507
+ };
2508
+ }
2509
+
2510
+ async onSetup() {
2511
+ // Install hooks for all resources
2512
+ for (const resource of Object.values(this.database.resources)) {
2513
+ this.installResourceHooks(resource);
2514
+ }
2515
+ }
2516
+
2517
+ installResourceHooks(resource) {
2518
+ this.config.events.forEach(event => {
2519
+ resource.on(event, async (data) => {
2520
+ await this.sendNotification(event, resource.name, data);
2521
+ });
2522
+ });
2523
+ }
2524
+
2525
+ async sendNotification(event, resourceName, data) {
2526
+ if (!this.config.webhookUrl) return;
2527
+
2528
+ const payload = {
2529
+ event,
2530
+ resource: resourceName,
2531
+ data,
2532
+ timestamp: new Date().toISOString()
2533
+ };
2534
+
2535
+ try {
2536
+ await fetch(this.config.webhookUrl, {
2537
+ method: 'POST',
2538
+ headers: { 'Content-Type': 'application/json' },
2539
+ body: JSON.stringify(payload)
2540
+ });
2541
+ } catch (error) {
2542
+ console.error('Notification failed:', error);
2543
+ }
2544
+ }
2545
+ }
2546
+ ```
2547
+
2548
+ ---
2549
+
2550
+ ## 💡 Plugin Combinations
2551
+
2552
+ Powerful workflows using multiple plugins together.
2553
+
2554
+ ### Complete Monitoring Stack
2555
+
2556
+ ```javascript
2557
+ import {
2558
+ S3db,
2559
+ CachePlugin,
2560
+ CostsPlugin,
2561
+ AuditPlugin,
2562
+ FullTextPlugin,
2563
+ MetricsPlugin,
2564
+ ReplicatorPlugin
2565
+ } from 's3db.js';
2566
+
2567
+ const s3db = new S3db({
2568
+ connectionString: "s3://ACCESS_KEY:SECRET_KEY@BUCKET_NAME/databases/myapp",
2569
+ plugins: [
2570
+ // Performance optimization
2571
+ new CachePlugin({
2572
+ driverType: 'memory',
2573
+ ttl: 600000
2574
+ }),
2575
+
2576
+ // Cost tracking
2577
+ CostsPlugin,
2578
+
2579
+ // Compliance and security
2580
+ new AuditPlugin({
2581
+ enabled: true,
2582
+ includeData: true,
2583
+ trackOperations: ['insert', 'update', 'delete', 'get']
2584
+ }),
2585
+
2586
+ // Search capabilities
2587
+ new FullTextPlugin({
2588
+ enabled: true,
2589
+ fields: ['name', 'description', 'content', 'tags']
2590
+ }),
2591
+
2592
+ // Performance monitoring
2593
+ new MetricsPlugin({
2594
+ enabled: true,
2595
+ collectPerformance: true,
2596
+ collectErrors: true,
2597
+ flushInterval: 30000
2598
+ }),
2599
+
2600
+ // Data replication
2601
+ new ReplicatorPlugin({
2602
+ replicators: [
2603
+ {
2604
+ driver: 's3db',
2605
+ resources: ['users', 'products', 'orders'],
2606
+ config: {
2607
+ connectionString: "s3://BACKUP_KEY:BACKUP_SECRET@BACKUP_BUCKET/backup"
2608
+ }
2609
+ }
2610
+ ]
2611
+ })
2612
+ ]
2613
+ });
2614
+
2615
+ await s3db.connect();
2616
+
2617
+ // All plugins work seamlessly together
2618
+ const products = s3db.resource('products');
2619
+
2620
+ // This single operation triggers:
2621
+ // - Audit logging
2622
+ // - Cost tracking
2623
+ // - Performance metrics
2624
+ // - Cache invalidation
2625
+ // - Data replication
2626
+ // - Search indexing
2627
+ await products.insert({
2628
+ name: 'New Product',
2629
+ description: 'Amazing new product with great features',
2630
+ price: 99.99,
2631
+ tags: ['new', 'featured', 'electronics']
2632
+ });
2633
+ ```
2634
+
2635
+ ### E-commerce Analytics Pipeline
2636
+
2637
+ ```javascript
2638
+ const s3db = new S3db({
2639
+ connectionString: "s3://ACCESS_KEY:SECRET_KEY@BUCKET_NAME/databases/ecommerce",
2640
+ plugins: [
2641
+ // Real-time search
2642
+ new FullTextPlugin({
2643
+ fields: ['name', 'description', 'brand', 'category'],
2644
+ language: 'en-US',
2645
+ stemming: true
2646
+ }),
2647
+
2648
+ // Performance monitoring
2649
+ new MetricsPlugin({
2650
+ collectPerformance: true,
2651
+ slowQueryThreshold: 500
2652
+ }),
2653
+
2654
+ // Multi-destination replication
2655
+ new ReplicatorPlugin({
2656
+ replicators: [
2657
+ // Backup
2658
+ { driver: 's3db', resources: '*', config: { connectionString: 'backup-db' } },
2659
+
2660
+ // Analytics warehouse
2661
+ {
2662
+ driver: 'bigquery',
2663
+ resources: {
2664
+ orders: 'fact_orders',
2665
+ products: 'dim_products',
2666
+ users: 'dim_customers'
2667
+ },
2668
+ config: { projectId: 'analytics', datasetId: 'ecommerce' }
2669
+ },
2670
+
2671
+ // Real-time events
2672
+ {
2673
+ driver: 'sqs',
2674
+ resources: ['orders', 'cart_events'],
2675
+ config: { queueUrl: 'order-events-queue' }
2676
+ }
2677
+ ]
2678
+ }),
2679
+
2680
+ // Comprehensive auditing
2681
+ new AuditPlugin({
2682
+ trackOperations: ['insert', 'update', 'delete'],
2683
+ includeData: true,
2684
+ excludeResources: ['sessions', 'temp_data']
2685
+ })
2686
+ ]
2687
+ });
2688
+ ```
2689
+
2690
+ ---
2691
+
2692
+ ## 🎯 Best Practices
2693
+
2694
+ ### Plugin Performance
2695
+
2696
+ 1. **Enable caching** for read-heavy workloads
2697
+ 2. **Monitor costs** in production environments
2698
+ 3. **Use appropriate sampling** for metrics collection
2699
+ 4. **Configure retention policies** for audit logs
2700
+ 5. **Test replicator connections** before deployment
2701
+
2702
+ ### Plugin Security
2703
+
2704
+ 1. **Exclude sensitive resources** from full-text indexing
2705
+ 2. **Limit audit data size** to prevent information leakage
2706
+ 3. **Use IAM roles** instead of access keys when possible
2707
+ 4. **Encrypt replication data** in transit and at rest
2708
+ 5. **Validate message sources** in queue consumers
2709
+
2710
+ ### Plugin Monitoring
2711
+
2712
+ 1. **Set up alerting** for replication failures
2713
+ 2. **Monitor plugin health** with metrics
2714
+ 3. **Track error rates** across all plugins
2715
+ 4. **Use structured logging** for debugging
2716
+ 5. **Implement circuit breakers** for external services
2717
+
2718
+ ---
2719
+
2720
+ **🎉 That's a wrap!** You now have comprehensive documentation for all s3db.js plugins. Each plugin is designed to work independently or in combination with others, providing a powerful and flexible foundation for your database needs.
2721
+
2722
+ For more examples and advanced use cases, check out the `/examples` directory in the s3db.js repository.
2723
+
2724
+ **Happy coding with s3db.js! 🚀**