s3db.js 6.1.0 → 7.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/PLUGINS.md +2724 -0
- package/README.md +377 -492
- package/UNLICENSE +24 -0
- package/dist/s3db.cjs.js +30054 -18189
- package/dist/s3db.cjs.min.js +1 -1
- package/dist/s3db.d.ts +373 -72
- package/dist/s3db.es.js +30040 -18186
- package/dist/s3db.es.min.js +1 -1
- package/dist/s3db.iife.js +29727 -17863
- package/dist/s3db.iife.min.js +1 -1
- package/package.json +44 -69
- package/src/behaviors/body-only.js +110 -0
- package/src/behaviors/body-overflow.js +153 -0
- package/src/behaviors/enforce-limits.js +195 -0
- package/src/behaviors/index.js +39 -0
- package/src/behaviors/truncate-data.js +204 -0
- package/src/behaviors/user-managed.js +147 -0
- package/src/client.class.js +515 -0
- package/src/concerns/base62.js +61 -0
- package/src/concerns/calculator.js +204 -0
- package/src/concerns/crypto.js +142 -0
- package/src/concerns/id.js +8 -0
- package/src/concerns/index.js +5 -0
- package/src/concerns/try-fn.js +151 -0
- package/src/connection-string.class.js +75 -0
- package/src/database.class.js +599 -0
- package/src/errors.js +261 -0
- package/src/index.js +17 -0
- package/src/plugins/audit.plugin.js +442 -0
- package/src/plugins/cache/cache.class.js +53 -0
- package/src/plugins/cache/index.js +6 -0
- package/src/plugins/cache/memory-cache.class.js +164 -0
- package/src/plugins/cache/s3-cache.class.js +189 -0
- package/src/plugins/cache.plugin.js +275 -0
- package/src/plugins/consumers/index.js +24 -0
- package/src/plugins/consumers/rabbitmq-consumer.js +56 -0
- package/src/plugins/consumers/sqs-consumer.js +102 -0
- package/src/plugins/costs.plugin.js +81 -0
- package/src/plugins/fulltext.plugin.js +473 -0
- package/src/plugins/index.js +12 -0
- package/src/plugins/metrics.plugin.js +603 -0
- package/src/plugins/plugin.class.js +210 -0
- package/src/plugins/plugin.obj.js +13 -0
- package/src/plugins/queue-consumer.plugin.js +134 -0
- package/src/plugins/replicator.plugin.js +769 -0
- package/src/plugins/replicators/base-replicator.class.js +85 -0
- package/src/plugins/replicators/bigquery-replicator.class.js +328 -0
- package/src/plugins/replicators/index.js +44 -0
- package/src/plugins/replicators/postgres-replicator.class.js +427 -0
- package/src/plugins/replicators/s3db-replicator.class.js +352 -0
- package/src/plugins/replicators/sqs-replicator.class.js +427 -0
- package/src/resource.class.js +2626 -0
- package/src/s3db.d.ts +1263 -0
- package/src/schema.class.js +706 -0
- package/src/stream/index.js +16 -0
- package/src/stream/resource-ids-page-reader.class.js +10 -0
- package/src/stream/resource-ids-reader.class.js +63 -0
- package/src/stream/resource-reader.class.js +81 -0
- package/src/stream/resource-writer.class.js +92 -0
- package/src/validator.class.js +97 -0
package/PLUGINS.md
ADDED
|
@@ -0,0 +1,2724 @@
|
|
|
1
|
+
# 🔌 s3db.js Plugins Documentation
|
|
2
|
+
|
|
3
|
+
<p align="center">
|
|
4
|
+
<strong>Comprehensive guide to all s3db.js plugins</strong><br>
|
|
5
|
+
<em>Extend your database with powerful features</em>
|
|
6
|
+
</p>
|
|
7
|
+
|
|
8
|
+
---
|
|
9
|
+
|
|
10
|
+
## 📋 Table of Contents
|
|
11
|
+
|
|
12
|
+
- [🚀 Getting Started](#-getting-started-with-plugins)
|
|
13
|
+
- [🧩 Available Plugins](#-available-plugins)
|
|
14
|
+
- [💾 Cache Plugin](#-cache-plugin)
|
|
15
|
+
- [💰 Costs Plugin](#-costs-plugin)
|
|
16
|
+
- [📝 Audit Plugin](#-audit-plugin)
|
|
17
|
+
- [🔍 FullText Plugin](#-fulltext-plugin)
|
|
18
|
+
- [📊 Metrics Plugin](#-metrics-plugin)
|
|
19
|
+
- [🔄 Replicator Plugin](#-replicator-plugin)
|
|
20
|
+
- [📬 Queue Consumer Plugin](#-queue-consumer-plugin)
|
|
21
|
+
- [🔧 Plugin Development](#-plugin-development)
|
|
22
|
+
- [💡 Plugin Combinations](#-plugin-combinations)
|
|
23
|
+
- [🎯 Best Practices](#-best-practices)
|
|
24
|
+
|
|
25
|
+
---
|
|
26
|
+
|
|
27
|
+
## 🚀 Getting Started with Plugins
|
|
28
|
+
|
|
29
|
+
Plugins extend s3db.js with additional functionality. They can be used individually or combined for powerful workflows.
|
|
30
|
+
|
|
31
|
+
### Basic Plugin Usage
|
|
32
|
+
|
|
33
|
+
```javascript
|
|
34
|
+
import { S3db, CachePlugin, CostsPlugin } from 's3db.js';
|
|
35
|
+
|
|
36
|
+
const s3db = new S3db({
|
|
37
|
+
connectionString: "s3://ACCESS_KEY:SECRET_KEY@BUCKET_NAME/databases/myapp",
|
|
38
|
+
plugins: [
|
|
39
|
+
new CachePlugin(),
|
|
40
|
+
CostsPlugin // Some plugins are static objects
|
|
41
|
+
]
|
|
42
|
+
});
|
|
43
|
+
|
|
44
|
+
await s3db.connect();
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
### Plugin Types
|
|
48
|
+
|
|
49
|
+
- **Instance Plugins**: Require `new` - `new CachePlugin(config)`
|
|
50
|
+
- **Static Plugins**: Used directly - `CostsPlugin`
|
|
51
|
+
- **Configurable**: Accept options for customization
|
|
52
|
+
- **Event-Driven**: Emit events for monitoring and integration
|
|
53
|
+
|
|
54
|
+
---
|
|
55
|
+
|
|
56
|
+
## 🧩 Available Plugins
|
|
57
|
+
|
|
58
|
+
## 💾 Cache Plugin
|
|
59
|
+
|
|
60
|
+
Intelligent caching system that reduces S3 API calls and improves performance by storing frequently accessed data in memory or S3.
|
|
61
|
+
|
|
62
|
+
### ⚡ Quick Start
|
|
63
|
+
|
|
64
|
+
```javascript
|
|
65
|
+
import { S3db, CachePlugin } from 's3db.js';
|
|
66
|
+
|
|
67
|
+
const s3db = new S3db({
|
|
68
|
+
connectionString: "s3://ACCESS_KEY:SECRET_KEY@BUCKET_NAME/databases/myapp",
|
|
69
|
+
plugins: [new CachePlugin()]
|
|
70
|
+
});
|
|
71
|
+
|
|
72
|
+
await s3db.connect();
|
|
73
|
+
|
|
74
|
+
// Cache is automatically used for read operations
|
|
75
|
+
const users = s3db.resource('users');
|
|
76
|
+
await users.count(); // Cached for default TTL
|
|
77
|
+
await users.list(); // Cached result
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
### ⚙️ Configuration Parameters
|
|
81
|
+
|
|
82
|
+
| Parameter | Type | Default | Description |
|
|
83
|
+
|-----------|------|---------|-------------|
|
|
84
|
+
| `driverType` | string | `'s3'` | Cache driver: `'memory'` or `'s3'` |
|
|
85
|
+
| `ttl` | number | `300000` | Time-to-live in milliseconds (5 minutes) |
|
|
86
|
+
| `maxSize` | number | `1000` | Maximum number of items in cache (memory driver) |
|
|
87
|
+
| `includePartitions` | boolean | `true` | Include partition values in cache keys |
|
|
88
|
+
| `driver` | object | `null` | Custom cache driver instance |
|
|
89
|
+
| `memoryOptions` | object | `{}` | Options for memory cache driver |
|
|
90
|
+
| `s3Options` | object | `{}` | Options for S3 cache driver |
|
|
91
|
+
|
|
92
|
+
### Memory Driver Options (`memoryOptions`)
|
|
93
|
+
|
|
94
|
+
| Parameter | Type | Default | Description |
|
|
95
|
+
|-----------|------|---------|-------------|
|
|
96
|
+
| `maxSize` | number | `1000` | Maximum items in memory |
|
|
97
|
+
| `ttl` | number | `300000` | Default TTL in milliseconds |
|
|
98
|
+
| `checkPeriod` | number | `600000` | Cleanup interval in milliseconds |
|
|
99
|
+
|
|
100
|
+
### S3 Driver Options (`s3Options`)
|
|
101
|
+
|
|
102
|
+
| Parameter | Type | Default | Description |
|
|
103
|
+
|-----------|------|---------|-------------|
|
|
104
|
+
| `bucket` | string | Same as database | S3 bucket for cache storage |
|
|
105
|
+
| `prefix` | string | `'cache/'` | S3 key prefix for cache objects |
|
|
106
|
+
| `client` | object | Database client | Custom S3 client instance |
|
|
107
|
+
|
|
108
|
+
### 🔧 Easy Example
|
|
109
|
+
|
|
110
|
+
```javascript
|
|
111
|
+
import { S3db, CachePlugin } from 's3db.js';
|
|
112
|
+
|
|
113
|
+
const s3db = new S3db({
|
|
114
|
+
connectionString: "s3://ACCESS_KEY:SECRET_KEY@BUCKET_NAME/databases/myapp",
|
|
115
|
+
plugins: [new CachePlugin({
|
|
116
|
+
driverType: 'memory',
|
|
117
|
+
ttl: 600000, // 10 minutes
|
|
118
|
+
maxSize: 500
|
|
119
|
+
})]
|
|
120
|
+
});
|
|
121
|
+
|
|
122
|
+
await s3db.connect();
|
|
123
|
+
|
|
124
|
+
const products = s3db.resource('products');
|
|
125
|
+
|
|
126
|
+
// First call hits the database
|
|
127
|
+
console.time('First call');
|
|
128
|
+
const result1 = await products.count();
|
|
129
|
+
console.timeEnd('First call'); // ~200ms
|
|
130
|
+
|
|
131
|
+
// Second call uses cache
|
|
132
|
+
console.time('Cached call');
|
|
133
|
+
const result2 = await products.count();
|
|
134
|
+
console.timeEnd('Cached call'); // ~2ms
|
|
135
|
+
|
|
136
|
+
// Cache is automatically cleared on write operations
|
|
137
|
+
await products.insert({ name: 'New Product', price: 29.99 });
|
|
138
|
+
|
|
139
|
+
// Next call will hit database again (cache cleared)
|
|
140
|
+
const result3 = await products.count(); // Fresh data
|
|
141
|
+
```
|
|
142
|
+
|
|
143
|
+
### 🚀 Advanced Configuration Example
|
|
144
|
+
|
|
145
|
+
```javascript
|
|
146
|
+
import { S3db, CachePlugin, MemoryCache, S3Cache } from 's3db.js';
|
|
147
|
+
|
|
148
|
+
// Custom cache driver with advanced configuration
|
|
149
|
+
const customCache = new MemoryCache({
|
|
150
|
+
maxSize: 2000,
|
|
151
|
+
ttl: 900000, // 15 minutes
|
|
152
|
+
checkPeriod: 300000, // 5 minutes cleanup
|
|
153
|
+
algorithm: 'lru' // Least Recently Used
|
|
154
|
+
});
|
|
155
|
+
|
|
156
|
+
const s3db = new S3db({
|
|
157
|
+
connectionString: "s3://ACCESS_KEY:SECRET_KEY@BUCKET_NAME/databases/myapp",
|
|
158
|
+
plugins: [new CachePlugin({
|
|
159
|
+
driver: customCache,
|
|
160
|
+
includePartitions: true,
|
|
161
|
+
|
|
162
|
+
// S3 cache fallback for persistence
|
|
163
|
+
s3Options: {
|
|
164
|
+
bucket: 'my-cache-bucket',
|
|
165
|
+
prefix: 'app-cache/',
|
|
166
|
+
ttl: 3600000, // 1 hour S3 cache
|
|
167
|
+
compression: true,
|
|
168
|
+
encryption: true
|
|
169
|
+
},
|
|
170
|
+
|
|
171
|
+
// Memory cache for speed
|
|
172
|
+
memoryOptions: {
|
|
173
|
+
maxSize: 5000,
|
|
174
|
+
ttl: 600000, // 10 minutes memory cache
|
|
175
|
+
checkPeriod: 120000, // 2 minutes cleanup
|
|
176
|
+
evictionPolicy: 'lru',
|
|
177
|
+
stats: true // Enable cache statistics
|
|
178
|
+
}
|
|
179
|
+
})]
|
|
180
|
+
});
|
|
181
|
+
|
|
182
|
+
await s3db.connect();
|
|
183
|
+
|
|
184
|
+
// Access cache methods on resources
|
|
185
|
+
const users = s3db.resource('users');
|
|
186
|
+
|
|
187
|
+
// Generate custom cache keys
|
|
188
|
+
const cacheKey = await users.cacheKeyFor({
|
|
189
|
+
action: 'list',
|
|
190
|
+
params: { limit: 10 },
|
|
191
|
+
partition: 'byStatus',
|
|
192
|
+
partitionValues: { status: 'active' }
|
|
193
|
+
});
|
|
194
|
+
|
|
195
|
+
// Manual cache operations
|
|
196
|
+
await users.cache.set(cacheKey, data, 1800000); // 30 minutes
|
|
197
|
+
const cached = await users.cache.get(cacheKey);
|
|
198
|
+
await users.cache.delete(cacheKey);
|
|
199
|
+
await users.cache.clear(); // Clear all cache
|
|
200
|
+
|
|
201
|
+
// Cache statistics (if enabled)
|
|
202
|
+
const stats = users.cache.stats();
|
|
203
|
+
console.log('Cache hit rate:', stats.hitRate);
|
|
204
|
+
console.log('Total hits:', stats.hits);
|
|
205
|
+
console.log('Total misses:', stats.misses);
|
|
206
|
+
```
|
|
207
|
+
|
|
208
|
+
---
|
|
209
|
+
|
|
210
|
+
## 💰 Costs Plugin
|
|
211
|
+
|
|
212
|
+
Track and monitor AWS S3 costs in real-time by calculating expenses for each API operation. Essential for cost optimization and budget management.
|
|
213
|
+
|
|
214
|
+
### ⚡ Quick Start
|
|
215
|
+
|
|
216
|
+
```javascript
|
|
217
|
+
import { S3db, CostsPlugin } from 's3db.js';
|
|
218
|
+
|
|
219
|
+
const s3db = new S3db({
|
|
220
|
+
connectionString: "s3://ACCESS_KEY:SECRET_KEY@BUCKET_NAME/databases/myapp",
|
|
221
|
+
plugins: [CostsPlugin] // Static plugin - no 'new' required
|
|
222
|
+
});
|
|
223
|
+
|
|
224
|
+
await s3db.connect();
|
|
225
|
+
|
|
226
|
+
// Use your database normally
|
|
227
|
+
const users = s3db.resource('users');
|
|
228
|
+
await users.insert({ name: 'John', email: 'john@example.com' });
|
|
229
|
+
await users.list();
|
|
230
|
+
|
|
231
|
+
// Check costs
|
|
232
|
+
console.log('Total cost:', s3db.client.costs.total);
|
|
233
|
+
console.log('Request breakdown:', s3db.client.costs.requests);
|
|
234
|
+
```
|
|
235
|
+
|
|
236
|
+
### ⚙️ Configuration Parameters
|
|
237
|
+
|
|
238
|
+
**Note**: CostsPlugin is a static plugin with no configuration options. It automatically tracks all S3 operations.
|
|
239
|
+
|
|
240
|
+
### Cost Tracking Details
|
|
241
|
+
|
|
242
|
+
| Operation | Cost per 1000 requests | Tracked Commands |
|
|
243
|
+
|-----------|------------------------|------------------|
|
|
244
|
+
| PUT operations | $0.005 | PutObjectCommand |
|
|
245
|
+
| GET operations | $0.0004 | GetObjectCommand |
|
|
246
|
+
| HEAD operations | $0.0004 | HeadObjectCommand |
|
|
247
|
+
| DELETE operations | $0.0004 | DeleteObjectCommand, DeleteObjectsCommand |
|
|
248
|
+
| LIST operations | $0.005 | ListObjectsV2Command |
|
|
249
|
+
|
|
250
|
+
### Cost Data Structure
|
|
251
|
+
|
|
252
|
+
```javascript
|
|
253
|
+
{
|
|
254
|
+
total: 0.000123, // Total cost in USD
|
|
255
|
+
prices: { // Cost per 1000 requests
|
|
256
|
+
put: 0.000005,
|
|
257
|
+
get: 0.0000004,
|
|
258
|
+
head: 0.0000004,
|
|
259
|
+
delete: 0.0000004,
|
|
260
|
+
list: 0.000005
|
|
261
|
+
},
|
|
262
|
+
requests: { // Request counters
|
|
263
|
+
total: 15,
|
|
264
|
+
put: 3,
|
|
265
|
+
get: 8,
|
|
266
|
+
head: 2,
|
|
267
|
+
delete: 1,
|
|
268
|
+
list: 1
|
|
269
|
+
},
|
|
270
|
+
events: { // Command-specific counters
|
|
271
|
+
total: 15,
|
|
272
|
+
PutObjectCommand: 3,
|
|
273
|
+
GetObjectCommand: 8,
|
|
274
|
+
HeadObjectCommand: 2,
|
|
275
|
+
DeleteObjectCommand: 1,
|
|
276
|
+
ListObjectsV2Command: 1
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
```
|
|
280
|
+
|
|
281
|
+
### 🔧 Easy Example
|
|
282
|
+
|
|
283
|
+
```javascript
|
|
284
|
+
import { S3db, CostsPlugin } from 's3db.js';
|
|
285
|
+
|
|
286
|
+
const s3db = new S3db({
|
|
287
|
+
connectionString: "s3://ACCESS_KEY:SECRET_KEY@BUCKET_NAME/databases/myapp",
|
|
288
|
+
plugins: [CostsPlugin]
|
|
289
|
+
});
|
|
290
|
+
|
|
291
|
+
await s3db.connect();
|
|
292
|
+
|
|
293
|
+
const products = s3db.resource('products');
|
|
294
|
+
|
|
295
|
+
// Perform operations and track costs
|
|
296
|
+
await products.insert({ name: 'Widget A', price: 19.99 });
|
|
297
|
+
await products.insert({ name: 'Widget B', price: 29.99 });
|
|
298
|
+
await products.list();
|
|
299
|
+
await products.count();
|
|
300
|
+
|
|
301
|
+
// Analyze costs
|
|
302
|
+
const costs = s3db.client.costs;
|
|
303
|
+
console.log(`Operations performed: ${costs.requests.total}`);
|
|
304
|
+
console.log(`Total cost: $${costs.total.toFixed(6)}`);
|
|
305
|
+
console.log(`Most expensive operation: PUT (${costs.requests.put} requests)`);
|
|
306
|
+
|
|
307
|
+
// Cost breakdown
|
|
308
|
+
console.log('\nCost breakdown:');
|
|
309
|
+
Object.entries(costs.requests).forEach(([operation, count]) => {
|
|
310
|
+
if (operation !== 'total' && count > 0) {
|
|
311
|
+
const operationCost = count * costs.prices[operation];
|
|
312
|
+
console.log(` ${operation.toUpperCase()}: ${count} requests = $${operationCost.toFixed(6)}`);
|
|
313
|
+
}
|
|
314
|
+
});
|
|
315
|
+
```
|
|
316
|
+
|
|
317
|
+
### 🚀 Advanced Monitoring Example
|
|
318
|
+
|
|
319
|
+
```javascript
|
|
320
|
+
import { S3db, CostsPlugin } from 's3db.js';
|
|
321
|
+
|
|
322
|
+
class CostMonitor {
|
|
323
|
+
constructor(s3db) {
|
|
324
|
+
this.s3db = s3db;
|
|
325
|
+
this.startTime = Date.now();
|
|
326
|
+
this.checkpoints = [];
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
checkpoint(label) {
|
|
330
|
+
const costs = { ...this.s3db.client.costs };
|
|
331
|
+
const timestamp = Date.now();
|
|
332
|
+
|
|
333
|
+
this.checkpoints.push({
|
|
334
|
+
label,
|
|
335
|
+
timestamp,
|
|
336
|
+
costs,
|
|
337
|
+
duration: timestamp - this.startTime
|
|
338
|
+
});
|
|
339
|
+
|
|
340
|
+
return costs;
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
report() {
|
|
344
|
+
console.log('\n=== Cost Analysis Report ===');
|
|
345
|
+
|
|
346
|
+
for (let i = 0; i < this.checkpoints.length; i++) {
|
|
347
|
+
const checkpoint = this.checkpoints[i];
|
|
348
|
+
const prevCheckpoint = i > 0 ? this.checkpoints[i - 1] : null;
|
|
349
|
+
|
|
350
|
+
console.log(`\n${checkpoint.label}:`);
|
|
351
|
+
console.log(` Time: ${checkpoint.duration}ms`);
|
|
352
|
+
console.log(` Total cost: $${checkpoint.costs.total.toFixed(6)}`);
|
|
353
|
+
|
|
354
|
+
if (prevCheckpoint) {
|
|
355
|
+
const costDiff = checkpoint.costs.total - prevCheckpoint.costs.total;
|
|
356
|
+
const requestDiff = checkpoint.costs.requests.total - prevCheckpoint.costs.requests.total;
|
|
357
|
+
console.log(` Cost increase: $${costDiff.toFixed(6)}`);
|
|
358
|
+
console.log(` New requests: ${requestDiff}`);
|
|
359
|
+
}
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
// Efficiency metrics
|
|
363
|
+
const finalCosts = this.checkpoints[this.checkpoints.length - 1].costs;
|
|
364
|
+
const totalTime = this.checkpoints[this.checkpoints.length - 1].duration;
|
|
365
|
+
|
|
366
|
+
console.log('\n=== Efficiency Metrics ===');
|
|
367
|
+
console.log(`Total execution time: ${totalTime}ms`);
|
|
368
|
+
console.log(`Total requests: ${finalCosts.requests.total}`);
|
|
369
|
+
console.log(`Requests per second: ${(finalCosts.requests.total / (totalTime / 1000)).toFixed(2)}`);
|
|
370
|
+
console.log(`Cost per request: $${(finalCosts.total / finalCosts.requests.total).toFixed(8)}`);
|
|
371
|
+
console.log(`Monthly projection (1M ops): $${(finalCosts.total * 1000000).toFixed(2)}`);
|
|
372
|
+
}
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
// Usage
|
|
376
|
+
const s3db = new S3db({
|
|
377
|
+
connectionString: "s3://ACCESS_KEY:SECRET_KEY@BUCKET_NAME/databases/myapp",
|
|
378
|
+
plugins: [CostsPlugin]
|
|
379
|
+
});
|
|
380
|
+
|
|
381
|
+
await s3db.connect();
|
|
382
|
+
|
|
383
|
+
const monitor = new CostMonitor(s3db);
|
|
384
|
+
const users = s3db.resource('users');
|
|
385
|
+
|
|
386
|
+
// Bulk operations with cost tracking
|
|
387
|
+
monitor.checkpoint('Initial state');
|
|
388
|
+
|
|
389
|
+
// Bulk insert
|
|
390
|
+
const userData = Array.from({ length: 100 }, (_, i) => ({
|
|
391
|
+
name: `User ${i}`,
|
|
392
|
+
email: `user${i}@example.com`,
|
|
393
|
+
role: i % 3 === 0 ? 'admin' : 'user'
|
|
394
|
+
}));
|
|
395
|
+
|
|
396
|
+
await users.insertMany(userData);
|
|
397
|
+
monitor.checkpoint('After bulk insert');
|
|
398
|
+
|
|
399
|
+
// Query operations
|
|
400
|
+
await users.count();
|
|
401
|
+
await users.list({ limit: 50 });
|
|
402
|
+
await users.list({ limit: 25, offset: 25 });
|
|
403
|
+
monitor.checkpoint('After queries');
|
|
404
|
+
|
|
405
|
+
// Update operations
|
|
406
|
+
const userList = await users.list({ limit: 10 });
|
|
407
|
+
for (const user of userList) {
|
|
408
|
+
await users.update(user.id, { lastLogin: new Date().toISOString() });
|
|
409
|
+
}
|
|
410
|
+
monitor.checkpoint('After updates');
|
|
411
|
+
|
|
412
|
+
// Generate detailed report
|
|
413
|
+
monitor.report();
|
|
414
|
+
|
|
415
|
+
// Set cost alerts
|
|
416
|
+
const currentCost = s3db.client.costs.total;
|
|
417
|
+
if (currentCost > 0.01) { // $0.01 threshold
|
|
418
|
+
console.warn(`⚠️ Cost threshold exceeded: $${currentCost.toFixed(6)}`);
|
|
419
|
+
}
|
|
420
|
+
|
|
421
|
+
// Export cost data for external analysis
|
|
422
|
+
const costData = {
|
|
423
|
+
timestamp: new Date().toISOString(),
|
|
424
|
+
sessionCosts: s3db.client.costs,
|
|
425
|
+
checkpoints: monitor.checkpoints,
|
|
426
|
+
summary: {
|
|
427
|
+
totalCost: s3db.client.costs.total,
|
|
428
|
+
totalRequests: s3db.client.costs.requests.total,
|
|
429
|
+
avgCostPerRequest: s3db.client.costs.total / s3db.client.costs.requests.total,
|
|
430
|
+
mostExpensiveOperation: Object.entries(s3db.client.costs.requests)
|
|
431
|
+
.filter(([key]) => key !== 'total')
|
|
432
|
+
.sort(([,a], [,b]) => b - a)[0]
|
|
433
|
+
}
|
|
434
|
+
};
|
|
435
|
+
|
|
436
|
+
console.log('\nExportable cost data:', JSON.stringify(costData, null, 2));
|
|
437
|
+
```
|
|
438
|
+
|
|
439
|
+
---
|
|
440
|
+
|
|
441
|
+
## 📝 Audit Plugin
|
|
442
|
+
|
|
443
|
+
Comprehensive audit logging system that tracks all database operations for compliance, security monitoring, and debugging purposes.
|
|
444
|
+
|
|
445
|
+
### ⚡ Quick Start
|
|
446
|
+
|
|
447
|
+
```javascript
|
|
448
|
+
import { S3db, AuditPlugin } from 's3db.js';
|
|
449
|
+
|
|
450
|
+
const s3db = new S3db({
|
|
451
|
+
connectionString: "s3://ACCESS_KEY:SECRET_KEY@BUCKET_NAME/databases/myapp",
|
|
452
|
+
plugins: [new AuditPlugin({ enabled: true })]
|
|
453
|
+
});
|
|
454
|
+
|
|
455
|
+
await s3db.connect();
|
|
456
|
+
|
|
457
|
+
// All operations are automatically logged
|
|
458
|
+
const users = s3db.resource('users');
|
|
459
|
+
await users.insert({ name: 'John', email: 'john@example.com' });
|
|
460
|
+
await users.update(userId, { name: 'John Doe' });
|
|
461
|
+
|
|
462
|
+
// Access audit logs
|
|
463
|
+
const auditResource = s3db.resource('audits');
|
|
464
|
+
const logs = await auditResource.list();
|
|
465
|
+
console.log('Audit trail:', logs);
|
|
466
|
+
```
|
|
467
|
+
|
|
468
|
+
### ⚙️ Configuration Parameters
|
|
469
|
+
|
|
470
|
+
| Parameter | Type | Default | Description |
|
|
471
|
+
|-----------|------|---------|-------------|
|
|
472
|
+
| `enabled` | boolean | `true` | Enable/disable audit logging |
|
|
473
|
+
| `includeData` | boolean | `true` | Include data payloads in audit logs |
|
|
474
|
+
| `includePartitions` | boolean | `true` | Include partition information in logs |
|
|
475
|
+
| `maxDataSize` | number | `10000` | Maximum data size to log (bytes) |
|
|
476
|
+
| `trackOperations` | array | `['insert', 'update', 'delete']` | Operations to audit |
|
|
477
|
+
| `excludeResources` | array | `[]` | Resources to exclude from auditing |
|
|
478
|
+
| `userId` | function | `null` | Function to extract user ID from context |
|
|
479
|
+
| `metadata` | function | `null` | Function to add custom metadata |
|
|
480
|
+
|
|
481
|
+
### Audit Log Structure
|
|
482
|
+
|
|
483
|
+
```javascript
|
|
484
|
+
{
|
|
485
|
+
id: 'audit-abc123',
|
|
486
|
+
resourceName: 'users',
|
|
487
|
+
operation: 'insert',
|
|
488
|
+
recordId: 'user-123',
|
|
489
|
+
userId: 'admin-456',
|
|
490
|
+
timestamp: '2024-01-15T10:30:00.000Z',
|
|
491
|
+
oldData: '{"name":"John"}', // For updates
|
|
492
|
+
newData: '{"name":"John Doe"}', // JSON string of data
|
|
493
|
+
partition: 'byStatus', // If using partitions
|
|
494
|
+
partitionValues: '{"status":"active"}',
|
|
495
|
+
metadata: '{"ip":"192.168.1.1"}', // Custom metadata
|
|
496
|
+
_v: 0 // Audit record version
|
|
497
|
+
}
|
|
498
|
+
```
|
|
499
|
+
|
|
500
|
+
### 🔧 Easy Example
|
|
501
|
+
|
|
502
|
+
```javascript
|
|
503
|
+
import { S3db, AuditPlugin } from 's3db.js';
|
|
504
|
+
|
|
505
|
+
const s3db = new S3db({
|
|
506
|
+
connectionString: "s3://ACCESS_KEY:SECRET_KEY@BUCKET_NAME/databases/myapp",
|
|
507
|
+
plugins: [new AuditPlugin({
|
|
508
|
+
enabled: true,
|
|
509
|
+
includeData: true,
|
|
510
|
+
trackOperations: ['insert', 'update', 'delete', 'get'],
|
|
511
|
+
maxDataSize: 5000
|
|
512
|
+
})]
|
|
513
|
+
});
|
|
514
|
+
|
|
515
|
+
await s3db.connect();
|
|
516
|
+
|
|
517
|
+
const products = s3db.resource('products');
|
|
518
|
+
const audits = s3db.resource('audits');
|
|
519
|
+
|
|
520
|
+
// Perform operations (automatically audited)
|
|
521
|
+
const product = await products.insert({
|
|
522
|
+
name: 'Gaming Laptop',
|
|
523
|
+
price: 1299.99,
|
|
524
|
+
category: 'electronics'
|
|
525
|
+
});
|
|
526
|
+
|
|
527
|
+
await products.update(product.id, { price: 1199.99 });
|
|
528
|
+
await products.get(product.id);
|
|
529
|
+
await products.delete(product.id);
|
|
530
|
+
|
|
531
|
+
// Review audit trail
|
|
532
|
+
const auditLogs = await audits.list();
|
|
533
|
+
|
|
534
|
+
console.log('\n=== Audit Trail ===');
|
|
535
|
+
auditLogs.forEach(log => {
|
|
536
|
+
console.log(`${log.timestamp} | ${log.operation.toUpperCase()} | ${log.resourceName} | ${log.recordId}`);
|
|
537
|
+
|
|
538
|
+
if (log.operation === 'update') {
|
|
539
|
+
const oldData = JSON.parse(log.oldData);
|
|
540
|
+
const newData = JSON.parse(log.newData);
|
|
541
|
+
console.log(` Price changed: $${oldData.price} → $${newData.price}`);
|
|
542
|
+
}
|
|
543
|
+
});
|
|
544
|
+
|
|
545
|
+
// Query specific audit logs
|
|
546
|
+
const updateLogs = await audits.list({
|
|
547
|
+
filter: log => log.operation === 'update'
|
|
548
|
+
});
|
|
549
|
+
|
|
550
|
+
console.log(`\nFound ${updateLogs.length} update operations`);
|
|
551
|
+
```
|
|
552
|
+
|
|
553
|
+
### 🚀 Advanced Configuration Example
|
|
554
|
+
|
|
555
|
+
```javascript
|
|
556
|
+
import { S3db, AuditPlugin } from 's3db.js';
|
|
557
|
+
|
|
558
|
+
const s3db = new S3db({
|
|
559
|
+
connectionString: "s3://ACCESS_KEY:SECRET_KEY@BUCKET_NAME/databases/myapp",
|
|
560
|
+
plugins: [new AuditPlugin({
|
|
561
|
+
enabled: true,
|
|
562
|
+
includeData: true,
|
|
563
|
+
includePartitions: true,
|
|
564
|
+
maxDataSize: 20000, // 20KB limit
|
|
565
|
+
|
|
566
|
+
// Track all operations including reads
|
|
567
|
+
trackOperations: ['insert', 'update', 'delete', 'get', 'list'],
|
|
568
|
+
|
|
569
|
+
// Exclude sensitive resources from auditing
|
|
570
|
+
excludeResources: ['sessions', 'temp_data'],
|
|
571
|
+
|
|
572
|
+
// Extract user ID from request context
|
|
573
|
+
userId: (context) => {
|
|
574
|
+
return context?.user?.id ||
|
|
575
|
+
context?.headers?.['x-user-id'] ||
|
|
576
|
+
'anonymous';
|
|
577
|
+
},
|
|
578
|
+
|
|
579
|
+
// Add custom metadata to audit logs
|
|
580
|
+
metadata: (operation, resourceName, data, context) => {
|
|
581
|
+
return {
|
|
582
|
+
ip: context?.ip,
|
|
583
|
+
userAgent: context?.userAgent,
|
|
584
|
+
sessionId: context?.sessionId,
|
|
585
|
+
apiVersion: '1.0',
|
|
586
|
+
environment: process.env.NODE_ENV,
|
|
587
|
+
requestId: context?.requestId,
|
|
588
|
+
|
|
589
|
+
// Operation-specific metadata
|
|
590
|
+
...(operation === 'insert' && {
|
|
591
|
+
createdVia: 'api',
|
|
592
|
+
validationPassed: true
|
|
593
|
+
}),
|
|
594
|
+
|
|
595
|
+
...(operation === 'update' && {
|
|
596
|
+
fieldsChanged: Object.keys(data || {}),
|
|
597
|
+
automaticUpdate: false
|
|
598
|
+
}),
|
|
599
|
+
|
|
600
|
+
...(operation === 'delete' && {
|
|
601
|
+
softDelete: false,
|
|
602
|
+
cascadeDelete: false
|
|
603
|
+
})
|
|
604
|
+
};
|
|
605
|
+
}
|
|
606
|
+
})]
|
|
607
|
+
});
|
|
608
|
+
|
|
609
|
+
await s3db.connect();
|
|
610
|
+
|
|
611
|
+
// Custom audit query functions
|
|
612
|
+
class AuditAnalyzer {
|
|
613
|
+
constructor(auditResource) {
|
|
614
|
+
this.audits = auditResource;
|
|
615
|
+
}
|
|
616
|
+
|
|
617
|
+
async getUserActivity(userId, timeRange = 24) {
|
|
618
|
+
const since = new Date(Date.now() - timeRange * 60 * 60 * 1000);
|
|
619
|
+
const logs = await this.audits.list();
|
|
620
|
+
|
|
621
|
+
return logs.filter(log =>
|
|
622
|
+
log.userId === userId &&
|
|
623
|
+
new Date(log.timestamp) > since
|
|
624
|
+
);
|
|
625
|
+
}
|
|
626
|
+
|
|
627
|
+
async getResourceActivity(resourceName, operation = null) {
|
|
628
|
+
const logs = await this.audits.list();
|
|
629
|
+
|
|
630
|
+
return logs.filter(log =>
|
|
631
|
+
log.resourceName === resourceName &&
|
|
632
|
+
(!operation || log.operation === operation)
|
|
633
|
+
);
|
|
634
|
+
}
|
|
635
|
+
|
|
636
|
+
async getDataChanges(resourceName, recordId) {
|
|
637
|
+
const logs = await this.audits.list();
|
|
638
|
+
|
|
639
|
+
return logs
|
|
640
|
+
.filter(log =>
|
|
641
|
+
log.resourceName === resourceName &&
|
|
642
|
+
log.recordId === recordId &&
|
|
643
|
+
log.operation === 'update'
|
|
644
|
+
)
|
|
645
|
+
.sort((a, b) => new Date(a.timestamp) - new Date(b.timestamp))
|
|
646
|
+
.map(log => ({
|
|
647
|
+
timestamp: log.timestamp,
|
|
648
|
+
oldData: JSON.parse(log.oldData || '{}'),
|
|
649
|
+
newData: JSON.parse(log.newData || '{}'),
|
|
650
|
+
userId: log.userId,
|
|
651
|
+
metadata: JSON.parse(log.metadata || '{}')
|
|
652
|
+
}));
|
|
653
|
+
}
|
|
654
|
+
|
|
655
|
+
async generateComplianceReport(startDate, endDate) {
|
|
656
|
+
const logs = await this.audits.list();
|
|
657
|
+
|
|
658
|
+
const filteredLogs = logs.filter(log => {
|
|
659
|
+
const logDate = new Date(log.timestamp);
|
|
660
|
+
return logDate >= startDate && logDate <= endDate;
|
|
661
|
+
});
|
|
662
|
+
|
|
663
|
+
const summary = {
|
|
664
|
+
totalOperations: filteredLogs.length,
|
|
665
|
+
operationBreakdown: {},
|
|
666
|
+
resourceActivity: {},
|
|
667
|
+
userActivity: {},
|
|
668
|
+
timeRange: { startDate, endDate }
|
|
669
|
+
};
|
|
670
|
+
|
|
671
|
+
filteredLogs.forEach(log => {
|
|
672
|
+
// Operation breakdown
|
|
673
|
+
summary.operationBreakdown[log.operation] =
|
|
674
|
+
(summary.operationBreakdown[log.operation] || 0) + 1;
|
|
675
|
+
|
|
676
|
+
// Resource activity
|
|
677
|
+
summary.resourceActivity[log.resourceName] =
|
|
678
|
+
(summary.resourceActivity[log.resourceName] || 0) + 1;
|
|
679
|
+
|
|
680
|
+
// User activity
|
|
681
|
+
summary.userActivity[log.userId] =
|
|
682
|
+
(summary.userActivity[log.userId] || 0) + 1;
|
|
683
|
+
});
|
|
684
|
+
|
|
685
|
+
return summary;
|
|
686
|
+
}
|
|
687
|
+
}
|
|
688
|
+
|
|
689
|
+
// Usage with context
|
|
690
|
+
const users = s3db.resource('users');
|
|
691
|
+
const audits = s3db.resource('audits');
|
|
692
|
+
const analyzer = new AuditAnalyzer(audits);
|
|
693
|
+
|
|
694
|
+
// Simulate operations with user context
|
|
695
|
+
const userContext = {
|
|
696
|
+
user: { id: 'admin-123', role: 'admin' },
|
|
697
|
+
ip: '192.168.1.100',
|
|
698
|
+
userAgent: 'Mozilla/5.0...',
|
|
699
|
+
sessionId: 'sess-789',
|
|
700
|
+
requestId: 'req-456'
|
|
701
|
+
};
|
|
702
|
+
|
|
703
|
+
// Operations with context (would be passed through middleware in real app)
|
|
704
|
+
await users.insert({
|
|
705
|
+
name: 'Alice Johnson',
|
|
706
|
+
email: 'alice@example.com'
|
|
707
|
+
}, userContext);
|
|
708
|
+
|
|
709
|
+
// Analyze audit data
|
|
710
|
+
const userActivity = await analyzer.getUserActivity('admin-123');
|
|
711
|
+
console.log('Recent user activity:', userActivity);
|
|
712
|
+
|
|
713
|
+
const complianceReport = await analyzer.generateComplianceReport(
|
|
714
|
+
new Date(Date.now() - 7 * 24 * 60 * 60 * 1000), // Last 7 days
|
|
715
|
+
new Date()
|
|
716
|
+
);
|
|
717
|
+
|
|
718
|
+
console.log('\n=== Compliance Report ===');
|
|
719
|
+
console.log(`Total operations: ${complianceReport.totalOperations}`);
|
|
720
|
+
console.log('Operation breakdown:', complianceReport.operationBreakdown);
|
|
721
|
+
console.log('Most active resource:',
|
|
722
|
+
Object.entries(complianceReport.resourceActivity)
|
|
723
|
+
.sort(([,a], [,b]) => b - a)[0]
|
|
724
|
+
);
|
|
725
|
+
|
|
726
|
+
// Real-time audit monitoring
|
|
727
|
+
audits.on('insert', (auditLog) => {
|
|
728
|
+
console.log(`🔍 New audit log: ${auditLog.operation} on ${auditLog.resourceName}`);
|
|
729
|
+
|
|
730
|
+
// Security alerts
|
|
731
|
+
if (auditLog.operation === 'delete' && auditLog.userId === 'anonymous') {
|
|
732
|
+
console.warn('🚨 SECURITY ALERT: Anonymous user performed delete operation');
|
|
733
|
+
}
|
|
734
|
+
|
|
735
|
+
if (auditLog.operation === 'get' && auditLog.resourceName === 'sensitive_data') {
|
|
736
|
+
console.warn('🔒 PRIVACY ALERT: Sensitive data accessed');
|
|
737
|
+
}
|
|
738
|
+
});
|
|
739
|
+
|
|
740
|
+
// Audit log retention and cleanup
|
|
741
|
+
setInterval(async () => {
|
|
742
|
+
const thirtyDaysAgo = new Date(Date.now() - 30 * 24 * 60 * 60 * 1000);
|
|
743
|
+
const oldLogs = await audits.list({
|
|
744
|
+
filter: log => new Date(log.timestamp) < thirtyDaysAgo
|
|
745
|
+
});
|
|
746
|
+
|
|
747
|
+
console.log(`Cleaning up ${oldLogs.length} old audit logs`);
|
|
748
|
+
|
|
749
|
+
for (const log of oldLogs) {
|
|
750
|
+
await audits.delete(log.id);
|
|
751
|
+
}
|
|
752
|
+
}, 24 * 60 * 60 * 1000); // Daily cleanup
|
|
753
|
+
```
|
|
754
|
+
|
|
755
|
+
---
|
|
756
|
+
|
|
757
|
+
## 🔍 FullText Plugin
|
|
758
|
+
|
|
759
|
+
Powerful full-text search engine with automatic indexing, scoring, and advanced search capabilities for your s3db resources.
|
|
760
|
+
|
|
761
|
+
### ⚡ Quick Start
|
|
762
|
+
|
|
763
|
+
```javascript
|
|
764
|
+
import { S3db, FullTextPlugin } from 's3db.js';
|
|
765
|
+
|
|
766
|
+
const s3db = new S3db({
|
|
767
|
+
connectionString: "s3://ACCESS_KEY:SECRET_KEY@BUCKET_NAME/databases/myapp",
|
|
768
|
+
plugins: [new FullTextPlugin({
|
|
769
|
+
enabled: true,
|
|
770
|
+
fields: ['title', 'description', 'content']
|
|
771
|
+
})]
|
|
772
|
+
});
|
|
773
|
+
|
|
774
|
+
await s3db.connect();
|
|
775
|
+
|
|
776
|
+
const articles = s3db.resource('articles');
|
|
777
|
+
|
|
778
|
+
// Insert data (automatically indexed)
|
|
779
|
+
await articles.insert({
|
|
780
|
+
title: 'Introduction to Machine Learning',
|
|
781
|
+
description: 'A comprehensive guide to ML basics',
|
|
782
|
+
content: 'Machine learning is a subset of artificial intelligence...'
|
|
783
|
+
});
|
|
784
|
+
|
|
785
|
+
// Search across indexed fields
|
|
786
|
+
const results = await s3db.plugins.fulltext.searchRecords('articles', 'machine learning');
|
|
787
|
+
console.log('Search results:', results);
|
|
788
|
+
```
|
|
789
|
+
|
|
790
|
+
### ⚙️ Configuration Parameters
|
|
791
|
+
|
|
792
|
+
| Parameter | Type | Default | Description |
|
|
793
|
+
|-----------|------|---------|-------------|
|
|
794
|
+
| `enabled` | boolean | `true` | Enable/disable full-text search |
|
|
795
|
+
| `fields` | array | `[]` | Fields to index for search |
|
|
796
|
+
| `minWordLength` | number | `3` | Minimum word length for indexing |
|
|
797
|
+
| `maxResults` | number | `100` | Maximum search results to return |
|
|
798
|
+
| `language` | string | `'en-US'` | Language for text processing |
|
|
799
|
+
| `stopWords` | array | `['the', 'a', 'an', ...]` | Words to exclude from indexing |
|
|
800
|
+
| `stemming` | boolean | `false` | Enable word stemming |
|
|
801
|
+
| `caseSensitive` | boolean | `false` | Case-sensitive search |
|
|
802
|
+
| `fuzzySearch` | boolean | `false` | Enable fuzzy matching |
|
|
803
|
+
| `indexName` | string | `'fulltext_indexes'` | Name of index resource |
|
|
804
|
+
|
|
805
|
+
### Search Result Structure
|
|
806
|
+
|
|
807
|
+
```javascript
|
|
808
|
+
{
|
|
809
|
+
id: 'article-123',
|
|
810
|
+
title: 'Introduction to Machine Learning',
|
|
811
|
+
description: 'A comprehensive guide to ML basics',
|
|
812
|
+
content: 'Machine learning is a subset...',
|
|
813
|
+
_searchScore: 0.85, // Relevance score (0-1)
|
|
814
|
+
_matchedFields: ['title', 'content'], // Fields with matches
|
|
815
|
+
_matchedWords: ['machine', 'learning'], // Matched search terms
|
|
816
|
+
_highlights: { // Highlighted snippets
|
|
817
|
+
title: 'Introduction to <mark>Machine Learning</mark>',
|
|
818
|
+
content: '<mark>Machine learning</mark> is a subset...'
|
|
819
|
+
}
|
|
820
|
+
}
|
|
821
|
+
```
|
|
822
|
+
|
|
823
|
+
### 🔧 Easy Example
|
|
824
|
+
|
|
825
|
+
```javascript
|
|
826
|
+
import { S3db, FullTextPlugin } from 's3db.js';
|
|
827
|
+
|
|
828
|
+
const s3db = new S3db({
|
|
829
|
+
connectionString: "s3://ACCESS_KEY:SECRET_KEY@BUCKET_NAME/databases/myapp",
|
|
830
|
+
plugins: [new FullTextPlugin({
|
|
831
|
+
enabled: true,
|
|
832
|
+
fields: ['name', 'description', 'tags'],
|
|
833
|
+
minWordLength: 2,
|
|
834
|
+
maxResults: 50
|
|
835
|
+
})]
|
|
836
|
+
});
|
|
837
|
+
|
|
838
|
+
await s3db.connect();
|
|
839
|
+
|
|
840
|
+
const products = s3db.resource('products');
|
|
841
|
+
|
|
842
|
+
// Add products with searchable content
|
|
843
|
+
await products.insertMany([
|
|
844
|
+
{
|
|
845
|
+
name: 'Gaming Laptop Pro',
|
|
846
|
+
description: 'High-performance laptop for gaming and productivity',
|
|
847
|
+
tags: ['gaming', 'laptop', 'computer', 'electronics']
|
|
848
|
+
},
|
|
849
|
+
{
|
|
850
|
+
name: 'Wireless Gaming Mouse',
|
|
851
|
+
description: 'Precision wireless mouse designed for gamers',
|
|
852
|
+
tags: ['gaming', 'mouse', 'wireless', 'electronics']
|
|
853
|
+
},
|
|
854
|
+
{
|
|
855
|
+
name: 'Mechanical Keyboard',
|
|
856
|
+
description: 'Professional mechanical keyboard with RGB lighting',
|
|
857
|
+
tags: ['keyboard', 'mechanical', 'typing', 'electronics']
|
|
858
|
+
}
|
|
859
|
+
]);
|
|
860
|
+
|
|
861
|
+
// Search for gaming products
|
|
862
|
+
const gamingProducts = await s3db.plugins.fulltext.searchRecords('products', 'gaming');
|
|
863
|
+
|
|
864
|
+
console.log('\n=== Gaming Products ===');
|
|
865
|
+
gamingProducts.forEach(product => {
|
|
866
|
+
console.log(`${product.name} (Score: ${product._searchScore.toFixed(2)})`);
|
|
867
|
+
console.log(` Matched fields: ${product._matchedFields.join(', ')}`);
|
|
868
|
+
console.log(` Description: ${product.description}`);
|
|
869
|
+
});
|
|
870
|
+
|
|
871
|
+
// Search for wireless devices
|
|
872
|
+
const wirelessProducts = await s3db.plugins.fulltext.searchRecords('products', 'wireless');
|
|
873
|
+
|
|
874
|
+
console.log('\n=== Wireless Products ===');
|
|
875
|
+
wirelessProducts.forEach(product => {
|
|
876
|
+
console.log(`${product.name} - ${product.description}`);
|
|
877
|
+
});
|
|
878
|
+
|
|
879
|
+
// Multi-word search
|
|
880
|
+
const laptopGaming = await s3db.plugins.fulltext.searchRecords('products', 'laptop gaming');
|
|
881
|
+
|
|
882
|
+
console.log('\n=== Laptop Gaming Search ===');
|
|
883
|
+
console.log(`Found ${laptopGaming.length} products matching "laptop gaming"`);
|
|
884
|
+
```
|
|
885
|
+
|
|
886
|
+
### 🚀 Advanced Configuration Example
|
|
887
|
+
|
|
888
|
+
```javascript
|
|
889
|
+
import { S3db, FullTextPlugin } from 's3db.js';
|
|
890
|
+
|
|
891
|
+
const s3db = new S3db({
|
|
892
|
+
connectionString: "s3://ACCESS_KEY:SECRET_KEY@BUCKET_NAME/databases/myapp",
|
|
893
|
+
plugins: [new FullTextPlugin({
|
|
894
|
+
enabled: true,
|
|
895
|
+
|
|
896
|
+
// Comprehensive field indexing
|
|
897
|
+
fields: ['title', 'description', 'content', 'tags', 'category', 'author'],
|
|
898
|
+
|
|
899
|
+
// Advanced text processing
|
|
900
|
+
minWordLength: 2,
|
|
901
|
+
maxResults: 200,
|
|
902
|
+
language: 'en-US',
|
|
903
|
+
stemming: true, // Enable word stemming (run/running/ran)
|
|
904
|
+
caseSensitive: false,
|
|
905
|
+
fuzzySearch: true, // Enable typo tolerance
|
|
906
|
+
|
|
907
|
+
// Custom stop words (words to ignore)
|
|
908
|
+
stopWords: [
|
|
909
|
+
'the', 'a', 'an', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for',
|
|
910
|
+
'of', 'with', 'by', 'is', 'are', 'was', 'were', 'be', 'been', 'being',
|
|
911
|
+
'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would', 'could',
|
|
912
|
+
'should', 'may', 'might', 'must', 'can', 'this', 'that', 'these', 'those'
|
|
913
|
+
],
|
|
914
|
+
|
|
915
|
+
// Advanced search options
|
|
916
|
+
highlightTags: {
|
|
917
|
+
start: '<mark class="highlight">',
|
|
918
|
+
end: '</mark>'
|
|
919
|
+
},
|
|
920
|
+
|
|
921
|
+
// Custom scoring weights per field
|
|
922
|
+
fieldWeights: {
|
|
923
|
+
title: 3.0, // Title matches score higher
|
|
924
|
+
description: 2.0, // Description is important
|
|
925
|
+
content: 1.0, // Content has normal weight
|
|
926
|
+
tags: 2.5, // Tags are highly relevant
|
|
927
|
+
category: 1.5, // Category is moderately important
|
|
928
|
+
author: 1.0 // Author has normal weight
|
|
929
|
+
},
|
|
930
|
+
|
|
931
|
+
// Indexing behavior
|
|
932
|
+
indexName: 'search_indexes',
|
|
933
|
+
autoReindex: true, // Automatically reindex on data changes
|
|
934
|
+
batchSize: 100, // Index batch size
|
|
935
|
+
maxIndexSize: 10000 // Maximum index entries
|
|
936
|
+
})]
|
|
937
|
+
});
|
|
938
|
+
|
|
939
|
+
await s3db.connect();
|
|
940
|
+
|
|
941
|
+
// Advanced search class with custom methods
|
|
942
|
+
class AdvancedSearch {
|
|
943
|
+
constructor(fulltextPlugin) {
|
|
944
|
+
this.plugin = fulltextPlugin;
|
|
945
|
+
}
|
|
946
|
+
|
|
947
|
+
async searchWithFilters(resourceName, query, filters = {}) {
|
|
948
|
+
let results = await this.plugin.searchRecords(resourceName, query);
|
|
949
|
+
|
|
950
|
+
// Apply additional filters
|
|
951
|
+
if (filters.category) {
|
|
952
|
+
results = results.filter(item => item.category === filters.category);
|
|
953
|
+
}
|
|
954
|
+
|
|
955
|
+
if (filters.minScore) {
|
|
956
|
+
results = results.filter(item => item._searchScore >= filters.minScore);
|
|
957
|
+
}
|
|
958
|
+
|
|
959
|
+
if (filters.dateRange) {
|
|
960
|
+
const { start, end } = filters.dateRange;
|
|
961
|
+
results = results.filter(item => {
|
|
962
|
+
const itemDate = new Date(item.createdAt);
|
|
963
|
+
return itemDate >= start && itemDate <= end;
|
|
964
|
+
});
|
|
965
|
+
}
|
|
966
|
+
|
|
967
|
+
return results;
|
|
968
|
+
}
|
|
969
|
+
|
|
970
|
+
async searchMultipleResources(resourceNames, query) {
|
|
971
|
+
const allResults = [];
|
|
972
|
+
|
|
973
|
+
for (const resourceName of resourceNames) {
|
|
974
|
+
const results = await this.plugin.searchRecords(resourceName, query);
|
|
975
|
+
allResults.push(...results.map(item => ({
|
|
976
|
+
...item,
|
|
977
|
+
_resourceType: resourceName
|
|
978
|
+
})));
|
|
979
|
+
}
|
|
980
|
+
|
|
981
|
+
// Sort by relevance across all resources
|
|
982
|
+
return allResults.sort((a, b) => b._searchScore - a._searchScore);
|
|
983
|
+
}
|
|
984
|
+
|
|
985
|
+
async suggestWords(resourceName, partial) {
|
|
986
|
+
// Get all indexed words that start with partial
|
|
987
|
+
const allIndexes = await this.plugin.indexResource.list();
|
|
988
|
+
|
|
989
|
+
const suggestions = allIndexes
|
|
990
|
+
.filter(index =>
|
|
991
|
+
index.resourceName === resourceName &&
|
|
992
|
+
index.word.toLowerCase().startsWith(partial.toLowerCase())
|
|
993
|
+
)
|
|
994
|
+
.sort((a, b) => b.count - a.count) // Sort by frequency
|
|
995
|
+
.slice(0, 10)
|
|
996
|
+
.map(index => index.word);
|
|
997
|
+
|
|
998
|
+
return [...new Set(suggestions)]; // Remove duplicates
|
|
999
|
+
}
|
|
1000
|
+
|
|
1001
|
+
async getSearchAnalytics(resourceName) {
|
|
1002
|
+
const indexes = await this.plugin.indexResource.list();
|
|
1003
|
+
const resourceIndexes = indexes.filter(i => i.resourceName === resourceName);
|
|
1004
|
+
|
|
1005
|
+
const analytics = {
|
|
1006
|
+
totalWords: resourceIndexes.length,
|
|
1007
|
+
totalOccurrences: resourceIndexes.reduce((sum, i) => sum + i.count, 0),
|
|
1008
|
+
avgWordsPerDocument: 0,
|
|
1009
|
+
topWords: resourceIndexes
|
|
1010
|
+
.sort((a, b) => b.count - a.count)
|
|
1011
|
+
.slice(0, 20)
|
|
1012
|
+
.map(i => ({ word: i.word, count: i.count })),
|
|
1013
|
+
wordDistribution: {},
|
|
1014
|
+
lastIndexed: Math.max(...resourceIndexes.map(i => new Date(i.lastUpdated)))
|
|
1015
|
+
};
|
|
1016
|
+
|
|
1017
|
+
// Calculate word distribution by frequency ranges
|
|
1018
|
+
resourceIndexes.forEach(index => {
|
|
1019
|
+
const range = index.count < 5 ? 'rare' :
|
|
1020
|
+
index.count < 20 ? 'common' : 'frequent';
|
|
1021
|
+
analytics.wordDistribution[range] = (analytics.wordDistribution[range] || 0) + 1;
|
|
1022
|
+
});
|
|
1023
|
+
|
|
1024
|
+
return analytics;
|
|
1025
|
+
}
|
|
1026
|
+
}
|
|
1027
|
+
|
|
1028
|
+
// Setup sample data
|
|
1029
|
+
const articles = s3db.resource('articles');
|
|
1030
|
+
const products = s3db.resource('products');
|
|
1031
|
+
|
|
1032
|
+
await articles.insertMany([
|
|
1033
|
+
{
|
|
1034
|
+
title: 'Advanced JavaScript Techniques',
|
|
1035
|
+
description: 'Deep dive into modern JavaScript features',
|
|
1036
|
+
content: 'JavaScript has evolved significantly with ES6+ features...',
|
|
1037
|
+
tags: ['javascript', 'programming', 'web-development'],
|
|
1038
|
+
category: 'technology',
|
|
1039
|
+
author: 'John Smith'
|
|
1040
|
+
},
|
|
1041
|
+
{
|
|
1042
|
+
title: 'Machine Learning Fundamentals',
|
|
1043
|
+
description: 'Introduction to ML concepts and algorithms',
|
|
1044
|
+
content: 'Machine learning is revolutionizing how we process data...',
|
|
1045
|
+
tags: ['machine-learning', 'ai', 'data-science'],
|
|
1046
|
+
category: 'technology',
|
|
1047
|
+
author: 'Jane Doe'
|
|
1048
|
+
},
|
|
1049
|
+
{
|
|
1050
|
+
title: 'Sustainable Cooking Tips',
|
|
1051
|
+
description: 'Eco-friendly approaches to home cooking',
|
|
1052
|
+
content: 'Sustainable cooking practices can reduce your environmental impact...',
|
|
1053
|
+
tags: ['cooking', 'sustainability', 'environment'],
|
|
1054
|
+
category: 'lifestyle',
|
|
1055
|
+
author: 'Chef Maria'
|
|
1056
|
+
}
|
|
1057
|
+
]);
|
|
1058
|
+
|
|
1059
|
+
// Initialize advanced search
|
|
1060
|
+
const search = new AdvancedSearch(s3db.plugins.fulltext);
|
|
1061
|
+
|
|
1062
|
+
// Complex search with filters
|
|
1063
|
+
const techArticles = await search.searchWithFilters('articles', 'javascript programming', {
|
|
1064
|
+
category: 'technology',
|
|
1065
|
+
minScore: 0.5
|
|
1066
|
+
});
|
|
1067
|
+
|
|
1068
|
+
console.log('\n=== Technology Articles ===');
|
|
1069
|
+
techArticles.forEach(article => {
|
|
1070
|
+
console.log(`${article.title} by ${article.author}`);
|
|
1071
|
+
console.log(` Score: ${article._searchScore.toFixed(3)}`);
|
|
1072
|
+
console.log(` Matches: ${article._matchedWords.join(', ')}`);
|
|
1073
|
+
console.log(` Highlighted: ${article._highlights?.title || article.title}`);
|
|
1074
|
+
});
|
|
1075
|
+
|
|
1076
|
+
// Multi-resource search
|
|
1077
|
+
const allContent = await search.searchMultipleResources(['articles', 'products'], 'technology');
|
|
1078
|
+
|
|
1079
|
+
console.log('\n=== Cross-Resource Search ===');
|
|
1080
|
+
allContent.forEach(item => {
|
|
1081
|
+
console.log(`[${item._resourceType.toUpperCase()}] ${item.title || item.name}`);
|
|
1082
|
+
console.log(` Score: ${item._searchScore.toFixed(3)}`);
|
|
1083
|
+
});
|
|
1084
|
+
|
|
1085
|
+
// Auto-complete suggestions
|
|
1086
|
+
const suggestions = await search.suggestWords('articles', 'java');
|
|
1087
|
+
console.log('\nSuggestions for "java":', suggestions);
|
|
1088
|
+
|
|
1089
|
+
// Search analytics
|
|
1090
|
+
const analytics = await search.getSearchAnalytics('articles');
|
|
1091
|
+
console.log('\n=== Search Analytics ===');
|
|
1092
|
+
console.log(`Total indexed words: ${analytics.totalWords}`);
|
|
1093
|
+
console.log(`Total word occurrences: ${analytics.totalOccurrences}`);
|
|
1094
|
+
console.log('Top words:', analytics.topWords.slice(0, 5));
|
|
1095
|
+
console.log('Word distribution:', analytics.wordDistribution);
|
|
1096
|
+
|
|
1097
|
+
// Real-time search monitoring
|
|
1098
|
+
s3db.plugins.fulltext.on('indexed', (data) => {
|
|
1099
|
+
console.log(`🔍 Indexed: ${data.resourceName} - ${data.recordId}`);
|
|
1100
|
+
});
|
|
1101
|
+
|
|
1102
|
+
s3db.plugins.fulltext.on('searched', (data) => {
|
|
1103
|
+
console.log(`🔎 Search: "${data.query}" in ${data.resourceName} (${data.results} results)`);
|
|
1104
|
+
});
|
|
1105
|
+
|
|
1106
|
+
// Performance monitoring
|
|
1107
|
+
console.time('Search Performance');
|
|
1108
|
+
const perfResults = await s3db.plugins.fulltext.searchRecords('articles', 'machine learning javascript');
|
|
1109
|
+
console.timeEnd('Search Performance');
|
|
1110
|
+
console.log(`Search returned ${perfResults.length} results`);
|
|
1111
|
+
```
|
|
1112
|
+
|
|
1113
|
+
---
|
|
1114
|
+
|
|
1115
|
+
## 📊 Metrics Plugin
|
|
1116
|
+
|
|
1117
|
+
Comprehensive performance monitoring and usage analytics system that tracks operation timing, resource usage, errors, and provides detailed insights.
|
|
1118
|
+
|
|
1119
|
+
### ⚡ Quick Start
|
|
1120
|
+
|
|
1121
|
+
```javascript
|
|
1122
|
+
import { S3db, MetricsPlugin } from 's3db.js';
|
|
1123
|
+
|
|
1124
|
+
const s3db = new S3db({
|
|
1125
|
+
connectionString: "s3://ACCESS_KEY:SECRET_KEY@BUCKET_NAME/databases/myapp",
|
|
1126
|
+
plugins: [new MetricsPlugin({ enabled: true })]
|
|
1127
|
+
});
|
|
1128
|
+
|
|
1129
|
+
await s3db.connect();
|
|
1130
|
+
|
|
1131
|
+
// Use your database normally - metrics are collected automatically
|
|
1132
|
+
const users = s3db.resource('users');
|
|
1133
|
+
await users.insert({ name: 'John', email: 'john@example.com' });
|
|
1134
|
+
await users.list();
|
|
1135
|
+
await users.count();
|
|
1136
|
+
|
|
1137
|
+
// Get comprehensive metrics
|
|
1138
|
+
const metrics = await s3db.plugins.metrics.getMetrics();
|
|
1139
|
+
console.log('Performance metrics:', metrics);
|
|
1140
|
+
```
|
|
1141
|
+
|
|
1142
|
+
### ⚙️ Configuration Parameters
|
|
1143
|
+
|
|
1144
|
+
| Parameter | Type | Default | Description |
|
|
1145
|
+
|-----------|------|---------|-------------|
|
|
1146
|
+
| `enabled` | boolean | `true` | Enable/disable metrics collection |
|
|
1147
|
+
| `collectPerformance` | boolean | `true` | Track operation timing and performance |
|
|
1148
|
+
| `collectErrors` | boolean | `true` | Track errors and failures |
|
|
1149
|
+
| `collectUsage` | boolean | `true` | Track resource usage patterns |
|
|
1150
|
+
| `retentionDays` | number | `30` | Days to retain metric data |
|
|
1151
|
+
| `flushInterval` | number | `60000` | Interval to flush metrics (ms) |
|
|
1152
|
+
| `sampleRate` | number | `1.0` | Sampling rate for metrics (0.0-1.0) |
|
|
1153
|
+
| `trackSlowQueries` | boolean | `true` | Track slow operations |
|
|
1154
|
+
| `slowQueryThreshold` | number | `1000` | Threshold for slow queries (ms) |
|
|
1155
|
+
| `batchSize` | number | `100` | Batch size for metric storage |
|
|
1156
|
+
|
|
1157
|
+
### Metrics Data Structure
|
|
1158
|
+
|
|
1159
|
+
```javascript
|
|
1160
|
+
{
|
|
1161
|
+
performance: {
|
|
1162
|
+
averageResponseTime: 245, // milliseconds
|
|
1163
|
+
totalRequests: 1250,
|
|
1164
|
+
requestsPerSecond: 12.5,
|
|
1165
|
+
slowestOperations: [
|
|
1166
|
+
{ operation: "list", resource: "users", avgTime: 450, count: 50 },
|
|
1167
|
+
{ operation: "get", resource: "products", avgTime: 320, count: 200 }
|
|
1168
|
+
],
|
|
1169
|
+
operationTiming: {
|
|
1170
|
+
insert: { avg: 180, min: 120, max: 350, total: 50 },
|
|
1171
|
+
update: { avg: 160, min: 90, max: 280, total: 30 },
|
|
1172
|
+
get: { avg: 95, min: 45, max: 180, total: 200 }
|
|
1173
|
+
}
|
|
1174
|
+
},
|
|
1175
|
+
usage: {
|
|
1176
|
+
resources: {
|
|
1177
|
+
users: { inserts: 150, updates: 75, deletes: 10, reads: 800 },
|
|
1178
|
+
products: { inserts: 300, updates: 120, deletes: 25, reads: 1200 }
|
|
1179
|
+
},
|
|
1180
|
+
totalOperations: 2680,
|
|
1181
|
+
mostActiveResource: "products",
|
|
1182
|
+
peakUsageHour: "14:00",
|
|
1183
|
+
dailyPatterns: { /* hourly usage data */ }
|
|
1184
|
+
},
|
|
1185
|
+
errors: {
|
|
1186
|
+
total: 15,
|
|
1187
|
+
byType: {
|
|
1188
|
+
"ValidationError": 8,
|
|
1189
|
+
"NotFoundError": 5,
|
|
1190
|
+
"PermissionError": 2
|
|
1191
|
+
},
|
|
1192
|
+
byResource: { users: 10, products: 5 },
|
|
1193
|
+
errorRate: 0.0056 // 0.56%
|
|
1194
|
+
},
|
|
1195
|
+
cache: {
|
|
1196
|
+
hitRate: 0.78,
|
|
1197
|
+
totalHits: 980,
|
|
1198
|
+
totalMisses: 270
|
|
1199
|
+
}
|
|
1200
|
+
}
|
|
1201
|
+
```
|
|
1202
|
+
|
|
1203
|
+
### 🔧 Easy Example
|
|
1204
|
+
|
|
1205
|
+
```javascript
|
|
1206
|
+
import { S3db, MetricsPlugin } from 's3db.js';
|
|
1207
|
+
|
|
1208
|
+
const s3db = new S3db({
|
|
1209
|
+
connectionString: "s3://ACCESS_KEY:SECRET_KEY@BUCKET_NAME/databases/myapp",
|
|
1210
|
+
plugins: [new MetricsPlugin({
|
|
1211
|
+
enabled: true,
|
|
1212
|
+
collectPerformance: true,
|
|
1213
|
+
collectErrors: true,
|
|
1214
|
+
flushInterval: 30000 // 30 seconds
|
|
1215
|
+
})]
|
|
1216
|
+
});
|
|
1217
|
+
|
|
1218
|
+
await s3db.connect();
|
|
1219
|
+
|
|
1220
|
+
const orders = s3db.resource('orders');
|
|
1221
|
+
|
|
1222
|
+
// Simulate various operations
|
|
1223
|
+
console.log('Performing operations...');
|
|
1224
|
+
|
|
1225
|
+
// Fast operations
|
|
1226
|
+
for (let i = 0; i < 10; i++) {
|
|
1227
|
+
await orders.insert({
|
|
1228
|
+
customerId: `customer-${i}`,
|
|
1229
|
+
amount: Math.random() * 1000,
|
|
1230
|
+
status: 'pending'
|
|
1231
|
+
});
|
|
1232
|
+
}
|
|
1233
|
+
|
|
1234
|
+
// Query operations
|
|
1235
|
+
await orders.count();
|
|
1236
|
+
await orders.list({ limit: 5 });
|
|
1237
|
+
|
|
1238
|
+
// Some updates
|
|
1239
|
+
const orderList = await orders.list({ limit: 3 });
|
|
1240
|
+
for (const order of orderList) {
|
|
1241
|
+
await orders.update(order.id, { status: 'processing' });
|
|
1242
|
+
}
|
|
1243
|
+
|
|
1244
|
+
// Get performance metrics
|
|
1245
|
+
const metrics = await s3db.plugins.metrics.getMetrics();
|
|
1246
|
+
|
|
1247
|
+
console.log('\n=== Performance Report ===');
|
|
1248
|
+
console.log(`Average response time: ${metrics.performance.averageResponseTime}ms`);
|
|
1249
|
+
console.log(`Total operations: ${metrics.usage.totalOperations}`);
|
|
1250
|
+
console.log(`Error rate: ${(metrics.errors.errorRate * 100).toFixed(2)}%`);
|
|
1251
|
+
|
|
1252
|
+
console.log('\n=== Operation Breakdown ===');
|
|
1253
|
+
Object.entries(metrics.performance.operationTiming).forEach(([op, timing]) => {
|
|
1254
|
+
console.log(`${op.toUpperCase()}: avg ${timing.avg}ms (${timing.total} operations)`);
|
|
1255
|
+
});
|
|
1256
|
+
|
|
1257
|
+
console.log('\n=== Resource Usage ===');
|
|
1258
|
+
Object.entries(metrics.usage.resources).forEach(([resource, usage]) => {
|
|
1259
|
+
const total = Object.values(usage).reduce((sum, count) => sum + count, 0);
|
|
1260
|
+
console.log(`${resource}: ${total} total operations`);
|
|
1261
|
+
});
|
|
1262
|
+
```
|
|
1263
|
+
|
|
1264
|
+
### 🚀 Advanced Configuration Example
|
|
1265
|
+
|
|
1266
|
+
```javascript
|
|
1267
|
+
import { S3db, MetricsPlugin } from 's3db.js';
|
|
1268
|
+
|
|
1269
|
+
const s3db = new S3db({
|
|
1270
|
+
connectionString: "s3://ACCESS_KEY:SECRET_KEY@BUCKET_NAME/databases/myapp",
|
|
1271
|
+
plugins: [new MetricsPlugin({
|
|
1272
|
+
enabled: true,
|
|
1273
|
+
|
|
1274
|
+
// Comprehensive monitoring
|
|
1275
|
+
collectPerformance: true,
|
|
1276
|
+
collectErrors: true,
|
|
1277
|
+
collectUsage: true,
|
|
1278
|
+
|
|
1279
|
+
// Advanced settings
|
|
1280
|
+
retentionDays: 90, // 3 months of data
|
|
1281
|
+
flushInterval: 10000, // 10 seconds
|
|
1282
|
+
sampleRate: 1.0, // 100% sampling
|
|
1283
|
+
|
|
1284
|
+
// Performance thresholds
|
|
1285
|
+
trackSlowQueries: true,
|
|
1286
|
+
slowQueryThreshold: 500, // 500ms threshold
|
|
1287
|
+
|
|
1288
|
+
// Storage optimization
|
|
1289
|
+
batchSize: 50,
|
|
1290
|
+
compressionEnabled: true,
|
|
1291
|
+
|
|
1292
|
+
// Custom alerting thresholds
|
|
1293
|
+
alertThresholds: {
|
|
1294
|
+
errorRate: 0.05, // 5% error rate
|
|
1295
|
+
avgResponseTime: 1000, // 1 second average
|
|
1296
|
+
memoryUsage: 0.9 // 90% memory usage
|
|
1297
|
+
},
|
|
1298
|
+
|
|
1299
|
+
// Event hooks
|
|
1300
|
+
onSlowQuery: (operation, resource, duration) => {
|
|
1301
|
+
console.warn(`🐌 Slow query: ${operation} on ${resource} took ${duration}ms`);
|
|
1302
|
+
},
|
|
1303
|
+
|
|
1304
|
+
onHighErrorRate: (resource, errorRate) => {
|
|
1305
|
+
console.error(`🚨 High error rate: ${resource} has ${(errorRate * 100).toFixed(1)}% errors`);
|
|
1306
|
+
},
|
|
1307
|
+
|
|
1308
|
+
onThresholdExceeded: (metric, value, threshold) => {
|
|
1309
|
+
console.warn(`⚠️ Threshold exceeded: ${metric} = ${value} (threshold: ${threshold})`);
|
|
1310
|
+
}
|
|
1311
|
+
})]
|
|
1312
|
+
});
|
|
1313
|
+
|
|
1314
|
+
await s3db.connect();
|
|
1315
|
+
|
|
1316
|
+
// Advanced metrics analysis class
|
|
1317
|
+
class MetricsAnalyzer {
|
|
1318
|
+
constructor(metricsPlugin) {
|
|
1319
|
+
this.plugin = metricsPlugin;
|
|
1320
|
+
this.alertHandlers = new Map();
|
|
1321
|
+
}
|
|
1322
|
+
|
|
1323
|
+
addAlertHandler(condition, handler) {
|
|
1324
|
+
this.alertHandlers.set(condition, handler);
|
|
1325
|
+
}
|
|
1326
|
+
|
|
1327
|
+
async analyzePerformance(timeRange = 3600000) { // 1 hour
|
|
1328
|
+
const metrics = await this.plugin.getMetrics();
|
|
1329
|
+
const analysis = {
|
|
1330
|
+
summary: {
|
|
1331
|
+
totalOperations: metrics.usage.totalOperations,
|
|
1332
|
+
avgResponseTime: metrics.performance.averageResponseTime,
|
|
1333
|
+
errorRate: metrics.errors.errorRate,
|
|
1334
|
+
slowQueries: metrics.performance.slowestOperations.length
|
|
1335
|
+
},
|
|
1336
|
+
recommendations: [],
|
|
1337
|
+
alerts: []
|
|
1338
|
+
};
|
|
1339
|
+
|
|
1340
|
+
// Performance analysis
|
|
1341
|
+
if (metrics.performance.averageResponseTime > 500) {
|
|
1342
|
+
analysis.recommendations.push({
|
|
1343
|
+
type: 'performance',
|
|
1344
|
+
message: 'Average response time is high. Consider adding caching or optimizing queries.',
|
|
1345
|
+
priority: 'high'
|
|
1346
|
+
});
|
|
1347
|
+
}
|
|
1348
|
+
|
|
1349
|
+
// Error rate analysis
|
|
1350
|
+
if (metrics.errors.errorRate > 0.02) { // 2%
|
|
1351
|
+
analysis.alerts.push({
|
|
1352
|
+
type: 'error_rate',
|
|
1353
|
+
message: `Error rate (${(metrics.errors.errorRate * 100).toFixed(2)}%) exceeds threshold`,
|
|
1354
|
+
severity: 'warning'
|
|
1355
|
+
});
|
|
1356
|
+
}
|
|
1357
|
+
|
|
1358
|
+
// Resource usage patterns
|
|
1359
|
+
const resourceUsage = Object.entries(metrics.usage.resources);
|
|
1360
|
+
const imbalancedResources = resourceUsage.filter(([name, usage]) => {
|
|
1361
|
+
const writes = usage.inserts + usage.updates + usage.deletes;
|
|
1362
|
+
const reads = usage.reads;
|
|
1363
|
+
return writes > 0 && (reads / writes) < 0.1; // Very low read/write ratio
|
|
1364
|
+
});
|
|
1365
|
+
|
|
1366
|
+
if (imbalancedResources.length > 0) {
|
|
1367
|
+
analysis.recommendations.push({
|
|
1368
|
+
type: 'usage_pattern',
|
|
1369
|
+
message: `Resources with low read/write ratio: ${imbalancedResources.map(([name]) => name).join(', ')}`,
|
|
1370
|
+
priority: 'medium'
|
|
1371
|
+
});
|
|
1372
|
+
}
|
|
1373
|
+
|
|
1374
|
+
return analysis;
|
|
1375
|
+
}
|
|
1376
|
+
|
|
1377
|
+
async generateReport(format = 'console') {
|
|
1378
|
+
const metrics = await this.plugin.getMetrics();
|
|
1379
|
+
const analysis = await this.analyzePerformance();
|
|
1380
|
+
|
|
1381
|
+
if (format === 'console') {
|
|
1382
|
+
console.log('\n=== 📊 COMPREHENSIVE METRICS REPORT ===');
|
|
1383
|
+
|
|
1384
|
+
// Performance Summary
|
|
1385
|
+
console.log('\n🚀 Performance Summary:');
|
|
1386
|
+
console.log(` Total Operations: ${analysis.summary.totalOperations.toLocaleString()}`);
|
|
1387
|
+
console.log(` Average Response Time: ${analysis.summary.avgResponseTime}ms`);
|
|
1388
|
+
console.log(` Error Rate: ${(analysis.summary.errorRate * 100).toFixed(2)}%`);
|
|
1389
|
+
console.log(` Slow Queries: ${analysis.summary.slowQueries}`);
|
|
1390
|
+
|
|
1391
|
+
// Operation Breakdown
|
|
1392
|
+
console.log('\n⏱️ Operation Timing:');
|
|
1393
|
+
Object.entries(metrics.performance.operationTiming).forEach(([op, timing]) => {
|
|
1394
|
+
console.log(` ${op.toUpperCase()}:`);
|
|
1395
|
+
console.log(` Average: ${timing.avg}ms`);
|
|
1396
|
+
console.log(` Range: ${timing.min}ms - ${timing.max}ms`);
|
|
1397
|
+
console.log(` Count: ${timing.total}`);
|
|
1398
|
+
});
|
|
1399
|
+
|
|
1400
|
+
// Resource Activity
|
|
1401
|
+
console.log('\n📈 Resource Activity:');
|
|
1402
|
+
Object.entries(metrics.usage.resources)
|
|
1403
|
+
.sort(([,a], [,b]) => {
|
|
1404
|
+
const totalA = Object.values(a).reduce((sum, val) => sum + val, 0);
|
|
1405
|
+
const totalB = Object.values(b).reduce((sum, val) => sum + val, 0);
|
|
1406
|
+
return totalB - totalA;
|
|
1407
|
+
})
|
|
1408
|
+
.forEach(([resource, usage]) => {
|
|
1409
|
+
const total = Object.values(usage).reduce((sum, val) => sum + val, 0);
|
|
1410
|
+
console.log(` ${resource}: ${total} operations`);
|
|
1411
|
+
console.log(` Reads: ${usage.reads}, Writes: ${usage.inserts + usage.updates + usage.deletes}`);
|
|
1412
|
+
});
|
|
1413
|
+
|
|
1414
|
+
// Error Analysis
|
|
1415
|
+
if (metrics.errors.total > 0) {
|
|
1416
|
+
console.log('\n🚨 Error Analysis:');
|
|
1417
|
+
console.log(` Total Errors: ${metrics.errors.total}`);
|
|
1418
|
+
console.log(' By Type:');
|
|
1419
|
+
Object.entries(metrics.errors.byType).forEach(([type, count]) => {
|
|
1420
|
+
console.log(` ${type}: ${count}`);
|
|
1421
|
+
});
|
|
1422
|
+
}
|
|
1423
|
+
|
|
1424
|
+
// Recommendations
|
|
1425
|
+
if (analysis.recommendations.length > 0) {
|
|
1426
|
+
console.log('\n💡 Recommendations:');
|
|
1427
|
+
analysis.recommendations.forEach(rec => {
|
|
1428
|
+
const emoji = rec.priority === 'high' ? '🔴' : rec.priority === 'medium' ? '🟡' : '🟢';
|
|
1429
|
+
console.log(` ${emoji} [${rec.priority.toUpperCase()}] ${rec.message}`);
|
|
1430
|
+
});
|
|
1431
|
+
}
|
|
1432
|
+
|
|
1433
|
+
// Alerts
|
|
1434
|
+
if (analysis.alerts.length > 0) {
|
|
1435
|
+
console.log('\n⚠️ Active Alerts:');
|
|
1436
|
+
analysis.alerts.forEach(alert => {
|
|
1437
|
+
console.log(` 🚨 ${alert.message}`);
|
|
1438
|
+
});
|
|
1439
|
+
}
|
|
1440
|
+
}
|
|
1441
|
+
|
|
1442
|
+
return { metrics, analysis };
|
|
1443
|
+
}
|
|
1444
|
+
|
|
1445
|
+
async exportMetrics(filename) {
|
|
1446
|
+
const metrics = await this.plugin.getMetrics();
|
|
1447
|
+
const data = {
|
|
1448
|
+
timestamp: new Date().toISOString(),
|
|
1449
|
+
metrics,
|
|
1450
|
+
analysis: await this.analyzePerformance()
|
|
1451
|
+
};
|
|
1452
|
+
|
|
1453
|
+
// In real implementation, save to file
|
|
1454
|
+
console.log(`📁 Metrics exported to ${filename}`);
|
|
1455
|
+
return data;
|
|
1456
|
+
}
|
|
1457
|
+
|
|
1458
|
+
startRealTimeMonitoring(interval = 5000) {
|
|
1459
|
+
const monitor = setInterval(async () => {
|
|
1460
|
+
const metrics = await this.plugin.getMetrics();
|
|
1461
|
+
|
|
1462
|
+
// Check alert conditions
|
|
1463
|
+
this.alertHandlers.forEach((handler, condition) => {
|
|
1464
|
+
if (condition(metrics)) {
|
|
1465
|
+
handler(metrics);
|
|
1466
|
+
}
|
|
1467
|
+
});
|
|
1468
|
+
|
|
1469
|
+
// Auto-optimization suggestions
|
|
1470
|
+
if (metrics.performance.averageResponseTime > 1000) {
|
|
1471
|
+
console.log('💡 Suggestion: Consider implementing caching for frequently accessed data');
|
|
1472
|
+
}
|
|
1473
|
+
|
|
1474
|
+
if (metrics.errors.errorRate > 0.05) {
|
|
1475
|
+
console.log('🚨 Alert: Error rate is above 5% - investigate immediately');
|
|
1476
|
+
}
|
|
1477
|
+
|
|
1478
|
+
}, interval);
|
|
1479
|
+
|
|
1480
|
+
return monitor;
|
|
1481
|
+
}
|
|
1482
|
+
}
|
|
1483
|
+
|
|
1484
|
+
// Simulate complex workload
|
|
1485
|
+
const users = s3db.resource('users');
|
|
1486
|
+
const products = s3db.resource('products');
|
|
1487
|
+
const orders = s3db.resource('orders');
|
|
1488
|
+
|
|
1489
|
+
// Setup metrics analyzer
|
|
1490
|
+
const analyzer = new MetricsAnalyzer(s3db.plugins.metrics);
|
|
1491
|
+
|
|
1492
|
+
// Add custom alert handlers
|
|
1493
|
+
analyzer.addAlertHandler(
|
|
1494
|
+
(metrics) => metrics.errors.errorRate > 0.03,
|
|
1495
|
+
(metrics) => console.log('🚨 Error rate alert triggered!')
|
|
1496
|
+
);
|
|
1497
|
+
|
|
1498
|
+
analyzer.addAlertHandler(
|
|
1499
|
+
(metrics) => metrics.performance.averageResponseTime > 800,
|
|
1500
|
+
(metrics) => console.log('⏰ Performance degradation detected!')
|
|
1501
|
+
);
|
|
1502
|
+
|
|
1503
|
+
// Simulate workload
|
|
1504
|
+
console.log('🔄 Simulating complex workload...');
|
|
1505
|
+
|
|
1506
|
+
// Bulk operations
|
|
1507
|
+
const userData = Array.from({ length: 50 }, (_, i) => ({
|
|
1508
|
+
name: `User ${i}`,
|
|
1509
|
+
email: `user${i}@example.com`,
|
|
1510
|
+
role: i % 3 === 0 ? 'admin' : 'user'
|
|
1511
|
+
}));
|
|
1512
|
+
|
|
1513
|
+
await users.insertMany(userData);
|
|
1514
|
+
|
|
1515
|
+
// Mixed operations with some errors
|
|
1516
|
+
for (let i = 0; i < 20; i++) {
|
|
1517
|
+
try {
|
|
1518
|
+
await products.insert({
|
|
1519
|
+
name: `Product ${i}`,
|
|
1520
|
+
price: Math.random() * 100,
|
|
1521
|
+
category: ['electronics', 'books', 'clothing'][i % 3]
|
|
1522
|
+
});
|
|
1523
|
+
|
|
1524
|
+
if (i % 5 === 0) {
|
|
1525
|
+
// Simulate some slow operations
|
|
1526
|
+
await new Promise(resolve => setTimeout(resolve, 600));
|
|
1527
|
+
await products.list({ limit: 20 });
|
|
1528
|
+
}
|
|
1529
|
+
|
|
1530
|
+
if (i % 10 === 0) {
|
|
1531
|
+
// Simulate some errors
|
|
1532
|
+
try {
|
|
1533
|
+
await products.get('non-existent-id');
|
|
1534
|
+
} catch (error) {
|
|
1535
|
+
// Expected error for testing
|
|
1536
|
+
}
|
|
1537
|
+
}
|
|
1538
|
+
|
|
1539
|
+
} catch (error) {
|
|
1540
|
+
// Handle errors
|
|
1541
|
+
}
|
|
1542
|
+
}
|
|
1543
|
+
|
|
1544
|
+
// Generate comprehensive report
|
|
1545
|
+
await analyzer.generateReport();
|
|
1546
|
+
|
|
1547
|
+
// Start real-time monitoring
|
|
1548
|
+
const monitor = analyzer.startRealTimeMonitoring(3000);
|
|
1549
|
+
|
|
1550
|
+
// Export metrics for external analysis
|
|
1551
|
+
await analyzer.exportMetrics('metrics-export.json');
|
|
1552
|
+
|
|
1553
|
+
// Stop monitoring after demo
|
|
1554
|
+
setTimeout(() => {
|
|
1555
|
+
clearInterval(monitor);
|
|
1556
|
+
console.log('\n✅ Metrics demonstration completed');
|
|
1557
|
+
}, 15000);
|
|
1558
|
+
```
|
|
1559
|
+
|
|
1560
|
+
---
|
|
1561
|
+
|
|
1562
|
+
## 🔄 Replicator Plugin
|
|
1563
|
+
|
|
1564
|
+
Powerful data replication system that synchronizes your s3db data to multiple targets including other S3DB instances, SQS queues, BigQuery, and PostgreSQL databases.
|
|
1565
|
+
|
|
1566
|
+
### ⚡ Quick Start
|
|
1567
|
+
|
|
1568
|
+
```javascript
|
|
1569
|
+
import { S3db, ReplicatorPlugin } from 's3db.js';
|
|
1570
|
+
|
|
1571
|
+
const s3db = new S3db({
|
|
1572
|
+
connectionString: "s3://ACCESS_KEY:SECRET_KEY@BUCKET_NAME/databases/myapp",
|
|
1573
|
+
plugins: [new ReplicatorPlugin({
|
|
1574
|
+
replicators: [
|
|
1575
|
+
{
|
|
1576
|
+
driver: 's3db',
|
|
1577
|
+
resources: ['users'],
|
|
1578
|
+
config: {
|
|
1579
|
+
connectionString: "s3://BACKUP_KEY:BACKUP_SECRET@BACKUP_BUCKET/backup"
|
|
1580
|
+
}
|
|
1581
|
+
}
|
|
1582
|
+
]
|
|
1583
|
+
})]
|
|
1584
|
+
});
|
|
1585
|
+
|
|
1586
|
+
await s3db.connect();
|
|
1587
|
+
|
|
1588
|
+
// Data is automatically replicated
|
|
1589
|
+
const users = s3db.resource('users');
|
|
1590
|
+
await users.insert({ name: 'John', email: 'john@example.com' });
|
|
1591
|
+
// This insert is automatically replicated to the backup database
|
|
1592
|
+
```
|
|
1593
|
+
|
|
1594
|
+
### ⚙️ Configuration Parameters
|
|
1595
|
+
|
|
1596
|
+
| Parameter | Type | Default | Description |
|
|
1597
|
+
|-----------|------|---------|-------------|
|
|
1598
|
+
| `enabled` | boolean | `true` | Enable/disable replication |
|
|
1599
|
+
| `replicators` | array | `[]` | Array of replicator configurations |
|
|
1600
|
+
| `persistReplicatorLog` | boolean | `false` | Store replication logs in database |
|
|
1601
|
+
| `replicatorLogResource` | string | `'replicator_logs'` | Name of log resource |
|
|
1602
|
+
| `batchSize` | number | `10` | Batch size for bulk operations |
|
|
1603
|
+
| `retryAttempts` | number | `3` | Retry failed replications |
|
|
1604
|
+
| `retryDelay` | number | `1000` | Delay between retries (ms) |
|
|
1605
|
+
| `syncInterval` | number | `0` | Auto-sync interval (0 = disabled) |
|
|
1606
|
+
|
|
1607
|
+
### Replicator Drivers
|
|
1608
|
+
|
|
1609
|
+
#### S3DB Replicator
|
|
1610
|
+
|
|
1611
|
+
Replicate to another S3DB instance:
|
|
1612
|
+
|
|
1613
|
+
```javascript
|
|
1614
|
+
{
|
|
1615
|
+
driver: 's3db',
|
|
1616
|
+
resources: ['users', 'products'],
|
|
1617
|
+
config: {
|
|
1618
|
+
connectionString: "s3://BACKUP_KEY:BACKUP_SECRET@BACKUP_BUCKET/backup"
|
|
1619
|
+
}
|
|
1620
|
+
}
|
|
1621
|
+
```
|
|
1622
|
+
|
|
1623
|
+
#### SQS Replicator
|
|
1624
|
+
|
|
1625
|
+
Send changes to AWS SQS queues:
|
|
1626
|
+
|
|
1627
|
+
```javascript
|
|
1628
|
+
{
|
|
1629
|
+
driver: 'sqs',
|
|
1630
|
+
resources: ['orders'],
|
|
1631
|
+
config: {
|
|
1632
|
+
queueUrl: 'https://sqs.us-east-1.amazonaws.com/123456789012/my-queue',
|
|
1633
|
+
region: 'us-east-1',
|
|
1634
|
+
messageGroupId: 's3db-replicator',
|
|
1635
|
+
deduplicationId: true
|
|
1636
|
+
}
|
|
1637
|
+
}
|
|
1638
|
+
```
|
|
1639
|
+
|
|
1640
|
+
#### BigQuery Replicator
|
|
1641
|
+
|
|
1642
|
+
Replicate to Google BigQuery:
|
|
1643
|
+
|
|
1644
|
+
```javascript
|
|
1645
|
+
{
|
|
1646
|
+
driver: 'bigquery',
|
|
1647
|
+
resources: {
|
|
1648
|
+
users: [{ actions: ['insert', 'update'], table: 'users_table' }],
|
|
1649
|
+
orders: 'orders_table'
|
|
1650
|
+
},
|
|
1651
|
+
config: {
|
|
1652
|
+
projectId: 'my-project',
|
|
1653
|
+
datasetId: 'analytics',
|
|
1654
|
+
credentials: { /* service account */ }
|
|
1655
|
+
}
|
|
1656
|
+
}
|
|
1657
|
+
```
|
|
1658
|
+
|
|
1659
|
+
#### PostgreSQL Replicator
|
|
1660
|
+
|
|
1661
|
+
Replicate to PostgreSQL database:
|
|
1662
|
+
|
|
1663
|
+
```javascript
|
|
1664
|
+
{
|
|
1665
|
+
driver: 'postgres',
|
|
1666
|
+
resources: {
|
|
1667
|
+
users: [{ actions: ['insert', 'update', 'delete'], table: 'users_table' }]
|
|
1668
|
+
},
|
|
1669
|
+
config: {
|
|
1670
|
+
connectionString: 'postgresql://user:pass@localhost:5432/analytics'
|
|
1671
|
+
}
|
|
1672
|
+
}
|
|
1673
|
+
```
|
|
1674
|
+
|
|
1675
|
+
### Resource Configuration Formats
|
|
1676
|
+
|
|
1677
|
+
Multiple formats supported for resource mapping:
|
|
1678
|
+
|
|
1679
|
+
```javascript
|
|
1680
|
+
// 1. Simple array (replicate to same name)
|
|
1681
|
+
resources: ['users', 'products']
|
|
1682
|
+
|
|
1683
|
+
// 2. Object mapping (source → destination)
|
|
1684
|
+
resources: { users: 'people', products: 'items' }
|
|
1685
|
+
|
|
1686
|
+
// 3. Advanced mapping with transformers
|
|
1687
|
+
resources: {
|
|
1688
|
+
users: [
|
|
1689
|
+
{
|
|
1690
|
+
resource: 'people',
|
|
1691
|
+
transformer: (data) => ({ ...data, fullName: `${data.first} ${data.last}` })
|
|
1692
|
+
}
|
|
1693
|
+
]
|
|
1694
|
+
}
|
|
1695
|
+
|
|
1696
|
+
// 4. Action-specific configuration (BigQuery/PostgreSQL)
|
|
1697
|
+
resources: {
|
|
1698
|
+
users: [
|
|
1699
|
+
{ actions: ['insert', 'update'], table: 'users_table' },
|
|
1700
|
+
{ actions: ['insert'], table: 'users_analytics' }
|
|
1701
|
+
]
|
|
1702
|
+
}
|
|
1703
|
+
```
|
|
1704
|
+
|
|
1705
|
+
### 🔧 Easy Example
|
|
1706
|
+
|
|
1707
|
+
```javascript
|
|
1708
|
+
import { S3db, ReplicatorPlugin } from 's3db.js';
|
|
1709
|
+
|
|
1710
|
+
const s3db = new S3db({
|
|
1711
|
+
connectionString: "s3://ACCESS_KEY:SECRET_KEY@BUCKET_NAME/databases/myapp",
|
|
1712
|
+
plugins: [new ReplicatorPlugin({
|
|
1713
|
+
persistReplicatorLog: true,
|
|
1714
|
+
replicators: [
|
|
1715
|
+
{
|
|
1716
|
+
driver: 's3db',
|
|
1717
|
+
resources: ['users', 'products'],
|
|
1718
|
+
config: {
|
|
1719
|
+
connectionString: "s3://BACKUP_KEY:BACKUP_SECRET@BACKUP_BUCKET/backup"
|
|
1720
|
+
}
|
|
1721
|
+
},
|
|
1722
|
+
{
|
|
1723
|
+
driver: 'sqs',
|
|
1724
|
+
resources: ['orders'],
|
|
1725
|
+
config: {
|
|
1726
|
+
queueUrl: 'https://sqs.us-east-1.amazonaws.com/123456789012/orders-queue.fifo',
|
|
1727
|
+
region: 'us-east-1',
|
|
1728
|
+
messageGroupId: 'order-updates'
|
|
1729
|
+
}
|
|
1730
|
+
}
|
|
1731
|
+
]
|
|
1732
|
+
})]
|
|
1733
|
+
});
|
|
1734
|
+
|
|
1735
|
+
await s3db.connect();
|
|
1736
|
+
|
|
1737
|
+
const users = s3db.resource('users');
|
|
1738
|
+
const orders = s3db.resource('orders');
|
|
1739
|
+
|
|
1740
|
+
// Monitor replication events
|
|
1741
|
+
const replicatorPlugin = s3db.plugins.find(p => p.constructor.name === 'ReplicatorPlugin');
|
|
1742
|
+
|
|
1743
|
+
replicatorPlugin.on('replicator.success', (data) => {
|
|
1744
|
+
console.log(`✅ Replicated: ${data.action} on ${data.resource} to ${data.replicator}`);
|
|
1745
|
+
});
|
|
1746
|
+
|
|
1747
|
+
replicatorPlugin.on('replicator.failed', (data) => {
|
|
1748
|
+
console.error(`❌ Replication failed: ${data.error}`);
|
|
1749
|
+
});
|
|
1750
|
+
|
|
1751
|
+
// Insert data (automatically replicated)
|
|
1752
|
+
const user = await users.insert({
|
|
1753
|
+
name: 'Alice Johnson',
|
|
1754
|
+
email: 'alice@example.com',
|
|
1755
|
+
role: 'customer'
|
|
1756
|
+
});
|
|
1757
|
+
|
|
1758
|
+
const order = await orders.insert({
|
|
1759
|
+
userId: user.id,
|
|
1760
|
+
amount: 99.99,
|
|
1761
|
+
items: ['item1', 'item2']
|
|
1762
|
+
});
|
|
1763
|
+
|
|
1764
|
+
// Check replication logs
|
|
1765
|
+
const replicatorLogs = s3db.resource('replicator_logs');
|
|
1766
|
+
const logs = await replicatorLogs.list();
|
|
1767
|
+
|
|
1768
|
+
console.log('\n=== Replication History ===');
|
|
1769
|
+
logs.forEach(log => {
|
|
1770
|
+
console.log(`${log.timestamp}: ${log.action} ${log.resource} → ${log.replicator}`);
|
|
1771
|
+
});
|
|
1772
|
+
```
|
|
1773
|
+
|
|
1774
|
+
### 🚀 Advanced Multi-Driver Example
|
|
1775
|
+
|
|
1776
|
+
```javascript
|
|
1777
|
+
import { S3db, ReplicatorPlugin } from 's3db.js';
|
|
1778
|
+
|
|
1779
|
+
const s3db = new S3db({
|
|
1780
|
+
connectionString: "s3://ACCESS_KEY:SECRET_KEY@BUCKET_NAME/databases/myapp",
|
|
1781
|
+
plugins: [new ReplicatorPlugin({
|
|
1782
|
+
enabled: true,
|
|
1783
|
+
persistReplicatorLog: true,
|
|
1784
|
+
replicatorLogResource: 'replication_audit',
|
|
1785
|
+
batchSize: 25,
|
|
1786
|
+
retryAttempts: 5,
|
|
1787
|
+
retryDelay: 2000,
|
|
1788
|
+
|
|
1789
|
+
replicators: [
|
|
1790
|
+
// Backup to another S3DB instance
|
|
1791
|
+
{
|
|
1792
|
+
driver: 's3db',
|
|
1793
|
+
resources: ['users', 'products', 'orders'],
|
|
1794
|
+
config: {
|
|
1795
|
+
connectionString: "s3://BACKUP_KEY:BACKUP_SECRET@BACKUP_BUCKET/backup",
|
|
1796
|
+
enabled: true,
|
|
1797
|
+
timeout: 30000
|
|
1798
|
+
}
|
|
1799
|
+
},
|
|
1800
|
+
|
|
1801
|
+
// Real-time events to SQS
|
|
1802
|
+
{
|
|
1803
|
+
driver: 'sqs',
|
|
1804
|
+
resources: ['orders', 'users'],
|
|
1805
|
+
config: {
|
|
1806
|
+
region: 'us-east-1',
|
|
1807
|
+
credentials: {
|
|
1808
|
+
accessKeyId: process.env.AWS_ACCESS_KEY_ID,
|
|
1809
|
+
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY
|
|
1810
|
+
},
|
|
1811
|
+
// Resource-specific queues
|
|
1812
|
+
queues: {
|
|
1813
|
+
orders: 'https://sqs.us-east-1.amazonaws.com/123456789012/order-events.fifo',
|
|
1814
|
+
users: 'https://sqs.us-east-1.amazonaws.com/123456789012/user-events.fifo'
|
|
1815
|
+
},
|
|
1816
|
+
// Default queue for unspecified resources
|
|
1817
|
+
defaultQueueUrl: 'https://sqs.us-east-1.amazonaws.com/123456789012/default-events.fifo',
|
|
1818
|
+
messageGroupId: 's3db-replicator',
|
|
1819
|
+
deduplicationId: true,
|
|
1820
|
+
messageAttributes: {
|
|
1821
|
+
source: { StringValue: 'production-db', DataType: 'String' },
|
|
1822
|
+
version: { StringValue: '1.0', DataType: 'String' }
|
|
1823
|
+
}
|
|
1824
|
+
}
|
|
1825
|
+
},
|
|
1826
|
+
|
|
1827
|
+
// Analytics to BigQuery
|
|
1828
|
+
{
|
|
1829
|
+
driver: 'bigquery',
|
|
1830
|
+
config: {
|
|
1831
|
+
projectId: 'my-analytics-project',
|
|
1832
|
+
datasetId: 's3db_analytics',
|
|
1833
|
+
location: 'US',
|
|
1834
|
+
logTable: 'replication_log',
|
|
1835
|
+
credentials: {
|
|
1836
|
+
client_email: 'service-account@project.iam.gserviceaccount.com',
|
|
1837
|
+
private_key: process.env.BIGQUERY_PRIVATE_KEY,
|
|
1838
|
+
project_id: 'my-analytics-project'
|
|
1839
|
+
}
|
|
1840
|
+
},
|
|
1841
|
+
resources: {
|
|
1842
|
+
// Multiple destinations for users
|
|
1843
|
+
users: [
|
|
1844
|
+
{ actions: ['insert', 'update'], table: 'dim_users' },
|
|
1845
|
+
{ actions: ['insert'], table: 'fact_user_activity' }
|
|
1846
|
+
],
|
|
1847
|
+
|
|
1848
|
+
// Orders to analytics tables
|
|
1849
|
+
orders: [
|
|
1850
|
+
{ actions: ['insert'], table: 'fact_orders' },
|
|
1851
|
+
{ actions: ['insert'], table: 'daily_revenue',
|
|
1852
|
+
transformer: (data) => ({
|
|
1853
|
+
date: data.createdAt?.split('T')[0],
|
|
1854
|
+
revenue: data.amount,
|
|
1855
|
+
customer_id: data.userId,
|
|
1856
|
+
order_count: 1
|
|
1857
|
+
})
|
|
1858
|
+
}
|
|
1859
|
+
],
|
|
1860
|
+
|
|
1861
|
+
// Products with transformation
|
|
1862
|
+
products: {
|
|
1863
|
+
table: 'dim_products',
|
|
1864
|
+
actions: ['insert', 'update'],
|
|
1865
|
+
transformer: (data) => ({
|
|
1866
|
+
...data,
|
|
1867
|
+
price_category: data.price > 100 ? 'premium' : 'standard',
|
|
1868
|
+
last_updated: new Date().toISOString()
|
|
1869
|
+
})
|
|
1870
|
+
}
|
|
1871
|
+
}
|
|
1872
|
+
},
|
|
1873
|
+
|
|
1874
|
+
// Operational database (PostgreSQL)
|
|
1875
|
+
{
|
|
1876
|
+
driver: 'postgres',
|
|
1877
|
+
config: {
|
|
1878
|
+
connectionString: 'postgresql://analytics:password@localhost:5432/operations',
|
|
1879
|
+
ssl: { rejectUnauthorized: false },
|
|
1880
|
+
logTable: 'replication_log',
|
|
1881
|
+
pool: {
|
|
1882
|
+
max: 20,
|
|
1883
|
+
idleTimeoutMillis: 30000,
|
|
1884
|
+
connectionTimeoutMillis: 2000
|
|
1885
|
+
}
|
|
1886
|
+
},
|
|
1887
|
+
resources: {
|
|
1888
|
+
users: [
|
|
1889
|
+
{
|
|
1890
|
+
actions: ['insert', 'update', 'delete'],
|
|
1891
|
+
table: 'operational_users',
|
|
1892
|
+
transformer: (data, action) => {
|
|
1893
|
+
if (action === 'delete') return { id: data.id, deleted_at: new Date() };
|
|
1894
|
+
return {
|
|
1895
|
+
...data,
|
|
1896
|
+
sync_timestamp: new Date(),
|
|
1897
|
+
source_system: 's3db'
|
|
1898
|
+
};
|
|
1899
|
+
}
|
|
1900
|
+
}
|
|
1901
|
+
],
|
|
1902
|
+
|
|
1903
|
+
orders: [
|
|
1904
|
+
{ actions: ['insert'], table: 'order_events' },
|
|
1905
|
+
{
|
|
1906
|
+
actions: ['update'],
|
|
1907
|
+
table: 'order_updates',
|
|
1908
|
+
transformer: (data) => ({
|
|
1909
|
+
order_id: data.id,
|
|
1910
|
+
updated_fields: Object.keys(data),
|
|
1911
|
+
update_timestamp: new Date()
|
|
1912
|
+
})
|
|
1913
|
+
}
|
|
1914
|
+
]
|
|
1915
|
+
}
|
|
1916
|
+
}
|
|
1917
|
+
]
|
|
1918
|
+
})]
|
|
1919
|
+
});
|
|
1920
|
+
|
|
1921
|
+
await s3db.connect();
|
|
1922
|
+
|
|
1923
|
+
// Advanced replicator management
|
|
1924
|
+
class ReplicatorManager {
|
|
1925
|
+
constructor(replicatorPlugin) {
|
|
1926
|
+
this.plugin = replicatorPlugin;
|
|
1927
|
+
this.stats = {
|
|
1928
|
+
totalReplications: 0,
|
|
1929
|
+
successfulReplications: 0,
|
|
1930
|
+
failedReplications: 0,
|
|
1931
|
+
byReplicator: {},
|
|
1932
|
+
byResource: {}
|
|
1933
|
+
};
|
|
1934
|
+
|
|
1935
|
+
this.setupEventListeners();
|
|
1936
|
+
}
|
|
1937
|
+
|
|
1938
|
+
setupEventListeners() {
|
|
1939
|
+
this.plugin.on('replicator.queued', (data) => {
|
|
1940
|
+
this.stats.totalReplications++;
|
|
1941
|
+
this.updateResourceStats(data.resource, 'queued');
|
|
1942
|
+
});
|
|
1943
|
+
|
|
1944
|
+
this.plugin.on('replicator.success', (data) => {
|
|
1945
|
+
this.stats.successfulReplications++;
|
|
1946
|
+
this.updateReplicatorStats(data.replicator, 'success');
|
|
1947
|
+
this.updateResourceStats(data.resource, 'success');
|
|
1948
|
+
});
|
|
1949
|
+
|
|
1950
|
+
this.plugin.on('replicator.failed', (data) => {
|
|
1951
|
+
this.stats.failedReplications++;
|
|
1952
|
+
this.updateReplicatorStats(data.replicator, 'failed');
|
|
1953
|
+
this.updateResourceStats(data.resource, 'failed');
|
|
1954
|
+
|
|
1955
|
+
// Advanced error handling
|
|
1956
|
+
if (data.error.includes('BigQuery')) {
|
|
1957
|
+
console.log('🔧 BigQuery error detected - checking schema compatibility...');
|
|
1958
|
+
} else if (data.error.includes('SQS')) {
|
|
1959
|
+
console.log('📮 SQS error detected - checking queue permissions...');
|
|
1960
|
+
}
|
|
1961
|
+
});
|
|
1962
|
+
}
|
|
1963
|
+
|
|
1964
|
+
updateReplicatorStats(replicator, status) {
|
|
1965
|
+
if (!this.stats.byReplicator[replicator]) {
|
|
1966
|
+
this.stats.byReplicator[replicator] = { success: 0, failed: 0 };
|
|
1967
|
+
}
|
|
1968
|
+
this.stats.byReplicator[replicator][status]++;
|
|
1969
|
+
}
|
|
1970
|
+
|
|
1971
|
+
updateResourceStats(resource, status) {
|
|
1972
|
+
if (!this.stats.byResource[resource]) {
|
|
1973
|
+
this.stats.byResource[resource] = { queued: 0, success: 0, failed: 0 };
|
|
1974
|
+
}
|
|
1975
|
+
this.stats.byResource[resource][status]++;
|
|
1976
|
+
}
|
|
1977
|
+
|
|
1978
|
+
async getReplicationHealth() {
|
|
1979
|
+
const totalAttempts = this.stats.successfulReplications + this.stats.failedReplications;
|
|
1980
|
+
const successRate = totalAttempts > 0 ? this.stats.successfulReplications / totalAttempts : 1;
|
|
1981
|
+
|
|
1982
|
+
return {
|
|
1983
|
+
overall: {
|
|
1984
|
+
successRate: successRate,
|
|
1985
|
+
totalReplications: this.stats.totalReplications,
|
|
1986
|
+
pending: this.stats.totalReplications - totalAttempts,
|
|
1987
|
+
health: successRate > 0.95 ? 'excellent' :
|
|
1988
|
+
successRate > 0.85 ? 'good' :
|
|
1989
|
+
successRate > 0.7 ? 'warning' : 'critical'
|
|
1990
|
+
},
|
|
1991
|
+
byReplicator: this.stats.byReplicator,
|
|
1992
|
+
byResource: this.stats.byResource
|
|
1993
|
+
};
|
|
1994
|
+
}
|
|
1995
|
+
|
|
1996
|
+
async pauseReplicator(replicatorId) {
|
|
1997
|
+
const replicator = this.plugin.replicators.find(r => r.id === replicatorId);
|
|
1998
|
+
if (replicator) {
|
|
1999
|
+
replicator.enabled = false;
|
|
2000
|
+
console.log(`⏸️ Paused replicator: ${replicatorId}`);
|
|
2001
|
+
}
|
|
2002
|
+
}
|
|
2003
|
+
|
|
2004
|
+
async resumeReplicator(replicatorId) {
|
|
2005
|
+
const replicator = this.plugin.replicators.find(r => r.id === replicatorId);
|
|
2006
|
+
if (replicator) {
|
|
2007
|
+
replicator.enabled = true;
|
|
2008
|
+
console.log(`▶️ Resumed replicator: ${replicatorId}`);
|
|
2009
|
+
}
|
|
2010
|
+
}
|
|
2011
|
+
|
|
2012
|
+
async testReplicatorConnections() {
|
|
2013
|
+
console.log('🔍 Testing replicator connections...');
|
|
2014
|
+
|
|
2015
|
+
for (const replicator of this.plugin.replicators) {
|
|
2016
|
+
try {
|
|
2017
|
+
const result = await replicator.testConnection();
|
|
2018
|
+
console.log(`✅ ${replicator.driver}: ${result.status}`);
|
|
2019
|
+
} catch (error) {
|
|
2020
|
+
console.log(`❌ ${replicator.driver}: ${error.message}`);
|
|
2021
|
+
}
|
|
2022
|
+
}
|
|
2023
|
+
}
|
|
2024
|
+
}
|
|
2025
|
+
|
|
2026
|
+
// Setup sample data and test all replicators
|
|
2027
|
+
const users = s3db.resource('users');
|
|
2028
|
+
const products = s3db.resource('products');
|
|
2029
|
+
const orders = s3db.resource('orders');
|
|
2030
|
+
|
|
2031
|
+
const replicatorPlugin = s3db.plugins.find(p => p.constructor.name === 'ReplicatorPlugin');
|
|
2032
|
+
const manager = new ReplicatorManager(replicatorPlugin);
|
|
2033
|
+
|
|
2034
|
+
// Test connections
|
|
2035
|
+
await manager.testReplicatorConnections();
|
|
2036
|
+
|
|
2037
|
+
// Create sample data
|
|
2038
|
+
console.log('🔄 Creating sample data with multi-driver replication...');
|
|
2039
|
+
|
|
2040
|
+
const sampleUsers = await users.insertMany([
|
|
2041
|
+
{ name: 'John Smith', email: 'john@example.com', role: 'admin' },
|
|
2042
|
+
{ name: 'Jane Doe', email: 'jane@example.com', role: 'user' },
|
|
2043
|
+
{ name: 'Bob Wilson', email: 'bob@example.com', role: 'user' }
|
|
2044
|
+
]);
|
|
2045
|
+
|
|
2046
|
+
const sampleProducts = await products.insertMany([
|
|
2047
|
+
{ name: 'Laptop Pro', price: 1299.99, category: 'electronics' },
|
|
2048
|
+
{ name: 'Wireless Mouse', price: 29.99, category: 'electronics' },
|
|
2049
|
+
{ name: 'Coffee Mug', price: 12.99, category: 'home' }
|
|
2050
|
+
]);
|
|
2051
|
+
|
|
2052
|
+
const sampleOrders = await orders.insertMany([
|
|
2053
|
+
{ userId: sampleUsers[0].id, amount: 1329.98, items: [sampleProducts[0].id, sampleProducts[1].id] },
|
|
2054
|
+
{ userId: sampleUsers[1].id, amount: 29.99, items: [sampleProducts[1].id] },
|
|
2055
|
+
{ userId: sampleUsers[2].id, amount: 12.99, items: [sampleProducts[2].id] }
|
|
2056
|
+
]);
|
|
2057
|
+
|
|
2058
|
+
// Wait for replications to complete
|
|
2059
|
+
await new Promise(resolve => setTimeout(resolve, 3000));
|
|
2060
|
+
|
|
2061
|
+
// Get replication statistics
|
|
2062
|
+
const health = await manager.getReplicationHealth();
|
|
2063
|
+
console.log('\n=== Replication Health Report ===');
|
|
2064
|
+
console.log(`Overall success rate: ${(health.overall.successRate * 100).toFixed(1)}%`);
|
|
2065
|
+
console.log(`Health status: ${health.overall.health.toUpperCase()}`);
|
|
2066
|
+
console.log(`Total replications: ${health.overall.totalReplications}`);
|
|
2067
|
+
console.log(`Pending: ${health.overall.pending}`);
|
|
2068
|
+
|
|
2069
|
+
console.log('\n=== By Replicator ===');
|
|
2070
|
+
Object.entries(health.byReplicator).forEach(([replicator, stats]) => {
|
|
2071
|
+
const total = stats.success + stats.failed;
|
|
2072
|
+
const rate = total > 0 ? (stats.success / total * 100).toFixed(1) : 0;
|
|
2073
|
+
console.log(`${replicator}: ${rate}% success (${stats.success}/${total})`);
|
|
2074
|
+
});
|
|
2075
|
+
|
|
2076
|
+
console.log('\n=== By Resource ===');
|
|
2077
|
+
Object.entries(health.byResource).forEach(([resource, stats]) => {
|
|
2078
|
+
console.log(`${resource}: queued ${stats.queued}, success ${stats.success}, failed ${stats.failed}`);
|
|
2079
|
+
});
|
|
2080
|
+
|
|
2081
|
+
// Get detailed replication logs
|
|
2082
|
+
const replicationLogs = await replicatorPlugin.getReplicatorLogs({ limit: 10 });
|
|
2083
|
+
console.log('\n=== Recent Replication Logs ===');
|
|
2084
|
+
replicationLogs.forEach(log => {
|
|
2085
|
+
const status = log.success ? '✅' : '❌';
|
|
2086
|
+
console.log(`${status} ${log.timestamp} | ${log.action} ${log.resource} → ${log.replicator}`);
|
|
2087
|
+
if (!log.success && log.error) {
|
|
2088
|
+
console.log(` Error: ${log.error}`);
|
|
2089
|
+
}
|
|
2090
|
+
});
|
|
2091
|
+
|
|
2092
|
+
console.log('\n✅ Multi-driver replication demonstration completed');
|
|
2093
|
+
```
|
|
2094
|
+
|
|
2095
|
+
---
|
|
2096
|
+
|
|
2097
|
+
## 📬 Queue Consumer Plugin
|
|
2098
|
+
|
|
2099
|
+
Consume messages from external queues (SQS, RabbitMQ) and automatically process them into your s3db resources.
|
|
2100
|
+
|
|
2101
|
+
### ⚡ Quick Start
|
|
2102
|
+
|
|
2103
|
+
```javascript
|
|
2104
|
+
import { S3db, QueueConsumerPlugin } from 's3db.js';
|
|
2105
|
+
|
|
2106
|
+
const s3db = new S3db({
|
|
2107
|
+
connectionString: "s3://ACCESS_KEY:SECRET_KEY@BUCKET_NAME/databases/myapp",
|
|
2108
|
+
plugins: [new QueueConsumerPlugin({
|
|
2109
|
+
consumers: [
|
|
2110
|
+
{
|
|
2111
|
+
driver: 'sqs',
|
|
2112
|
+
config: {
|
|
2113
|
+
queueUrl: 'https://sqs.us-east-1.amazonaws.com/123456789012/my-queue',
|
|
2114
|
+
region: 'us-east-1'
|
|
2115
|
+
},
|
|
2116
|
+
consumers: [
|
|
2117
|
+
{ resources: 'users' }
|
|
2118
|
+
]
|
|
2119
|
+
}
|
|
2120
|
+
]
|
|
2121
|
+
})]
|
|
2122
|
+
});
|
|
2123
|
+
|
|
2124
|
+
await s3db.connect();
|
|
2125
|
+
// Queue messages are automatically processed into your resources
|
|
2126
|
+
```
|
|
2127
|
+
|
|
2128
|
+
### ⚙️ Configuration Parameters
|
|
2129
|
+
|
|
2130
|
+
| Parameter | Type | Default | Description |
|
|
2131
|
+
|-----------|------|---------|-------------|
|
|
2132
|
+
| `enabled` | boolean | `true` | Enable/disable queue consumption |
|
|
2133
|
+
| `consumers` | array | `[]` | Array of consumer configurations |
|
|
2134
|
+
| `batchSize` | number | `10` | Messages to process per batch |
|
|
2135
|
+
| `concurrency` | number | `5` | Concurrent message processing |
|
|
2136
|
+
| `retryAttempts` | number | `3` | Retry failed message processing |
|
|
2137
|
+
| `retryDelay` | number | `1000` | Delay between retries (ms) |
|
|
2138
|
+
| `deadLetterQueue` | string | `null` | DLQ for failed messages |
|
|
2139
|
+
|
|
2140
|
+
### Supported Drivers
|
|
2141
|
+
|
|
2142
|
+
#### SQS Consumer
|
|
2143
|
+
|
|
2144
|
+
Consume from AWS SQS queues:
|
|
2145
|
+
|
|
2146
|
+
```javascript
|
|
2147
|
+
{
|
|
2148
|
+
driver: 'sqs',
|
|
2149
|
+
config: {
|
|
2150
|
+
queueUrl: 'https://sqs.us-east-1.amazonaws.com/123456789012/my-queue',
|
|
2151
|
+
region: 'us-east-1',
|
|
2152
|
+
credentials: { accessKeyId: '...', secretAccessKey: '...' },
|
|
2153
|
+
pollingInterval: 1000,
|
|
2154
|
+
maxMessages: 10,
|
|
2155
|
+
visibilityTimeout: 300
|
|
2156
|
+
},
|
|
2157
|
+
consumers: [
|
|
2158
|
+
{ resources: ['users'], queueUrl: 'specific-queue-url' }
|
|
2159
|
+
]
|
|
2160
|
+
}
|
|
2161
|
+
```
|
|
2162
|
+
|
|
2163
|
+
#### RabbitMQ Consumer
|
|
2164
|
+
|
|
2165
|
+
Consume from RabbitMQ queues:
|
|
2166
|
+
|
|
2167
|
+
```javascript
|
|
2168
|
+
{
|
|
2169
|
+
driver: 'rabbitmq',
|
|
2170
|
+
config: {
|
|
2171
|
+
amqpUrl: 'amqp://user:pass@localhost:5672',
|
|
2172
|
+
exchange: 'my-exchange',
|
|
2173
|
+
prefetch: 10,
|
|
2174
|
+
reconnectInterval: 2000
|
|
2175
|
+
},
|
|
2176
|
+
consumers: [
|
|
2177
|
+
{ resources: ['orders'], queue: 'orders-queue' }
|
|
2178
|
+
]
|
|
2179
|
+
}
|
|
2180
|
+
```
|
|
2181
|
+
|
|
2182
|
+
### Message Format
|
|
2183
|
+
|
|
2184
|
+
Expected message structure:
|
|
2185
|
+
|
|
2186
|
+
```javascript
|
|
2187
|
+
{
|
|
2188
|
+
resource: 'users', // Target resource name
|
|
2189
|
+
action: 'insert', // Operation: insert, update, delete
|
|
2190
|
+
data: { // Data payload
|
|
2191
|
+
name: 'John Doe',
|
|
2192
|
+
email: 'john@example.com'
|
|
2193
|
+
}
|
|
2194
|
+
}
|
|
2195
|
+
```
|
|
2196
|
+
|
|
2197
|
+
### 🔧 Easy Example
|
|
2198
|
+
|
|
2199
|
+
```javascript
|
|
2200
|
+
import { S3db, QueueConsumerPlugin } from 's3db.js';
|
|
2201
|
+
|
|
2202
|
+
const s3db = new S3db({
|
|
2203
|
+
connectionString: "s3://ACCESS_KEY:SECRET_KEY@BUCKET_NAME/databases/myapp",
|
|
2204
|
+
plugins: [new QueueConsumerPlugin({
|
|
2205
|
+
enabled: true,
|
|
2206
|
+
consumers: [
|
|
2207
|
+
{
|
|
2208
|
+
driver: 'sqs',
|
|
2209
|
+
config: {
|
|
2210
|
+
queueUrl: 'https://sqs.us-east-1.amazonaws.com/123456789012/user-updates.fifo',
|
|
2211
|
+
region: 'us-east-1',
|
|
2212
|
+
pollingInterval: 2000,
|
|
2213
|
+
maxMessages: 5
|
|
2214
|
+
},
|
|
2215
|
+
consumers: [
|
|
2216
|
+
{ resources: ['users', 'profiles'] }
|
|
2217
|
+
]
|
|
2218
|
+
}
|
|
2219
|
+
]
|
|
2220
|
+
})]
|
|
2221
|
+
});
|
|
2222
|
+
|
|
2223
|
+
await s3db.connect();
|
|
2224
|
+
|
|
2225
|
+
// Messages are automatically consumed and processed
|
|
2226
|
+
console.log('Queue consumer started - listening for messages...');
|
|
2227
|
+
|
|
2228
|
+
// Simulate sending a message (in real use, external systems send these)
|
|
2229
|
+
const testMessage = {
|
|
2230
|
+
resource: 'users',
|
|
2231
|
+
action: 'insert',
|
|
2232
|
+
data: {
|
|
2233
|
+
name: 'Queue User',
|
|
2234
|
+
email: 'queue@example.com',
|
|
2235
|
+
source: 'external-system'
|
|
2236
|
+
}
|
|
2237
|
+
};
|
|
2238
|
+
|
|
2239
|
+
console.log('Processing message:', testMessage);
|
|
2240
|
+
```
|
|
2241
|
+
|
|
2242
|
+
### 🚀 Advanced Multi-Driver Example
|
|
2243
|
+
|
|
2244
|
+
```javascript
|
|
2245
|
+
import { S3db, QueueConsumerPlugin } from 's3db.js';
|
|
2246
|
+
|
|
2247
|
+
const s3db = new S3db({
|
|
2248
|
+
connectionString: "s3://ACCESS_KEY:SECRET_KEY@BUCKET_NAME/databases/myapp",
|
|
2249
|
+
plugins: [new QueueConsumerPlugin({
|
|
2250
|
+
enabled: true,
|
|
2251
|
+
batchSize: 20,
|
|
2252
|
+
concurrency: 10,
|
|
2253
|
+
retryAttempts: 5,
|
|
2254
|
+
retryDelay: 2000,
|
|
2255
|
+
|
|
2256
|
+
consumers: [
|
|
2257
|
+
// SQS Consumer for user events
|
|
2258
|
+
{
|
|
2259
|
+
driver: 'sqs',
|
|
2260
|
+
config: {
|
|
2261
|
+
region: 'us-east-1',
|
|
2262
|
+
credentials: {
|
|
2263
|
+
accessKeyId: process.env.AWS_ACCESS_KEY_ID,
|
|
2264
|
+
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY
|
|
2265
|
+
},
|
|
2266
|
+
pollingInterval: 1000,
|
|
2267
|
+
maxMessages: 10,
|
|
2268
|
+
visibilityTimeout: 300,
|
|
2269
|
+
waitTimeSeconds: 20 // Long polling
|
|
2270
|
+
},
|
|
2271
|
+
consumers: [
|
|
2272
|
+
{
|
|
2273
|
+
resources: ['users'],
|
|
2274
|
+
queueUrl: 'https://sqs.us-east-1.amazonaws.com/123456789012/user-events.fifo',
|
|
2275
|
+
messageGroupId: 'user-processing'
|
|
2276
|
+
},
|
|
2277
|
+
{
|
|
2278
|
+
resources: ['orders'],
|
|
2279
|
+
queueUrl: 'https://sqs.us-east-1.amazonaws.com/123456789012/order-events.fifo',
|
|
2280
|
+
messageGroupId: 'order-processing'
|
|
2281
|
+
}
|
|
2282
|
+
]
|
|
2283
|
+
},
|
|
2284
|
+
|
|
2285
|
+
// RabbitMQ Consumer for analytics events
|
|
2286
|
+
{
|
|
2287
|
+
driver: 'rabbitmq',
|
|
2288
|
+
config: {
|
|
2289
|
+
amqpUrl: 'amqp://analytics:password@localhost:5672',
|
|
2290
|
+
exchange: 'analytics-events',
|
|
2291
|
+
exchangeType: 'topic',
|
|
2292
|
+
prefetch: 15,
|
|
2293
|
+
reconnectInterval: 3000,
|
|
2294
|
+
heartbeat: 60
|
|
2295
|
+
},
|
|
2296
|
+
consumers: [
|
|
2297
|
+
{
|
|
2298
|
+
resources: ['analytics', 'metrics'],
|
|
2299
|
+
queue: 'analytics-queue',
|
|
2300
|
+
routingKey: 'analytics.*',
|
|
2301
|
+
durable: true
|
|
2302
|
+
},
|
|
2303
|
+
{
|
|
2304
|
+
resources: ['logs'],
|
|
2305
|
+
queue: 'logs-queue',
|
|
2306
|
+
routingKey: 'logs.*',
|
|
2307
|
+
durable: true
|
|
2308
|
+
}
|
|
2309
|
+
]
|
|
2310
|
+
}
|
|
2311
|
+
]
|
|
2312
|
+
})]
|
|
2313
|
+
});
|
|
2314
|
+
|
|
2315
|
+
await s3db.connect();
|
|
2316
|
+
|
|
2317
|
+
// Advanced message processing with custom handlers
|
|
2318
|
+
class QueueMessageProcessor {
|
|
2319
|
+
constructor(queuePlugin) {
|
|
2320
|
+
this.plugin = queuePlugin;
|
|
2321
|
+
this.stats = {
|
|
2322
|
+
processed: 0,
|
|
2323
|
+
errors: 0,
|
|
2324
|
+
byResource: {},
|
|
2325
|
+
byAction: {}
|
|
2326
|
+
};
|
|
2327
|
+
|
|
2328
|
+
this.setupEventListeners();
|
|
2329
|
+
}
|
|
2330
|
+
|
|
2331
|
+
setupEventListeners() {
|
|
2332
|
+
// Listen for message processing events
|
|
2333
|
+
this.plugin.on('message.received', (data) => {
|
|
2334
|
+
console.log(`📨 Received message: ${data.action} on ${data.resource}`);
|
|
2335
|
+
});
|
|
2336
|
+
|
|
2337
|
+
this.plugin.on('message.processed', (data) => {
|
|
2338
|
+
this.stats.processed++;
|
|
2339
|
+
this.updateStats(data.resource, data.action, 'success');
|
|
2340
|
+
console.log(`✅ Processed: ${data.action} on ${data.resource}`);
|
|
2341
|
+
});
|
|
2342
|
+
|
|
2343
|
+
this.plugin.on('message.failed', (data) => {
|
|
2344
|
+
this.stats.errors++;
|
|
2345
|
+
this.updateStats(data.resource, data.action, 'error');
|
|
2346
|
+
console.error(`❌ Failed: ${data.error}`);
|
|
2347
|
+
|
|
2348
|
+
// Custom error handling
|
|
2349
|
+
this.handleProcessingError(data);
|
|
2350
|
+
});
|
|
2351
|
+
}
|
|
2352
|
+
|
|
2353
|
+
updateStats(resource, action, status) {
|
|
2354
|
+
if (!this.stats.byResource[resource]) {
|
|
2355
|
+
this.stats.byResource[resource] = { success: 0, error: 0 };
|
|
2356
|
+
}
|
|
2357
|
+
if (!this.stats.byAction[action]) {
|
|
2358
|
+
this.stats.byAction[action] = { success: 0, error: 0 };
|
|
2359
|
+
}
|
|
2360
|
+
|
|
2361
|
+
this.stats.byResource[resource][status]++;
|
|
2362
|
+
this.stats.byAction[action][status]++;
|
|
2363
|
+
}
|
|
2364
|
+
|
|
2365
|
+
handleProcessingError(errorData) {
|
|
2366
|
+
const { resource, action, error, attempts } = errorData;
|
|
2367
|
+
|
|
2368
|
+
// Log to external monitoring system
|
|
2369
|
+
console.log(`🚨 Error processing ${action} on ${resource}: ${error}`);
|
|
2370
|
+
|
|
2371
|
+
// Custom retry logic
|
|
2372
|
+
if (attempts >= 3) {
|
|
2373
|
+
console.log(`💀 Moving to dead letter queue after ${attempts} attempts`);
|
|
2374
|
+
// In real implementation, move to DLQ
|
|
2375
|
+
}
|
|
2376
|
+
|
|
2377
|
+
// Resource-specific error handling
|
|
2378
|
+
if (resource === 'users' && error.includes('validation')) {
|
|
2379
|
+
console.log('👤 User validation error - checking schema compatibility');
|
|
2380
|
+
} else if (resource === 'orders' && error.includes('duplicate')) {
|
|
2381
|
+
console.log('🛒 Duplicate order detected - implementing idempotency check');
|
|
2382
|
+
}
|
|
2383
|
+
}
|
|
2384
|
+
|
|
2385
|
+
getProcessingStats() {
|
|
2386
|
+
const totalMessages = this.stats.processed + this.stats.errors;
|
|
2387
|
+
const successRate = totalMessages > 0 ? this.stats.processed / totalMessages : 1;
|
|
2388
|
+
|
|
2389
|
+
return {
|
|
2390
|
+
summary: {
|
|
2391
|
+
totalProcessed: this.stats.processed,
|
|
2392
|
+
totalErrors: this.stats.errors,
|
|
2393
|
+
successRate: successRate,
|
|
2394
|
+
health: successRate > 0.95 ? 'excellent' :
|
|
2395
|
+
successRate > 0.85 ? 'good' :
|
|
2396
|
+
successRate > 0.7 ? 'warning' : 'critical'
|
|
2397
|
+
},
|
|
2398
|
+
byResource: this.stats.byResource,
|
|
2399
|
+
byAction: this.stats.byAction
|
|
2400
|
+
};
|
|
2401
|
+
}
|
|
2402
|
+
|
|
2403
|
+
async pauseConsumption() {
|
|
2404
|
+
console.log('⏸️ Pausing queue consumption...');
|
|
2405
|
+
await this.plugin.pause();
|
|
2406
|
+
}
|
|
2407
|
+
|
|
2408
|
+
async resumeConsumption() {
|
|
2409
|
+
console.log('▶️ Resuming queue consumption...');
|
|
2410
|
+
await this.plugin.resume();
|
|
2411
|
+
}
|
|
2412
|
+
}
|
|
2413
|
+
|
|
2414
|
+
// Setup message processing
|
|
2415
|
+
const queuePlugin = s3db.plugins.find(p => p.constructor.name === 'QueueConsumerPlugin');
|
|
2416
|
+
const processor = new QueueMessageProcessor(queuePlugin);
|
|
2417
|
+
|
|
2418
|
+
// Simulate processing for demonstration
|
|
2419
|
+
console.log('🔄 Queue consumers started - processing messages...');
|
|
2420
|
+
|
|
2421
|
+
// In real scenario, messages come from external systems
|
|
2422
|
+
// Here we simulate the processing results
|
|
2423
|
+
setTimeout(async () => {
|
|
2424
|
+
const stats = processor.getProcessingStats();
|
|
2425
|
+
|
|
2426
|
+
console.log('\n=== Queue Processing Stats ===');
|
|
2427
|
+
console.log(`Total processed: ${stats.summary.totalProcessed}`);
|
|
2428
|
+
console.log(`Total errors: ${stats.summary.totalErrors}`);
|
|
2429
|
+
console.log(`Success rate: ${(stats.summary.successRate * 100).toFixed(1)}%`);
|
|
2430
|
+
console.log(`Health: ${stats.summary.health.toUpperCase()}`);
|
|
2431
|
+
|
|
2432
|
+
console.log('\n=== By Resource ===');
|
|
2433
|
+
Object.entries(stats.byResource).forEach(([resource, counts]) => {
|
|
2434
|
+
const total = counts.success + counts.error;
|
|
2435
|
+
console.log(`${resource}: ${counts.success}/${total} successful`);
|
|
2436
|
+
});
|
|
2437
|
+
|
|
2438
|
+
console.log('\n=== By Action ===');
|
|
2439
|
+
Object.entries(stats.byAction).forEach(([action, counts]) => {
|
|
2440
|
+
const total = counts.success + counts.error;
|
|
2441
|
+
console.log(`${action}: ${counts.success}/${total} successful`);
|
|
2442
|
+
});
|
|
2443
|
+
|
|
2444
|
+
}, 5000);
|
|
2445
|
+
|
|
2446
|
+
console.log('\n✅ Queue consumer demonstration completed');
|
|
2447
|
+
```
|
|
2448
|
+
|
|
2449
|
+
---
|
|
2450
|
+
|
|
2451
|
+
## 🔧 Plugin Development
|
|
2452
|
+
|
|
2453
|
+
Create custom plugins to extend s3db.js with your specific requirements.
|
|
2454
|
+
|
|
2455
|
+
### Plugin Base Class
|
|
2456
|
+
|
|
2457
|
+
```javascript
|
|
2458
|
+
import { Plugin } from 's3db.js';
|
|
2459
|
+
|
|
2460
|
+
class MyCustomPlugin extends Plugin {
|
|
2461
|
+
constructor(options = {}) {
|
|
2462
|
+
super(options);
|
|
2463
|
+
this.config = {
|
|
2464
|
+
enabled: options.enabled !== false,
|
|
2465
|
+
...options
|
|
2466
|
+
};
|
|
2467
|
+
}
|
|
2468
|
+
|
|
2469
|
+
async onSetup() {
|
|
2470
|
+
// Initialize plugin after database connection
|
|
2471
|
+
console.log('Setting up MyCustomPlugin');
|
|
2472
|
+
}
|
|
2473
|
+
|
|
2474
|
+
async onStart() {
|
|
2475
|
+
// Plugin is ready to operate
|
|
2476
|
+
console.log('MyCustomPlugin started');
|
|
2477
|
+
}
|
|
2478
|
+
|
|
2479
|
+
async onStop() {
|
|
2480
|
+
// Cleanup before shutdown
|
|
2481
|
+
console.log('MyCustomPlugin stopped');
|
|
2482
|
+
}
|
|
2483
|
+
}
|
|
2484
|
+
```
|
|
2485
|
+
|
|
2486
|
+
### Plugin Lifecycle
|
|
2487
|
+
|
|
2488
|
+
1. **Constructor**: Configure plugin options
|
|
2489
|
+
2. **setup()**: Called when database connects
|
|
2490
|
+
3. **onSetup()**: Initialize plugin resources
|
|
2491
|
+
4. **start()**: Called when database is ready
|
|
2492
|
+
5. **onStart()**: Begin plugin operations
|
|
2493
|
+
6. **stop()**: Called during shutdown
|
|
2494
|
+
7. **onStop()**: Cleanup plugin resources
|
|
2495
|
+
|
|
2496
|
+
### Custom Plugin Example
|
|
2497
|
+
|
|
2498
|
+
```javascript
|
|
2499
|
+
class NotificationPlugin extends Plugin {
|
|
2500
|
+
constructor(options = {}) {
|
|
2501
|
+
super(options);
|
|
2502
|
+
this.config = {
|
|
2503
|
+
enabled: options.enabled !== false,
|
|
2504
|
+
webhookUrl: options.webhookUrl,
|
|
2505
|
+
events: options.events || ['insert', 'update', 'delete'],
|
|
2506
|
+
...options
|
|
2507
|
+
};
|
|
2508
|
+
}
|
|
2509
|
+
|
|
2510
|
+
async onSetup() {
|
|
2511
|
+
// Install hooks for all resources
|
|
2512
|
+
for (const resource of Object.values(this.database.resources)) {
|
|
2513
|
+
this.installResourceHooks(resource);
|
|
2514
|
+
}
|
|
2515
|
+
}
|
|
2516
|
+
|
|
2517
|
+
installResourceHooks(resource) {
|
|
2518
|
+
this.config.events.forEach(event => {
|
|
2519
|
+
resource.on(event, async (data) => {
|
|
2520
|
+
await this.sendNotification(event, resource.name, data);
|
|
2521
|
+
});
|
|
2522
|
+
});
|
|
2523
|
+
}
|
|
2524
|
+
|
|
2525
|
+
async sendNotification(event, resourceName, data) {
|
|
2526
|
+
if (!this.config.webhookUrl) return;
|
|
2527
|
+
|
|
2528
|
+
const payload = {
|
|
2529
|
+
event,
|
|
2530
|
+
resource: resourceName,
|
|
2531
|
+
data,
|
|
2532
|
+
timestamp: new Date().toISOString()
|
|
2533
|
+
};
|
|
2534
|
+
|
|
2535
|
+
try {
|
|
2536
|
+
await fetch(this.config.webhookUrl, {
|
|
2537
|
+
method: 'POST',
|
|
2538
|
+
headers: { 'Content-Type': 'application/json' },
|
|
2539
|
+
body: JSON.stringify(payload)
|
|
2540
|
+
});
|
|
2541
|
+
} catch (error) {
|
|
2542
|
+
console.error('Notification failed:', error);
|
|
2543
|
+
}
|
|
2544
|
+
}
|
|
2545
|
+
}
|
|
2546
|
+
```
|
|
2547
|
+
|
|
2548
|
+
---
|
|
2549
|
+
|
|
2550
|
+
## 💡 Plugin Combinations
|
|
2551
|
+
|
|
2552
|
+
Powerful workflows using multiple plugins together.
|
|
2553
|
+
|
|
2554
|
+
### Complete Monitoring Stack
|
|
2555
|
+
|
|
2556
|
+
```javascript
|
|
2557
|
+
import {
|
|
2558
|
+
S3db,
|
|
2559
|
+
CachePlugin,
|
|
2560
|
+
CostsPlugin,
|
|
2561
|
+
AuditPlugin,
|
|
2562
|
+
FullTextPlugin,
|
|
2563
|
+
MetricsPlugin,
|
|
2564
|
+
ReplicatorPlugin
|
|
2565
|
+
} from 's3db.js';
|
|
2566
|
+
|
|
2567
|
+
const s3db = new S3db({
|
|
2568
|
+
connectionString: "s3://ACCESS_KEY:SECRET_KEY@BUCKET_NAME/databases/myapp",
|
|
2569
|
+
plugins: [
|
|
2570
|
+
// Performance optimization
|
|
2571
|
+
new CachePlugin({
|
|
2572
|
+
driverType: 'memory',
|
|
2573
|
+
ttl: 600000
|
|
2574
|
+
}),
|
|
2575
|
+
|
|
2576
|
+
// Cost tracking
|
|
2577
|
+
CostsPlugin,
|
|
2578
|
+
|
|
2579
|
+
// Compliance and security
|
|
2580
|
+
new AuditPlugin({
|
|
2581
|
+
enabled: true,
|
|
2582
|
+
includeData: true,
|
|
2583
|
+
trackOperations: ['insert', 'update', 'delete', 'get']
|
|
2584
|
+
}),
|
|
2585
|
+
|
|
2586
|
+
// Search capabilities
|
|
2587
|
+
new FullTextPlugin({
|
|
2588
|
+
enabled: true,
|
|
2589
|
+
fields: ['name', 'description', 'content', 'tags']
|
|
2590
|
+
}),
|
|
2591
|
+
|
|
2592
|
+
// Performance monitoring
|
|
2593
|
+
new MetricsPlugin({
|
|
2594
|
+
enabled: true,
|
|
2595
|
+
collectPerformance: true,
|
|
2596
|
+
collectErrors: true,
|
|
2597
|
+
flushInterval: 30000
|
|
2598
|
+
}),
|
|
2599
|
+
|
|
2600
|
+
// Data replication
|
|
2601
|
+
new ReplicatorPlugin({
|
|
2602
|
+
replicators: [
|
|
2603
|
+
{
|
|
2604
|
+
driver: 's3db',
|
|
2605
|
+
resources: ['users', 'products', 'orders'],
|
|
2606
|
+
config: {
|
|
2607
|
+
connectionString: "s3://BACKUP_KEY:BACKUP_SECRET@BACKUP_BUCKET/backup"
|
|
2608
|
+
}
|
|
2609
|
+
}
|
|
2610
|
+
]
|
|
2611
|
+
})
|
|
2612
|
+
]
|
|
2613
|
+
});
|
|
2614
|
+
|
|
2615
|
+
await s3db.connect();
|
|
2616
|
+
|
|
2617
|
+
// All plugins work seamlessly together
|
|
2618
|
+
const products = s3db.resource('products');
|
|
2619
|
+
|
|
2620
|
+
// This single operation triggers:
|
|
2621
|
+
// - Audit logging
|
|
2622
|
+
// - Cost tracking
|
|
2623
|
+
// - Performance metrics
|
|
2624
|
+
// - Cache invalidation
|
|
2625
|
+
// - Data replication
|
|
2626
|
+
// - Search indexing
|
|
2627
|
+
await products.insert({
|
|
2628
|
+
name: 'New Product',
|
|
2629
|
+
description: 'Amazing new product with great features',
|
|
2630
|
+
price: 99.99,
|
|
2631
|
+
tags: ['new', 'featured', 'electronics']
|
|
2632
|
+
});
|
|
2633
|
+
```
|
|
2634
|
+
|
|
2635
|
+
### E-commerce Analytics Pipeline
|
|
2636
|
+
|
|
2637
|
+
```javascript
|
|
2638
|
+
const s3db = new S3db({
|
|
2639
|
+
connectionString: "s3://ACCESS_KEY:SECRET_KEY@BUCKET_NAME/databases/ecommerce",
|
|
2640
|
+
plugins: [
|
|
2641
|
+
// Real-time search
|
|
2642
|
+
new FullTextPlugin({
|
|
2643
|
+
fields: ['name', 'description', 'brand', 'category'],
|
|
2644
|
+
language: 'en-US',
|
|
2645
|
+
stemming: true
|
|
2646
|
+
}),
|
|
2647
|
+
|
|
2648
|
+
// Performance monitoring
|
|
2649
|
+
new MetricsPlugin({
|
|
2650
|
+
collectPerformance: true,
|
|
2651
|
+
slowQueryThreshold: 500
|
|
2652
|
+
}),
|
|
2653
|
+
|
|
2654
|
+
// Multi-destination replication
|
|
2655
|
+
new ReplicatorPlugin({
|
|
2656
|
+
replicators: [
|
|
2657
|
+
// Backup
|
|
2658
|
+
{ driver: 's3db', resources: '*', config: { connectionString: 'backup-db' } },
|
|
2659
|
+
|
|
2660
|
+
// Analytics warehouse
|
|
2661
|
+
{
|
|
2662
|
+
driver: 'bigquery',
|
|
2663
|
+
resources: {
|
|
2664
|
+
orders: 'fact_orders',
|
|
2665
|
+
products: 'dim_products',
|
|
2666
|
+
users: 'dim_customers'
|
|
2667
|
+
},
|
|
2668
|
+
config: { projectId: 'analytics', datasetId: 'ecommerce' }
|
|
2669
|
+
},
|
|
2670
|
+
|
|
2671
|
+
// Real-time events
|
|
2672
|
+
{
|
|
2673
|
+
driver: 'sqs',
|
|
2674
|
+
resources: ['orders', 'cart_events'],
|
|
2675
|
+
config: { queueUrl: 'order-events-queue' }
|
|
2676
|
+
}
|
|
2677
|
+
]
|
|
2678
|
+
}),
|
|
2679
|
+
|
|
2680
|
+
// Comprehensive auditing
|
|
2681
|
+
new AuditPlugin({
|
|
2682
|
+
trackOperations: ['insert', 'update', 'delete'],
|
|
2683
|
+
includeData: true,
|
|
2684
|
+
excludeResources: ['sessions', 'temp_data']
|
|
2685
|
+
})
|
|
2686
|
+
]
|
|
2687
|
+
});
|
|
2688
|
+
```
|
|
2689
|
+
|
|
2690
|
+
---
|
|
2691
|
+
|
|
2692
|
+
## 🎯 Best Practices
|
|
2693
|
+
|
|
2694
|
+
### Plugin Performance
|
|
2695
|
+
|
|
2696
|
+
1. **Enable caching** for read-heavy workloads
|
|
2697
|
+
2. **Monitor costs** in production environments
|
|
2698
|
+
3. **Use appropriate sampling** for metrics collection
|
|
2699
|
+
4. **Configure retention policies** for audit logs
|
|
2700
|
+
5. **Test replicator connections** before deployment
|
|
2701
|
+
|
|
2702
|
+
### Plugin Security
|
|
2703
|
+
|
|
2704
|
+
1. **Exclude sensitive resources** from full-text indexing
|
|
2705
|
+
2. **Limit audit data size** to prevent information leakage
|
|
2706
|
+
3. **Use IAM roles** instead of access keys when possible
|
|
2707
|
+
4. **Encrypt replication data** in transit and at rest
|
|
2708
|
+
5. **Validate message sources** in queue consumers
|
|
2709
|
+
|
|
2710
|
+
### Plugin Monitoring
|
|
2711
|
+
|
|
2712
|
+
1. **Set up alerting** for replication failures
|
|
2713
|
+
2. **Monitor plugin health** with metrics
|
|
2714
|
+
3. **Track error rates** across all plugins
|
|
2715
|
+
4. **Use structured logging** for debugging
|
|
2716
|
+
5. **Implement circuit breakers** for external services
|
|
2717
|
+
|
|
2718
|
+
---
|
|
2719
|
+
|
|
2720
|
+
**🎉 That's a wrap!** You now have comprehensive documentation for all s3db.js plugins. Each plugin is designed to work independently or in combination with others, providing a powerful and flexible foundation for your database needs.
|
|
2721
|
+
|
|
2722
|
+
For more examples and advanced use cases, check out the `/examples` directory in the s3db.js repository.
|
|
2723
|
+
|
|
2724
|
+
**Happy coding with s3db.js! 🚀**
|