s3db.js 12.2.3 → 12.2.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +915 -1669
- package/dist/s3db.cjs.js +284 -10
- package/dist/s3db.cjs.js.map +1 -1
- package/dist/s3db.es.js +284 -10
- package/dist/s3db.es.js.map +1 -1
- package/package.json +1 -1
- package/src/plugins/vector.plugin.js +361 -9
package/README.md
CHANGED
|
@@ -97,22 +97,12 @@
|
|
|
97
97
|
- [✨ Key Features](#-key-features)
|
|
98
98
|
- [🚀 Quick Start](#-quick-start)
|
|
99
99
|
- [💾 Installation](#-installation)
|
|
100
|
-
- [
|
|
101
|
-
- [
|
|
102
|
-
- [
|
|
103
|
-
- [
|
|
104
|
-
- [
|
|
105
|
-
- [
|
|
106
|
-
- [🎛️ Resource Behaviors](#️-resource-behaviors)
|
|
107
|
-
- [🔄 Advanced Streaming API](#-advanced-streaming-api)
|
|
108
|
-
- [📁 Binary Content Management](#-binary-content-management)
|
|
109
|
-
- [🗂️ Advanced Partitioning](#️-advanced-partitioning)
|
|
110
|
-
- [🎣 Advanced Hooks System](#-advanced-hooks-system)
|
|
111
|
-
- [🧩 Resource Middlewares](#-resource-middlewares)
|
|
112
|
-
- [🎧 Event Listeners Configuration](#-event-listeners-configuration)
|
|
113
|
-
- [🤖 MCP Server](#-mcp-server)
|
|
114
|
-
- [🔧 Troubleshooting](#-troubleshooting)
|
|
115
|
-
- [📖 API Reference](#-api-reference)
|
|
100
|
+
- [🗄️ Database](#️-database)
|
|
101
|
+
- [📋 Resources](#-resources)
|
|
102
|
+
- [🔌 Plugins](#-plugins)
|
|
103
|
+
- [🤖 MCP & Integrations](#-mcp--integrations)
|
|
104
|
+
- [🔧 CLI](#-cli)
|
|
105
|
+
- [📖 Documentation](#-documentation)
|
|
116
106
|
|
|
117
107
|
---
|
|
118
108
|
|
|
@@ -234,11 +224,45 @@ const s3db = new S3db({
|
|
|
234
224
|
|
|
235
225
|
---
|
|
236
226
|
|
|
237
|
-
##
|
|
227
|
+
## 💾 Installation
|
|
228
|
+
|
|
229
|
+
### Package Manager
|
|
230
|
+
|
|
231
|
+
```bash
|
|
232
|
+
# npm
|
|
233
|
+
npm install s3db.js
|
|
234
|
+
# pnpm
|
|
235
|
+
pnpm add s3db.js
|
|
236
|
+
# yarn
|
|
237
|
+
yarn add s3db.js
|
|
238
|
+
```
|
|
239
|
+
|
|
240
|
+
### 📦 Optional Dependencies
|
|
241
|
+
|
|
242
|
+
Some features require additional dependencies to be installed manually:
|
|
243
|
+
|
|
244
|
+
#### Replicator Dependencies
|
|
245
|
+
|
|
246
|
+
If you plan to use the replicator system with external services, install the corresponding dependencies:
|
|
247
|
+
|
|
248
|
+
```bash
|
|
249
|
+
# For SQS replicator (AWS SQS queues)
|
|
250
|
+
npm install @aws-sdk/client-sqs
|
|
251
|
+
|
|
252
|
+
# For BigQuery replicator (Google BigQuery)
|
|
253
|
+
npm install @google-cloud/bigquery
|
|
254
|
+
|
|
255
|
+
# For PostgreSQL replicator (PostgreSQL databases)
|
|
256
|
+
npm install pg
|
|
257
|
+
```
|
|
258
|
+
|
|
259
|
+
**Why manual installation?** These are marked as `peerDependencies` to keep the main package lightweight. Only install what you need!
|
|
260
|
+
|
|
261
|
+
### 📘 TypeScript Support
|
|
238
262
|
|
|
239
263
|
s3db.js includes comprehensive TypeScript definitions out of the box. Get full type safety, autocomplete, and IntelliSense support in your IDE!
|
|
240
264
|
|
|
241
|
-
|
|
265
|
+
#### Basic Usage (Automatic Types)
|
|
242
266
|
|
|
243
267
|
```typescript
|
|
244
268
|
import { Database, DatabaseConfig, Resource } from 's3db.js';
|
|
@@ -268,7 +292,7 @@ const users: Resource<any> = db.resources.users;
|
|
|
268
292
|
const user = await users.insert({ name: 'Alice', email: 'alice@example.com', age: 28 });
|
|
269
293
|
```
|
|
270
294
|
|
|
271
|
-
|
|
295
|
+
#### Advanced: Generate Resource Types
|
|
272
296
|
|
|
273
297
|
For even better type safety, auto-generate TypeScript interfaces from your resources:
|
|
274
298
|
|
|
@@ -279,117 +303,55 @@ import { generateTypes } from 's3db.js/typescript-generator';
|
|
|
279
303
|
await generateTypes(db, { outputPath: './types/database.d.ts' });
|
|
280
304
|
```
|
|
281
305
|
|
|
282
|
-
This creates type-safe interfaces:
|
|
283
|
-
|
|
284
|
-
```typescript
|
|
285
|
-
// types/database.d.ts (auto-generated)
|
|
286
|
-
export interface Users {
|
|
287
|
-
id: string;
|
|
288
|
-
name: string;
|
|
289
|
-
email: string;
|
|
290
|
-
age: number;
|
|
291
|
-
createdAt: string;
|
|
292
|
-
updatedAt: string;
|
|
293
|
-
}
|
|
294
|
-
|
|
295
|
-
export interface ResourceMap {
|
|
296
|
-
users: Resource<Users>;
|
|
297
|
-
posts: Resource<Posts>;
|
|
298
|
-
}
|
|
299
|
-
|
|
300
|
-
declare module 's3db.js' {
|
|
301
|
-
interface Database {
|
|
302
|
-
resources: ResourceMap;
|
|
303
|
-
}
|
|
304
|
-
}
|
|
305
|
-
```
|
|
306
|
-
|
|
307
|
-
Now get full autocomplete for your specific fields:
|
|
308
|
-
|
|
309
|
-
```typescript
|
|
310
|
-
import { Users } from './types/database';
|
|
311
|
-
|
|
312
|
-
const user = await db.resources.users.get('id');
|
|
313
|
-
// user is typed as Users - full autocomplete!
|
|
314
|
-
|
|
315
|
-
user.name // ✅ Autocomplete works
|
|
316
|
-
user.email // ✅ Autocomplete works
|
|
317
|
-
user.emai // ❌ Compile error - typo detected!
|
|
318
|
-
```
|
|
319
|
-
|
|
320
|
-
### Benefits
|
|
321
|
-
|
|
322
|
-
- ✅ **Full Type Safety** - Catch errors at compile time
|
|
323
|
-
- ✅ **IDE Autocomplete** - IntelliSense for all methods and properties
|
|
324
|
-
- ✅ **Refactoring Support** - Rename fields safely across your codebase
|
|
325
|
-
- ✅ **Documentation** - Hover tooltips show method signatures
|
|
326
|
-
- ✅ **Type Inference** - TypeScript infers return types automatically
|
|
327
|
-
|
|
328
306
|
See the complete example in [`docs/examples/typescript-usage-example.ts`](docs/examples/typescript-usage-example.ts).
|
|
329
307
|
|
|
330
308
|
---
|
|
331
309
|
|
|
332
|
-
##
|
|
333
|
-
|
|
334
|
-
### Package Manager
|
|
335
|
-
|
|
336
|
-
```bash
|
|
337
|
-
# npm
|
|
338
|
-
npm install s3db.js
|
|
339
|
-
# pnpm
|
|
340
|
-
pnpm add s3db.js
|
|
341
|
-
# yarn
|
|
342
|
-
yarn add s3db.js
|
|
343
|
-
```
|
|
344
|
-
|
|
345
|
-
### 📦 Optional Dependencies
|
|
346
|
-
|
|
347
|
-
Some features require additional dependencies to be installed manually:
|
|
310
|
+
## 🗄️ Database
|
|
348
311
|
|
|
349
|
-
|
|
312
|
+
A Database is a logical container for your resources, stored in a specific S3 bucket path. The database manages resource metadata, connections, and provides the core interface for all operations.
|
|
350
313
|
|
|
351
|
-
|
|
314
|
+
### Configuration Parameters
|
|
352
315
|
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
316
|
+
| Parameter | Type | Default | Description |
|
|
317
|
+
|-----------|------|---------|-------------|
|
|
318
|
+
| `connectionString` | `string` | **required** | S3 connection string (see formats below) |
|
|
319
|
+
| `httpClientOptions` | `object` | optimized | HTTP client configuration for S3 requests |
|
|
320
|
+
| `verbose` | `boolean` | `false` | Enable verbose logging for debugging |
|
|
321
|
+
| `parallelism` | `number` | `10` | Concurrent operations for bulk operations |
|
|
322
|
+
| `versioningEnabled` | `boolean` | `false` | Enable automatic resource versioning |
|
|
323
|
+
| `passphrase` | `string` | `'secret'` | Default passphrase for field encryption |
|
|
324
|
+
| `plugins` | `array` | `[]` | Array of plugin instances to extend functionality |
|
|
356
325
|
|
|
357
|
-
|
|
358
|
-
npm install @google-cloud/bigquery
|
|
326
|
+
### Connection Strings
|
|
359
327
|
|
|
360
|
-
|
|
361
|
-
npm install pg
|
|
362
|
-
```
|
|
328
|
+
s3db.js supports multiple connection string formats for different S3 providers:
|
|
363
329
|
|
|
364
|
-
|
|
330
|
+
```javascript
|
|
331
|
+
// AWS S3 (with credentials)
|
|
332
|
+
"s3://ACCESS_KEY:SECRET_KEY@BUCKET_NAME/databases/myapp?region=us-east-1"
|
|
365
333
|
|
|
366
|
-
|
|
334
|
+
// AWS S3 (IAM role - recommended for production)
|
|
335
|
+
"s3://BUCKET_NAME/databases/myapp?region=us-east-1"
|
|
367
336
|
|
|
368
|
-
|
|
337
|
+
// MinIO (self-hosted)
|
|
338
|
+
"http://minioadmin:minioadmin@localhost:9000/bucket/databases/myapp"
|
|
369
339
|
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
AWS_SECRET_ACCESS_KEY=your_secret_key
|
|
373
|
-
AWS_BUCKET=your_bucket_name
|
|
374
|
-
DATABASE_NAME=myapp
|
|
375
|
-
```
|
|
340
|
+
// Digital Ocean Spaces
|
|
341
|
+
"https://SPACES_KEY:SPACES_SECRET@nyc3.digitaloceanspaces.com/SPACE_NAME/databases/myapp"
|
|
376
342
|
|
|
377
|
-
|
|
343
|
+
// LocalStack (local testing)
|
|
344
|
+
"http://test:test@localhost:4566/mybucket/databases/myapp"
|
|
378
345
|
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
dotenv.config();
|
|
346
|
+
// Backblaze B2
|
|
347
|
+
"https://KEY_ID:APPLICATION_KEY@s3.us-west-002.backblazeb2.com/BUCKET/databases/myapp"
|
|
382
348
|
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
connectionString: `s3://${process.env.AWS_ACCESS_KEY_ID}:${process.env.AWS_SECRET_ACCESS_KEY}@${process.env.AWS_BUCKET}/databases/${process.env.DATABASE_NAME}`
|
|
386
|
-
});
|
|
349
|
+
// Cloudflare R2
|
|
350
|
+
"https://ACCESS_KEY:SECRET_KEY@ACCOUNT_ID.r2.cloudflarestorage.com/BUCKET/databases/myapp"
|
|
387
351
|
```
|
|
388
352
|
|
|
389
|
-
### Authentication Methods
|
|
390
|
-
|
|
391
353
|
<details>
|
|
392
|
-
<summary><strong>🔑
|
|
354
|
+
<summary><strong>🔑 Complete authentication examples</strong></summary>
|
|
393
355
|
|
|
394
356
|
#### 1. Access Keys (Development)
|
|
395
357
|
```javascript
|
|
@@ -412,1868 +374,1152 @@ const s3db = new S3db({
|
|
|
412
374
|
const s3db = new S3db({
|
|
413
375
|
connectionString: "http://minioadmin:minioadmin@localhost:9000/mybucket/databases/myapp"
|
|
414
376
|
});
|
|
415
|
-
|
|
416
|
-
// MinIO on custom server
|
|
417
|
-
const s3db = new S3db({
|
|
418
|
-
connectionString: "http://ACCESS_KEY:SECRET_KEY@minio.example.com:9000/BUCKET_NAME/databases/myapp"
|
|
419
|
-
});
|
|
420
377
|
```
|
|
421
378
|
|
|
422
379
|
#### 4. Digital Ocean Spaces (SaaS)
|
|
423
380
|
```javascript
|
|
424
|
-
// Digital Ocean Spaces (NYC3 datacenter)
|
|
381
|
+
// Digital Ocean Spaces (NYC3 datacenter)
|
|
425
382
|
const s3db = new S3db({
|
|
426
383
|
connectionString: "https://SPACES_KEY:SPACES_SECRET@nyc3.digitaloceanspaces.com/SPACE_NAME/databases/myapp"
|
|
427
384
|
});
|
|
428
|
-
|
|
429
|
-
// Other regions available: sfo3, ams3, sgp1, fra1, syd1
|
|
430
|
-
const s3db = new S3db({
|
|
431
|
-
connectionString: "https://SPACES_KEY:SPACES_SECRET@sgp1.digitaloceanspaces.com/SPACE_NAME/databases/myapp"
|
|
432
|
-
});
|
|
433
385
|
```
|
|
434
386
|
|
|
435
|
-
|
|
387
|
+
</details>
|
|
388
|
+
|
|
389
|
+
### S3 Bucket Structure
|
|
390
|
+
|
|
391
|
+
When you create a database, s3db.js organizes your data in a structured way within your S3 bucket:
|
|
392
|
+
|
|
393
|
+
```
|
|
394
|
+
bucket-name/
|
|
395
|
+
└── databases/
|
|
396
|
+
└── myapp/ # Database root (from connection string)
|
|
397
|
+
├── s3db.json # Database metadata & resource definitions
|
|
398
|
+
│
|
|
399
|
+
├── resource=users/ # Resource: users
|
|
400
|
+
│ ├── data/
|
|
401
|
+
│ │ ├── id=user-123 # Document (metadata in S3 metadata, optional body)
|
|
402
|
+
│ │ └── id=user-456
|
|
403
|
+
│ └── partition=byRegion/ # Partition: byRegion
|
|
404
|
+
│ ├── region=US/
|
|
405
|
+
│ │ ├── id=user-123 # Partition reference
|
|
406
|
+
│ │ └── id=user-789
|
|
407
|
+
│ └── region=EU/
|
|
408
|
+
│ └── id=user-456
|
|
409
|
+
│
|
|
410
|
+
├── resource=posts/ # Resource: posts
|
|
411
|
+
│ └── data/
|
|
412
|
+
│ ├── id=post-abc
|
|
413
|
+
│ └── id=post-def
|
|
414
|
+
│
|
|
415
|
+
├── resource=sessions/ # Resource: sessions (with TTL)
|
|
416
|
+
│ └── data/
|
|
417
|
+
│ ├── id=session-xyz
|
|
418
|
+
│ └── id=session-qwe
|
|
419
|
+
│
|
|
420
|
+
├── plugin=cache/ # Plugin: CachePlugin (global data)
|
|
421
|
+
│ ├── config # Plugin configuration
|
|
422
|
+
│ └── locks/
|
|
423
|
+
│ └── cache-cleanup # Distributed lock
|
|
424
|
+
│
|
|
425
|
+
└── resource=wallets/ # Resource: wallets
|
|
426
|
+
├── data/
|
|
427
|
+
│ └── id=wallet-123
|
|
428
|
+
└── plugin=eventual-consistency/ # Plugin: scoped to resource
|
|
429
|
+
├── balance/
|
|
430
|
+
│ └── transactions/
|
|
431
|
+
│ └── id=txn-123 # Plugin-specific data
|
|
432
|
+
└── locks/
|
|
433
|
+
└── balance-sync # Resource-scoped lock
|
|
434
|
+
```
|
|
435
|
+
|
|
436
|
+
**Key Path Patterns:**
|
|
437
|
+
|
|
438
|
+
| Type | Pattern | Example |
|
|
439
|
+
|------|---------|---------|
|
|
440
|
+
| **Metadata** | `s3db.json` | Database schema, resources, versions |
|
|
441
|
+
| **Document** | `resource={name}/data/id={id}` | `resource=users/data/id=user-123` |
|
|
442
|
+
| **Partition** | `resource={name}/partition={partition}/{field}={value}/id={id}` | `resource=users/partition=byRegion/region=US/id=user-123` |
|
|
443
|
+
| **Plugin (global)** | `plugin={slug}/{path}` | `plugin=cache/config` |
|
|
444
|
+
| **Plugin (resource)** | `resource={name}/plugin={slug}/{path}` | `resource=wallets/plugin=eventual-consistency/balance/transactions/id=txn-123` |
|
|
445
|
+
| **Lock (global)** | `plugin={slug}/locks/{lockName}` | `plugin=ttl/locks/cleanup` |
|
|
446
|
+
| **Lock (resource)** | `resource={name}/plugin={slug}/locks/{lockName}` | `resource=wallets/plugin=eventual-consistency/locks/balance-sync` |
|
|
447
|
+
|
|
448
|
+
**Storage Layers:**
|
|
449
|
+
|
|
450
|
+
1. **Documents** - User data stored in resources
|
|
451
|
+
- Metadata: Stored in S3 object metadata (up to 2KB)
|
|
452
|
+
- Body: Large content stored in S3 object body (unlimited)
|
|
453
|
+
|
|
454
|
+
2. **Partitions** - Organized references for O(1) queries
|
|
455
|
+
- Hierarchical paths with field values
|
|
456
|
+
- References point to main document
|
|
457
|
+
|
|
458
|
+
3. **Plugin Storage** - Plugin-specific data
|
|
459
|
+
- **Global**: `plugin={slug}/...` - Shared config, caches, locks
|
|
460
|
+
- **Resource-scoped**: `resource={name}/plugin={slug}/...` - Per-resource data
|
|
461
|
+
- Supports same behaviors as resources (body-overflow, body-only, etc.)
|
|
462
|
+
- 3-5x faster than creating full resources
|
|
463
|
+
- Examples: EventualConsistency transactions, TTL expiration queues, Cache entries, Audit logs
|
|
464
|
+
|
|
465
|
+
**Why This Structure?**
|
|
466
|
+
|
|
467
|
+
- ✅ **Flat hierarchy** - No deep nesting, better S3 performance
|
|
468
|
+
- ✅ **Self-documenting** - Path tells you what data it contains
|
|
469
|
+
- ✅ **Partition-friendly** - O(1) lookups via S3 prefix queries
|
|
470
|
+
- ✅ **Plugin isolation** - Each plugin has its own namespace
|
|
471
|
+
- ✅ **Consistent naming** - `resource=`, `partition=`, `plugin=`, `id=` prefixes
|
|
472
|
+
|
|
473
|
+
### Creating a Database
|
|
474
|
+
|
|
436
475
|
```javascript
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
476
|
+
import { S3db } from 's3db.js';
|
|
477
|
+
|
|
478
|
+
// Simple connection
|
|
479
|
+
const db = new S3db({
|
|
480
|
+
connectionString: 's3://ACCESS_KEY:SECRET@bucket/databases/myapp'
|
|
440
481
|
});
|
|
441
482
|
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
483
|
+
await db.connect();
|
|
484
|
+
|
|
485
|
+
// With plugins and options
|
|
486
|
+
const db = new S3db({
|
|
487
|
+
connectionString: 's3://bucket/databases/myapp',
|
|
488
|
+
verbose: true,
|
|
489
|
+
parallelism: 20,
|
|
490
|
+
versioningEnabled: true,
|
|
491
|
+
plugins: [
|
|
492
|
+
new CachePlugin({ ttl: 300000 }),
|
|
493
|
+
new MetricsPlugin()
|
|
494
|
+
],
|
|
495
|
+
httpClientOptions: {
|
|
496
|
+
keepAlive: true,
|
|
497
|
+
maxSockets: 100,
|
|
498
|
+
timeout: 60000
|
|
499
|
+
}
|
|
445
500
|
});
|
|
501
|
+
|
|
502
|
+
await db.connect();
|
|
446
503
|
```
|
|
447
504
|
|
|
448
|
-
|
|
505
|
+
### Database Methods
|
|
506
|
+
|
|
507
|
+
| Method | Description |
|
|
508
|
+
|--------|-------------|
|
|
509
|
+
| `connect()` | Initialize database connection and load metadata |
|
|
510
|
+
| `createResource(config)` | Create or update a resource |
|
|
511
|
+
| `getResource(name, options?)` | Get existing resource instance |
|
|
512
|
+
| `resourceExists(name)` | Check if resource exists |
|
|
513
|
+
| `resources.{name}` | Access resource by property |
|
|
514
|
+
| `uploadMetadataFile()` | Save metadata changes to S3 |
|
|
515
|
+
|
|
516
|
+
### HTTP Client Configuration
|
|
517
|
+
|
|
518
|
+
Customize HTTP performance for your workload:
|
|
519
|
+
|
|
449
520
|
```javascript
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
521
|
+
const db = new S3db({
|
|
522
|
+
connectionString: '...',
|
|
523
|
+
httpClientOptions: {
|
|
524
|
+
keepAlive: true, // Enable connection reuse
|
|
525
|
+
keepAliveMsecs: 1000, // Keep connections alive for 1s
|
|
526
|
+
maxSockets: 50, // Max 50 concurrent connections
|
|
527
|
+
maxFreeSockets: 10, // Keep 10 free connections in pool
|
|
528
|
+
timeout: 60000 // 60 second timeout
|
|
529
|
+
}
|
|
453
530
|
});
|
|
531
|
+
```
|
|
454
532
|
|
|
455
|
-
|
|
456
|
-
const s3db = new S3db({
|
|
457
|
-
connectionString: "https://ACCESS_KEY:SECRET_KEY@s3.wasabisys.com/BUCKET_NAME/databases/myapp"
|
|
458
|
-
});
|
|
533
|
+
**Presets:**
|
|
459
534
|
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
connectionString: "https://ACCESS_KEY:SECRET_KEY@ACCOUNT_ID.r2.cloudflarestorage.com/BUCKET_NAME/databases/myapp"
|
|
463
|
-
});
|
|
535
|
+
<details>
|
|
536
|
+
<summary><strong>High Concurrency (APIs)</strong></summary>
|
|
464
537
|
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
538
|
+
```javascript
|
|
539
|
+
httpClientOptions: {
|
|
540
|
+
keepAlive: true,
|
|
541
|
+
keepAliveMsecs: 1000,
|
|
542
|
+
maxSockets: 100, // Higher concurrency
|
|
543
|
+
maxFreeSockets: 20, // More free connections
|
|
544
|
+
timeout: 60000
|
|
545
|
+
}
|
|
469
546
|
```
|
|
547
|
+
</details>
|
|
470
548
|
|
|
549
|
+
<details>
|
|
550
|
+
<summary><strong>Aggressive Performance (High-throughput)</strong></summary>
|
|
551
|
+
|
|
552
|
+
```javascript
|
|
553
|
+
httpClientOptions: {
|
|
554
|
+
keepAlive: true,
|
|
555
|
+
keepAliveMsecs: 5000, // Longer keep-alive
|
|
556
|
+
maxSockets: 200, // High concurrency
|
|
557
|
+
maxFreeSockets: 50, // Large connection pool
|
|
558
|
+
timeout: 120000 // 2 minute timeout
|
|
559
|
+
}
|
|
560
|
+
```
|
|
471
561
|
</details>
|
|
472
562
|
|
|
563
|
+
**Complete documentation**: See above for all Database configuration options
|
|
564
|
+
|
|
473
565
|
---
|
|
474
566
|
|
|
475
|
-
##
|
|
567
|
+
## 📋 Resources
|
|
476
568
|
|
|
477
|
-
|
|
478
|
-
A logical container for your resources, stored in a specific S3 prefix.
|
|
569
|
+
Resources are the core abstraction in s3db.js - they define your data structure, validation rules, and behavior. Think of them as tables in traditional databases, but with much more flexibility and features.
|
|
479
570
|
|
|
480
|
-
|
|
481
|
-
const s3db = new S3db({
|
|
482
|
-
connectionString: "s3://ACCESS_KEY:SECRET_KEY@BUCKET_NAME/databases/myapp"
|
|
483
|
-
});
|
|
484
|
-
```
|
|
571
|
+
### TL;DR
|
|
485
572
|
|
|
486
|
-
|
|
487
|
-
|
|
573
|
+
Resources provide:
|
|
574
|
+
- ✅ **Schema validation** with 30+ field types
|
|
575
|
+
- ✅ **5 behavior strategies** for handling 2KB S3 metadata limit
|
|
576
|
+
- ✅ **Partitioning** for O(1) queries vs O(n) scans
|
|
577
|
+
- ✅ **Hooks & middlewares** for custom logic
|
|
578
|
+
- ✅ **Events** for real-time notifications
|
|
579
|
+
- ✅ **Versioning** for schema evolution
|
|
580
|
+
- ✅ **Encryption** for sensitive fields
|
|
581
|
+
- ✅ **Streaming** for large datasets
|
|
488
582
|
|
|
583
|
+
**Quick example:**
|
|
489
584
|
```javascript
|
|
490
|
-
const users = await
|
|
491
|
-
name:
|
|
585
|
+
const users = await db.createResource({
|
|
586
|
+
name: 'users',
|
|
492
587
|
attributes: {
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
age:
|
|
496
|
-
isActive: "boolean",
|
|
497
|
-
profile: {
|
|
498
|
-
bio: "string|optional",
|
|
499
|
-
avatar: "url|optional"
|
|
500
|
-
},
|
|
501
|
-
tags: "array|items:string|unique",
|
|
502
|
-
password: "secret"
|
|
588
|
+
email: 'email|required|unique',
|
|
589
|
+
password: 'secret|required',
|
|
590
|
+
age: 'number|min:18|max:120'
|
|
503
591
|
},
|
|
592
|
+
behavior: 'enforce-limits',
|
|
504
593
|
timestamps: true,
|
|
505
|
-
behavior: "user-managed",
|
|
506
594
|
partitions: {
|
|
507
|
-
|
|
595
|
+
byAge: { fields: { age: 'number' } }
|
|
508
596
|
}
|
|
509
597
|
});
|
|
510
|
-
```
|
|
511
|
-
|
|
512
|
-
### 🔍 Schema Validation
|
|
513
|
-
Built-in validation using [@icebob/fastest-validator](https://github.com/icebob/fastest-validator) with comprehensive rule support and excellent performance.
|
|
514
598
|
|
|
515
|
-
|
|
599
|
+
await users.insert({ email: 'john@example.com', password: 'secret123', age: 25 });
|
|
600
|
+
```
|
|
516
601
|
|
|
517
|
-
|
|
602
|
+
### Schema & Field Types
|
|
518
603
|
|
|
519
|
-
|
|
604
|
+
Define your data structure with powerful validation:
|
|
520
605
|
|
|
521
|
-
|
|
606
|
+
#### Basic Types
|
|
522
607
|
|
|
523
|
-
|
|
608
|
+
| Type | Example | Validation Rules |
|
|
609
|
+
|------|---------|------------------|
|
|
610
|
+
| `string` | `"name: 'string\|required'"` | `min`, `max`, `length`, `pattern`, `enum` |
|
|
611
|
+
| `number` | `"age: 'number\|min:0'"` | `min`, `max`, `integer`, `positive`, `negative` |
|
|
612
|
+
| `boolean` | `"isActive: 'boolean'"` | `true`, `false` |
|
|
613
|
+
| `email` | `"email: 'email\|required'"` | RFC 5322 validation |
|
|
614
|
+
| `url` | `"website: 'url'"` | Valid URL format |
|
|
615
|
+
| `date` | `"createdAt: 'date'"` | ISO 8601 dates |
|
|
616
|
+
| `array` | `"tags: 'array\|items:string'"` | `items`, `min`, `max`, `unique` |
|
|
617
|
+
| `object` | `"profile: { type: 'object', props: {...} }"` | Nested validation |
|
|
524
618
|
|
|
525
|
-
|
|
526
|
-
|-------------|-------------|---------|
|
|
527
|
-
| **ISO Timestamps** | 67% | `2024-01-15T10:30:00Z` → `ism8LiNFkz90` |
|
|
528
|
-
| **UUIDs** | 33% | `550e8400-e29b-41d4-a716-446655440000` → `uVQ6EAOKbQdShbkRmRUQAAA==` |
|
|
529
|
-
| **Dictionary Values** | 95% | `active` → `da` |
|
|
530
|
-
| **Hex Strings** | 33% | MD5/SHA hashes compressed with base64 |
|
|
531
|
-
| **Large Numbers** | 40-46% | Unix timestamps with base62 encoding |
|
|
532
|
-
| **UTF-8 Memory Cache** | 2-3x faster | Cached byte calculations |
|
|
619
|
+
#### Advanced Types (with encoding)
|
|
533
620
|
|
|
534
|
-
|
|
621
|
+
| Type | Savings | Example |
|
|
622
|
+
|------|---------|---------|
|
|
623
|
+
| `secret` | Encrypted | `"password: 'secret\|required'"` - AES-256-GCM |
|
|
624
|
+
| `embedding:N` | 77% | `"vector: 'embedding:1536'"` - Fixed-point Base62 |
|
|
625
|
+
| `ip4` | 47% | `"ipAddress: 'ip4'"` - Binary Base64 |
|
|
626
|
+
| `ip6` | 44% | `"ipv6: 'ip6'"` - Binary Base64 |
|
|
535
627
|
|
|
536
|
-
|
|
628
|
+
**Encoding optimizations:**
|
|
629
|
+
- ISO timestamps → Unix Base62 (67% savings)
|
|
630
|
+
- UUIDs → Binary Base64 (33% savings)
|
|
631
|
+
- Dictionary values → Single bytes (95% savings)
|
|
537
632
|
|
|
538
|
-
|
|
633
|
+
#### Schema Examples
|
|
539
634
|
|
|
540
635
|
```javascript
|
|
541
|
-
//
|
|
542
|
-
|
|
636
|
+
// Simple schema
|
|
637
|
+
{
|
|
638
|
+
name: 'string|required|min:2|max:100',
|
|
639
|
+
email: 'email|required|unique',
|
|
640
|
+
age: 'number|integer|min:0|max:150'
|
|
641
|
+
}
|
|
543
642
|
|
|
544
|
-
//
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
643
|
+
// Nested objects
|
|
644
|
+
{
|
|
645
|
+
name: 'string|required',
|
|
646
|
+
profile: {
|
|
647
|
+
type: 'object',
|
|
648
|
+
props: {
|
|
649
|
+
bio: 'string|max:500',
|
|
650
|
+
avatar: 'url|optional',
|
|
651
|
+
social: {
|
|
652
|
+
type: 'object',
|
|
653
|
+
props: {
|
|
654
|
+
twitter: 'string|optional',
|
|
655
|
+
github: 'string|optional'
|
|
656
|
+
}
|
|
657
|
+
}
|
|
658
|
+
}
|
|
659
|
+
}
|
|
660
|
+
}
|
|
550
661
|
|
|
551
|
-
//
|
|
552
|
-
|
|
662
|
+
// Arrays with validation
|
|
663
|
+
{
|
|
664
|
+
name: 'string|required',
|
|
665
|
+
tags: 'array|items:string|min:1|max:10|unique',
|
|
666
|
+
scores: 'array|items:number|min:0|max:100'
|
|
667
|
+
}
|
|
553
668
|
|
|
554
|
-
//
|
|
555
|
-
|
|
669
|
+
// Encrypted fields
|
|
670
|
+
{
|
|
671
|
+
email: 'email|required',
|
|
672
|
+
password: 'secret|required',
|
|
673
|
+
apiKey: 'secret|required'
|
|
674
|
+
}
|
|
556
675
|
```
|
|
557
676
|
|
|
558
|
-
|
|
677
|
+
### Behaviors (Handling 2KB Metadata Limit)
|
|
559
678
|
|
|
560
|
-
|
|
679
|
+
S3 metadata has a 2KB limit. Behaviors define how to handle data that exceeds this:
|
|
561
680
|
|
|
562
|
-
|
|
|
563
|
-
|
|
564
|
-
|
|
|
565
|
-
|
|
|
566
|
-
|
|
|
567
|
-
|
|
|
568
|
-
|
|
|
569
|
-
| **Partition Queries** | ~20ms | Organized data access |
|
|
570
|
-
|
|
571
|
-
### 📦 Partitions
|
|
572
|
-
|
|
573
|
-
Organize data efficiently with partitions for faster queries:
|
|
681
|
+
| Behavior | Enforcement | Data Loss | Use Case |
|
|
682
|
+
|----------|-------------|-----------|----------|
|
|
683
|
+
| `user-managed` | None | Possible | Dev/Test - warnings only |
|
|
684
|
+
| `enforce-limits` | Strict | No | Production - throws errors |
|
|
685
|
+
| `truncate-data` | Truncates | Yes | Content management - smart truncation |
|
|
686
|
+
| `body-overflow` | Splits | No | Mixed data - metadata + body |
|
|
687
|
+
| `body-only` | Unlimited | No | Large docs - everything in body |
|
|
574
688
|
|
|
575
689
|
```javascript
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
690
|
+
// Enforce limits (recommended for production)
|
|
691
|
+
const users = await db.createResource({
|
|
692
|
+
name: 'users',
|
|
693
|
+
behavior: 'enforce-limits',
|
|
694
|
+
attributes: { name: 'string', bio: 'string' }
|
|
695
|
+
});
|
|
696
|
+
|
|
697
|
+
// Body overflow for large content
|
|
698
|
+
const blogs = await db.createResource({
|
|
699
|
+
name: 'blogs',
|
|
700
|
+
behavior: 'body-overflow',
|
|
701
|
+
attributes: { title: 'string', content: 'string' }
|
|
587
702
|
});
|
|
588
703
|
|
|
589
|
-
//
|
|
590
|
-
const
|
|
591
|
-
|
|
592
|
-
|
|
704
|
+
// Body-only for documents
|
|
705
|
+
const documents = await db.createResource({
|
|
706
|
+
name: 'documents',
|
|
707
|
+
behavior: 'body-only',
|
|
708
|
+
attributes: { title: 'string', content: 'string', metadata: 'object' }
|
|
593
709
|
});
|
|
594
710
|
```
|
|
595
711
|
|
|
596
|
-
###
|
|
712
|
+
### Resource Methods
|
|
597
713
|
|
|
598
|
-
|
|
714
|
+
#### CRUD Operations
|
|
599
715
|
|
|
600
716
|
```javascript
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
}],
|
|
609
|
-
afterInsert: [async (data) => {
|
|
610
|
-
console.log(`📦 Product ${data.name} created`);
|
|
611
|
-
}]
|
|
612
|
-
}
|
|
613
|
-
});
|
|
614
|
-
```
|
|
717
|
+
// Create
|
|
718
|
+
const user = await users.insert({ name: 'John', email: 'john@example.com' });
|
|
719
|
+
|
|
720
|
+
// Read
|
|
721
|
+
const user = await users.get('user-123');
|
|
722
|
+
const all = await users.list({ limit: 10, offset: 0 });
|
|
723
|
+
const filtered = await users.query({ isActive: true });
|
|
615
724
|
|
|
616
|
-
|
|
725
|
+
// Update (3 methods with different performance)
|
|
726
|
+
await users.update(id, { name: 'Jane' }); // GET+PUT merge (baseline)
|
|
727
|
+
await users.patch(id, { name: 'Jane' }); // HEAD+COPY (40-60% faster*)
|
|
728
|
+
await users.replace(id, fullObject); // PUT only (30-40% faster)
|
|
729
|
+
// *patch() uses HEAD+COPY for metadata-only behaviors
|
|
730
|
+
|
|
731
|
+
// Delete
|
|
732
|
+
await users.delete('user-123');
|
|
733
|
+
```
|
|
617
734
|
|
|
618
|
-
|
|
735
|
+
#### Bulk Operations
|
|
619
736
|
|
|
620
737
|
```javascript
|
|
621
|
-
//
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
// Bulk
|
|
628
|
-
const
|
|
629
|
-
|
|
630
|
-
|
|
738
|
+
// Bulk insert
|
|
739
|
+
await users.insertMany([
|
|
740
|
+
{ name: 'User 1', email: 'user1@example.com' },
|
|
741
|
+
{ name: 'User 2', email: 'user2@example.com' }
|
|
742
|
+
]);
|
|
743
|
+
|
|
744
|
+
// Bulk get
|
|
745
|
+
const data = await users.getMany(['user-1', 'user-2', 'user-3']);
|
|
746
|
+
|
|
747
|
+
// Bulk delete
|
|
748
|
+
await users.deleteMany(['user-1', 'user-2']);
|
|
631
749
|
```
|
|
632
750
|
|
|
633
|
-
###
|
|
751
|
+
### Partitions (O(1) Queries)
|
|
634
752
|
|
|
635
|
-
|
|
753
|
+
Organize data for fast queries without scanning:
|
|
636
754
|
|
|
637
755
|
```javascript
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
connectionString: "s3://ACCESS_KEY:SECRET_KEY@BUCKET_NAME/databases/myapp",
|
|
641
|
-
versioningEnabled: true
|
|
642
|
-
});
|
|
643
|
-
|
|
644
|
-
// Create versioned resource
|
|
645
|
-
const users = await s3db.createResource({
|
|
646
|
-
name: "users",
|
|
756
|
+
const analytics = await db.createResource({
|
|
757
|
+
name: 'analytics',
|
|
647
758
|
attributes: {
|
|
648
|
-
|
|
649
|
-
|
|
759
|
+
userId: 'string',
|
|
760
|
+
event: 'string',
|
|
761
|
+
timestamp: 'date',
|
|
762
|
+
region: 'string'
|
|
650
763
|
},
|
|
651
|
-
|
|
652
|
-
|
|
764
|
+
partitions: {
|
|
765
|
+
// Single field
|
|
766
|
+
byEvent: { fields: { event: 'string' } },
|
|
653
767
|
|
|
654
|
-
//
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
768
|
+
// Multiple fields (composite)
|
|
769
|
+
byEventAndRegion: {
|
|
770
|
+
fields: {
|
|
771
|
+
event: 'string',
|
|
772
|
+
region: 'string'
|
|
773
|
+
}
|
|
774
|
+
},
|
|
659
775
|
|
|
660
|
-
//
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
age: "number|optional"
|
|
776
|
+
// Nested field
|
|
777
|
+
byUserCountry: {
|
|
778
|
+
fields: {
|
|
779
|
+
'profile.country': 'string'
|
|
780
|
+
}
|
|
781
|
+
}
|
|
667
782
|
},
|
|
668
|
-
versioningEnabled: true
|
|
669
|
-
});
|
|
670
783
|
|
|
671
|
-
//
|
|
672
|
-
|
|
673
|
-
|
|
784
|
+
// Async partitions for 70-100% faster writes
|
|
785
|
+
asyncPartitions: true
|
|
786
|
+
});
|
|
674
787
|
|
|
675
|
-
// Query by
|
|
676
|
-
const
|
|
677
|
-
partition:
|
|
678
|
-
partitionValues: {
|
|
788
|
+
// Query by partition (O(1))
|
|
789
|
+
const usEvents = await analytics.list({
|
|
790
|
+
partition: 'byEventAndRegion',
|
|
791
|
+
partitionValues: { event: 'click', region: 'US' }
|
|
679
792
|
});
|
|
680
793
|
```
|
|
681
794
|
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
Flexible ID generation strategies:
|
|
795
|
+
**Automatic timestamp partitions:**
|
|
685
796
|
|
|
686
797
|
```javascript
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
name:
|
|
690
|
-
|
|
691
|
-
idSize: 8 // Generate 8-character IDs
|
|
692
|
-
});
|
|
693
|
-
|
|
694
|
-
// UUID support
|
|
695
|
-
import { v4 as uuidv4 } from 'uuid';
|
|
696
|
-
const uuidUsers = await s3db.createResource({
|
|
697
|
-
name: "uuid-users",
|
|
698
|
-
attributes: { name: "string|required" },
|
|
699
|
-
idGenerator: uuidv4
|
|
798
|
+
const events = await db.createResource({
|
|
799
|
+
name: 'events',
|
|
800
|
+
attributes: { name: 'string', data: 'object' },
|
|
801
|
+
timestamps: true // Auto-creates byCreatedDate and byUpdatedDate partitions
|
|
700
802
|
});
|
|
701
803
|
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
attributes: { name: "string|required" },
|
|
706
|
-
idGenerator: uuidv1
|
|
707
|
-
});
|
|
708
|
-
|
|
709
|
-
// Custom ID function
|
|
710
|
-
const timestampUsers = await s3db.createResource({
|
|
711
|
-
name: "timestamp-users",
|
|
712
|
-
attributes: { name: "string|required" },
|
|
713
|
-
idGenerator: () => `user_${Date.now()}`
|
|
804
|
+
const todayEvents = await events.list({
|
|
805
|
+
partition: 'byCreatedDate',
|
|
806
|
+
partitionValues: { createdAt: '2024-01-15' }
|
|
714
807
|
});
|
|
715
808
|
```
|
|
716
809
|
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
| Data Type | Original | Compressed | Space Saved |
|
|
722
|
-
|-----------|----------|------------|-------------|
|
|
723
|
-
| `10000` | `10000` (5 digits) | `2Bi` (3 digits) | **40%** |
|
|
724
|
-
| `123456789` | `123456789` (9 digits) | `8m0Kx` (5 digits) | **44%** |
|
|
725
|
-
| Large arrays | `[1,2,3,999999]` (13 chars) | `1,2,3,hBxM` (9 chars) | **31%** |
|
|
726
|
-
|
|
727
|
-
**Performance Benefits:**
|
|
728
|
-
- ⚡ **5x faster** encoding for large numbers vs Base36
|
|
729
|
-
- 🗜️ **41% compression** for typical numeric data
|
|
730
|
-
- 🚀 **Space efficient** - more data fits in S3 metadata
|
|
731
|
-
- 🔄 **Automatic** - no configuration required
|
|
732
|
-
|
|
733
|
-
### 🔌 Plugin System
|
|
734
|
-
|
|
735
|
-
Extend s3db.js with powerful plugins for caching, monitoring, replication, search, and more:
|
|
736
|
-
|
|
737
|
-
> **📦 Important:** Some plugins require additional dependencies. The s3db.js core package is lightweight (~500KB) and **does not include** plugin-specific dependencies. Install only what you need:
|
|
738
|
-
>
|
|
739
|
-
> ```bash
|
|
740
|
-
> # For PostgreSQL replication
|
|
741
|
-
> pnpm add pg
|
|
742
|
-
>
|
|
743
|
-
> # For BigQuery replication
|
|
744
|
-
> pnpm add @google-cloud/bigquery
|
|
745
|
-
>
|
|
746
|
-
> # For SQS replication/consumption
|
|
747
|
-
> pnpm add @aws-sdk/client-sqs
|
|
748
|
-
>
|
|
749
|
-
> # For RabbitMQ consumption
|
|
750
|
-
> pnpm add amqplib
|
|
751
|
-
> ```
|
|
752
|
-
>
|
|
753
|
-
> s3db.js will automatically validate dependencies at runtime and provide clear error messages with install commands if something is missing. See [Plugin Dependencies](#plugin-dependencies) for details.
|
|
810
|
+
### Hooks (Lifecycle Functions)
|
|
811
|
+
|
|
812
|
+
Add custom logic before/after operations:
|
|
754
813
|
|
|
755
814
|
```javascript
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
815
|
+
const products = await db.createResource({
|
|
816
|
+
name: 'products',
|
|
817
|
+
attributes: { name: 'string', price: 'number', sku: 'string' },
|
|
818
|
+
hooks: {
|
|
819
|
+
// Before operations
|
|
820
|
+
beforeInsert: [
|
|
821
|
+
async (data) => {
|
|
822
|
+
data.sku = `PROD-${Date.now()}`;
|
|
823
|
+
return data;
|
|
824
|
+
}
|
|
825
|
+
],
|
|
826
|
+
beforeUpdate: [
|
|
827
|
+
async (data) => {
|
|
828
|
+
data.updatedAt = new Date().toISOString();
|
|
829
|
+
return data;
|
|
830
|
+
}
|
|
831
|
+
],
|
|
764
832
|
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
}]
|
|
778
|
-
}),
|
|
779
|
-
new AuditPlugin() // 📝 Audit logging
|
|
780
|
-
]
|
|
833
|
+
// After operations
|
|
834
|
+
afterInsert: [
|
|
835
|
+
async (data) => {
|
|
836
|
+
console.log(`Product ${data.name} created with SKU ${data.sku}`);
|
|
837
|
+
}
|
|
838
|
+
],
|
|
839
|
+
afterDelete: [
|
|
840
|
+
async (data) => {
|
|
841
|
+
await notifyWarehouse(data.sku);
|
|
842
|
+
}
|
|
843
|
+
]
|
|
844
|
+
}
|
|
781
845
|
});
|
|
782
|
-
|
|
783
|
-
await s3db.connect();
|
|
784
|
-
|
|
785
|
-
// All plugins work together seamlessly
|
|
786
|
-
await users.insert({ name: "John", email: "john@example.com" });
|
|
787
|
-
// ✅ Data cached, costs tracked, indexed for search, metrics recorded, replicated, and audited
|
|
788
846
|
```
|
|
789
847
|
|
|
790
|
-
|
|
791
|
-
|
|
848
|
+
**Available hooks:**
|
|
849
|
+
- `beforeInsert`, `afterInsert`
|
|
850
|
+
- `beforeUpdate`, `afterUpdate`
|
|
851
|
+
- `beforeDelete`, `afterDelete`
|
|
852
|
+
- `beforeGet`, `afterGet`
|
|
853
|
+
- `beforeList`, `afterList`
|
|
792
854
|
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
#### Plugin Dependencies
|
|
855
|
+
### Middlewares (Method Wrappers)
|
|
796
856
|
|
|
797
|
-
|
|
857
|
+
Intercept and transform method calls:
|
|
798
858
|
|
|
799
|
-
|
|
859
|
+
```javascript
|
|
860
|
+
// Authentication middleware
|
|
861
|
+
users.useMiddleware('insert', async (ctx, next) => {
|
|
862
|
+
if (!ctx.args[0].userId) {
|
|
863
|
+
throw new Error('Authentication required');
|
|
864
|
+
}
|
|
865
|
+
return await next();
|
|
866
|
+
});
|
|
800
867
|
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
868
|
+
// Logging middleware
|
|
869
|
+
users.useMiddleware('update', async (ctx, next) => {
|
|
870
|
+
const start = Date.now();
|
|
871
|
+
const result = await next();
|
|
872
|
+
console.log(`Update took ${Date.now() - start}ms`);
|
|
873
|
+
return result;
|
|
874
|
+
});
|
|
805
875
|
|
|
806
|
-
|
|
876
|
+
// Validation middleware
|
|
877
|
+
users.useMiddleware('insert', async (ctx, next) => {
|
|
878
|
+
ctx.args[0].name = ctx.args[0].name.toUpperCase();
|
|
879
|
+
return await next();
|
|
880
|
+
});
|
|
881
|
+
```
|
|
807
882
|
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
| PostgreSQL Replicator | `pg` | `^8.0.0` | `pnpm add pg` |
|
|
811
|
-
| BigQuery Replicator | `@google-cloud/bigquery` | `^7.0.0` | `pnpm add @google-cloud/bigquery` |
|
|
812
|
-
| SQS Replicator | `@aws-sdk/client-sqs` | `^3.0.0` | `pnpm add @aws-sdk/client-sqs` |
|
|
813
|
-
| SQS Consumer | `@aws-sdk/client-sqs` | `^3.0.0` | `pnpm add @aws-sdk/client-sqs` |
|
|
814
|
-
| RabbitMQ Consumer | `amqplib` | `^0.10.0` | `pnpm add amqplib` |
|
|
815
|
-
| Tfstate Plugin | `node-cron` | `^4.0.0` | `pnpm add node-cron` |
|
|
883
|
+
**Supported methods:**
|
|
884
|
+
`get`, `list`, `insert`, `update`, `delete`, `deleteMany`, `exists`, `getMany`, `count`, `page`, `listIds`, `getAll`
|
|
816
885
|
|
|
817
|
-
|
|
886
|
+
### Events (Real-time Notifications)
|
|
818
887
|
|
|
819
|
-
|
|
820
|
-
Error: PostgreSQL Replicator - Missing dependencies detected!
|
|
888
|
+
Listen to resource operations:
|
|
821
889
|
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
890
|
+
```javascript
|
|
891
|
+
const users = await db.createResource({
|
|
892
|
+
name: 'users',
|
|
893
|
+
attributes: { name: 'string', email: 'string' },
|
|
826
894
|
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
```javascript
|
|
834
|
-
import { getPluginDependencyReport } from 's3db.js/plugins/concerns/plugin-dependencies';
|
|
835
|
-
|
|
836
|
-
// Get a full report of all plugin dependencies
|
|
837
|
-
const report = await getPluginDependencyReport();
|
|
838
|
-
console.log(report);
|
|
839
|
-
```
|
|
840
|
-
|
|
841
|
-
**Example:** See `docs/examples/e46-plugin-dependency-validation.js` for complete examples of dependency validation.
|
|
842
|
-
|
|
843
|
-
---
|
|
844
|
-
|
|
845
|
-
### 🎛️ Resource Behaviors
|
|
846
|
-
|
|
847
|
-
Choose the right behavior strategy for your use case:
|
|
848
|
-
|
|
849
|
-
#### Behavior Comparison
|
|
850
|
-
|
|
851
|
-
| Behavior | Enforcement | Data Loss | Event Emission | Use Case |
|
|
852
|
-
|------------------|-------------|-----------|----------------|-------------------------|
|
|
853
|
-
| `user-managed` | None | Possible | Warns | Dev/Test/Advanced users |
|
|
854
|
-
| `enforce-limits` | Strict | No | Throws | Production |
|
|
855
|
-
| `truncate-data` | Truncates | Yes | Warns | Content Mgmt |
|
|
856
|
-
| `body-overflow` | Truncates/Splits | Yes | Warns | Large objects |
|
|
857
|
-
| `body-only` | Unlimited | No | No | Large JSON/Logs |
|
|
858
|
-
|
|
859
|
-
#### User Managed Behavior (Default)
|
|
860
|
-
|
|
861
|
-
The `user-managed` behavior is the default for s3db resources. It provides no automatic enforcement of S3 metadata or body size limits, and does not modify or truncate data. Instead, it emits warnings via the `exceedsLimit` event when S3 metadata limits are exceeded, but allows all operations to proceed.
|
|
862
|
-
|
|
863
|
-
**Purpose & Use Cases:**
|
|
864
|
-
- For development, testing, or advanced users who want full control over resource metadata and body size.
|
|
865
|
-
- Useful when you want to handle S3 metadata limits yourself, or implement custom logic for warnings.
|
|
866
|
-
- Not recommended for production unless you have custom enforcement or validation in place.
|
|
867
|
-
|
|
868
|
-
**How It Works:**
|
|
869
|
-
- Emits an `exceedsLimit` event (with details) when a resource's metadata size exceeds the S3 2KB limit.
|
|
870
|
-
- Does NOT block, truncate, or modify data—operations always proceed.
|
|
871
|
-
- No automatic enforcement of any limits; user is responsible for handling warnings and data integrity.
|
|
872
|
-
|
|
873
|
-
**Event Emission:**
|
|
874
|
-
- Event: `exceedsLimit`
|
|
875
|
-
- Payload:
|
|
876
|
-
- `operation`: 'insert' | 'update' | 'upsert'
|
|
877
|
-
- `id` (for update/upsert): resource id
|
|
878
|
-
- `totalSize`: total metadata size in bytes
|
|
879
|
-
- `limit`: S3 metadata limit (2048 bytes)
|
|
880
|
-
- `excess`: number of bytes over the limit
|
|
881
|
-
- `data`: the offending data object
|
|
882
|
-
|
|
883
|
-
```javascript
|
|
884
|
-
// Flexible behavior - warns but doesn't block
|
|
885
|
-
const users = await s3db.createResource({
|
|
886
|
-
name: "users",
|
|
887
|
-
attributes: { name: "string", bio: "string" },
|
|
888
|
-
behavior: "user-managed" // Default
|
|
889
|
-
});
|
|
890
|
-
|
|
891
|
-
// Listen for limit warnings
|
|
892
|
-
users.on('exceedsLimit', (data) => {
|
|
893
|
-
console.warn(`Data exceeds 2KB limit by ${data.excess} bytes`, data);
|
|
894
|
-
});
|
|
895
|
-
|
|
896
|
-
// Operation continues despite warning
|
|
897
|
-
await users.insert({
|
|
898
|
-
name: "John",
|
|
899
|
-
bio: "A".repeat(3000) // > 2KB
|
|
900
|
-
});
|
|
901
|
-
```
|
|
902
|
-
|
|
903
|
-
**Best Practices & Warnings:**
|
|
904
|
-
- Exceeding S3 metadata limits will cause silent data loss or errors at the storage layer.
|
|
905
|
-
- Use this behavior only if you have custom logic to handle warnings and enforce limits.
|
|
906
|
-
- For production, prefer `enforce-limits` or `truncate-data` to avoid data loss.
|
|
907
|
-
|
|
908
|
-
**Migration Tips:**
|
|
909
|
-
- To migrate to a stricter behavior, change the resource's behavior to `enforce-limits` or `truncate-data`.
|
|
910
|
-
- Review emitted warnings to identify resources at risk of exceeding S3 limits.
|
|
911
|
-
|
|
912
|
-
#### Enforce Limits Behavior
|
|
913
|
-
|
|
914
|
-
```javascript
|
|
915
|
-
// Strict validation - throws error if limit exceeded
|
|
916
|
-
const settings = await s3db.createResource({
|
|
917
|
-
name: "settings",
|
|
918
|
-
attributes: { key: "string", value: "string" },
|
|
919
|
-
behavior: "enforce-limits"
|
|
920
|
-
});
|
|
921
|
-
|
|
922
|
-
// Throws error if data > 2KB
|
|
923
|
-
await settings.insert({
|
|
924
|
-
key: "large_setting",
|
|
925
|
-
value: "A".repeat(3000) // Throws: "S3 metadata size exceeds 2KB limit"
|
|
926
|
-
});
|
|
927
|
-
```
|
|
928
|
-
|
|
929
|
-
#### Data Truncate Behavior
|
|
930
|
-
|
|
931
|
-
```javascript
|
|
932
|
-
// Smart truncation - preserves structure, truncates content
|
|
933
|
-
const summaries = await s3db.createResource({
|
|
934
|
-
name: "summaries",
|
|
935
|
-
attributes: {
|
|
936
|
-
title: "string",
|
|
937
|
-
description: "string",
|
|
938
|
-
content: "string"
|
|
939
|
-
},
|
|
940
|
-
behavior: "truncate-data"
|
|
941
|
-
});
|
|
942
|
-
|
|
943
|
-
// Automatically truncates to fit within 2KB
|
|
944
|
-
const result = await summaries.insert({
|
|
945
|
-
title: "Short Title",
|
|
946
|
-
description: "A".repeat(1000),
|
|
947
|
-
content: "B".repeat(2000) // Will be truncated with "..."
|
|
948
|
-
});
|
|
949
|
-
|
|
950
|
-
// Retrieved data shows truncation
|
|
951
|
-
const retrieved = await summaries.get(result.id);
|
|
952
|
-
console.log(retrieved.content); // "B...B..." (truncated)
|
|
953
|
-
```
|
|
954
|
-
|
|
955
|
-
#### Body Overflow Behavior
|
|
956
|
-
|
|
957
|
-
```javascript
|
|
958
|
-
// Preserve all data by using S3 object body
|
|
959
|
-
const blogs = await s3db.createResource({
|
|
960
|
-
name: "blogs",
|
|
961
|
-
attributes: {
|
|
962
|
-
title: "string",
|
|
963
|
-
content: "string", // Can be very large
|
|
964
|
-
author: "string"
|
|
965
|
-
},
|
|
966
|
-
behavior: "body-overflow"
|
|
967
|
-
});
|
|
968
|
-
|
|
969
|
-
// Large content is automatically split between metadata and body
|
|
970
|
-
const blog = await blogs.insert({
|
|
971
|
-
title: "My Blog Post",
|
|
972
|
-
content: "A".repeat(5000), // Large content
|
|
973
|
-
author: "John Doe"
|
|
974
|
-
});
|
|
975
|
-
|
|
976
|
-
// All data is preserved and accessible
|
|
977
|
-
const retrieved = await blogs.get(blog.id);
|
|
978
|
-
console.log(retrieved.content.length); // 5000 (full content preserved)
|
|
979
|
-
console.log(retrieved._hasContent); // true (indicates body usage)
|
|
980
|
-
```
|
|
981
|
-
|
|
982
|
-
#### Body Only Behavior
|
|
895
|
+
// Declarative event listeners
|
|
896
|
+
events: {
|
|
897
|
+
insert: (event) => {
|
|
898
|
+
console.log('User created:', event.id, event.name);
|
|
899
|
+
},
|
|
983
900
|
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
},
|
|
993
|
-
behavior: "body-only"
|
|
994
|
-
});
|
|
901
|
+
update: [
|
|
902
|
+
(event) => console.log('Update detected:', event.id),
|
|
903
|
+
(event) => {
|
|
904
|
+
if (event.$before.email !== event.$after.email) {
|
|
905
|
+
console.log('Email changed!');
|
|
906
|
+
}
|
|
907
|
+
}
|
|
908
|
+
],
|
|
995
909
|
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
content: "A".repeat(100000), // 100KB content
|
|
1000
|
-
metadata: {
|
|
1001
|
-
author: "John Doe",
|
|
1002
|
-
tags: ["large", "document"],
|
|
1003
|
-
version: "1.0"
|
|
910
|
+
delete: (event) => {
|
|
911
|
+
console.log('User deleted:', event.id);
|
|
912
|
+
}
|
|
1004
913
|
}
|
|
1005
914
|
});
|
|
1006
915
|
|
|
1007
|
-
//
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
console.log(retrieved.metadata.author); // "John Doe"
|
|
1011
|
-
console.log(retrieved._hasContent); // true (indicates body usage)
|
|
1012
|
-
|
|
1013
|
-
// Perfect for storing large JSON documents, logs, or any large content
|
|
1014
|
-
const logEntry = await documents.insert({
|
|
1015
|
-
title: "Application Log",
|
|
1016
|
-
content: JSON.stringify({
|
|
1017
|
-
timestamp: new Date().toISOString(),
|
|
1018
|
-
level: "INFO",
|
|
1019
|
-
message: "Application started",
|
|
1020
|
-
details: {
|
|
1021
|
-
// ... large log details
|
|
1022
|
-
}
|
|
1023
|
-
}),
|
|
1024
|
-
metadata: { source: "api-server", environment: "production" }
|
|
916
|
+
// Programmatic listeners
|
|
917
|
+
users.on('insert', (event) => {
|
|
918
|
+
sendWelcomeEmail(event.email);
|
|
1025
919
|
});
|
|
1026
920
|
```
|
|
1027
921
|
|
|
1028
|
-
|
|
922
|
+
**Available events:**
|
|
923
|
+
`insert`, `update`, `delete`, `insertMany`, `deleteMany`, `list`, `count`, `get`, `getMany`
|
|
1029
924
|
|
|
1030
|
-
|
|
925
|
+
### Streaming API
|
|
1031
926
|
|
|
1032
|
-
|
|
927
|
+
Process large datasets efficiently:
|
|
1033
928
|
|
|
1034
929
|
```javascript
|
|
1035
|
-
//
|
|
930
|
+
// Readable stream
|
|
1036
931
|
const readableStream = await users.readable({
|
|
1037
|
-
batchSize: 50,
|
|
1038
|
-
concurrency: 10
|
|
932
|
+
batchSize: 50,
|
|
933
|
+
concurrency: 10
|
|
1039
934
|
});
|
|
1040
935
|
|
|
1041
|
-
// Process data as it streams
|
|
1042
936
|
readableStream.on('data', (user) => {
|
|
1043
|
-
console.log(
|
|
1044
|
-
// Process each user individually
|
|
1045
|
-
});
|
|
1046
|
-
|
|
1047
|
-
readableStream.on('error', (error) => {
|
|
1048
|
-
console.error('Stream error:', error);
|
|
937
|
+
console.log('Processing:', user.name);
|
|
1049
938
|
});
|
|
1050
939
|
|
|
1051
940
|
readableStream.on('end', () => {
|
|
1052
941
|
console.log('Stream completed');
|
|
1053
942
|
});
|
|
1054
943
|
|
|
1055
|
-
//
|
|
1056
|
-
readableStream.pause();
|
|
1057
|
-
setTimeout(() => readableStream.resume(), 1000);
|
|
1058
|
-
```
|
|
1059
|
-
|
|
1060
|
-
#### Writable Streams
|
|
1061
|
-
|
|
1062
|
-
```javascript
|
|
1063
|
-
// Configure writable stream for bulk operations
|
|
944
|
+
// Writable stream
|
|
1064
945
|
const writableStream = await users.writable({
|
|
1065
|
-
batchSize: 25,
|
|
1066
|
-
concurrency: 5
|
|
1067
|
-
});
|
|
1068
|
-
|
|
1069
|
-
// Write data to stream
|
|
1070
|
-
const userData = [
|
|
1071
|
-
{ name: 'User 1', email: 'user1@example.com' },
|
|
1072
|
-
{ name: 'User 2', email: 'user2@example.com' },
|
|
1073
|
-
// ... thousands more
|
|
1074
|
-
];
|
|
1075
|
-
|
|
1076
|
-
userData.forEach(user => {
|
|
1077
|
-
writableStream.write(user);
|
|
1078
|
-
});
|
|
1079
|
-
|
|
1080
|
-
// End stream and wait for completion
|
|
1081
|
-
writableStream.on('finish', () => {
|
|
1082
|
-
console.log('All users written successfully');
|
|
1083
|
-
});
|
|
1084
|
-
|
|
1085
|
-
writableStream.on('error', (error) => {
|
|
1086
|
-
console.error('Write error:', error);
|
|
946
|
+
batchSize: 25,
|
|
947
|
+
concurrency: 5
|
|
1087
948
|
});
|
|
1088
949
|
|
|
950
|
+
userData.forEach(user => writableStream.write(user));
|
|
1089
951
|
writableStream.end();
|
|
1090
952
|
```
|
|
1091
953
|
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
```javascript
|
|
1095
|
-
// Handle errors gracefully in streams
|
|
1096
|
-
const stream = await users.readable();
|
|
1097
|
-
|
|
1098
|
-
stream.on('error', (error, item) => {
|
|
1099
|
-
console.error(`Error processing item:`, error);
|
|
1100
|
-
console.log('Problematic item:', item);
|
|
1101
|
-
// Continue processing other items
|
|
1102
|
-
});
|
|
1103
|
-
|
|
1104
|
-
// Custom error handling for specific operations
|
|
1105
|
-
stream.on('data', async (user) => {
|
|
1106
|
-
try {
|
|
1107
|
-
await processUser(user);
|
|
1108
|
-
} catch (error) {
|
|
1109
|
-
console.error(`Failed to process user ${user.id}:`, error);
|
|
1110
|
-
}
|
|
1111
|
-
});
|
|
1112
|
-
```
|
|
1113
|
-
|
|
1114
|
-
### 📁 Binary Content Management
|
|
954
|
+
### Complete Resource Example
|
|
1115
955
|
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
#### Set Binary Content
|
|
956
|
+
A complex, production-ready resource showing all capabilities:
|
|
1119
957
|
|
|
1120
958
|
```javascript
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
// Set image content for user profile
|
|
1124
|
-
const imageBuffer = fs.readFileSync('profile.jpg');
|
|
1125
|
-
await users.setContent({
|
|
1126
|
-
id: 'user-123',
|
|
1127
|
-
buffer: imageBuffer,
|
|
1128
|
-
contentType: 'image/jpeg'
|
|
1129
|
-
});
|
|
1130
|
-
|
|
1131
|
-
// Set document content
|
|
1132
|
-
const documentBuffer = fs.readFileSync('document.pdf');
|
|
1133
|
-
await users.setContent({
|
|
1134
|
-
id: 'user-123',
|
|
1135
|
-
buffer: documentBuffer,
|
|
1136
|
-
contentType: 'application/pdf'
|
|
1137
|
-
});
|
|
1138
|
-
|
|
1139
|
-
// Set text content
|
|
1140
|
-
await users.setContent({
|
|
1141
|
-
id: 'user-123',
|
|
1142
|
-
buffer: 'Hello World',
|
|
1143
|
-
contentType: 'text/plain'
|
|
1144
|
-
});
|
|
1145
|
-
```
|
|
959
|
+
const orders = await db.createResource({
|
|
960
|
+
name: 'orders',
|
|
1146
961
|
|
|
1147
|
-
|
|
962
|
+
// Schema with all features
|
|
963
|
+
attributes: {
|
|
964
|
+
// Basic fields
|
|
965
|
+
orderId: 'string|required|unique',
|
|
966
|
+
userId: 'string|required',
|
|
967
|
+
status: 'string|required|enum:pending,processing,completed,cancelled',
|
|
968
|
+
total: 'number|required|min:0',
|
|
969
|
+
|
|
970
|
+
// Encrypted sensitive data
|
|
971
|
+
paymentToken: 'secret|required',
|
|
972
|
+
|
|
973
|
+
// Nested objects
|
|
974
|
+
customer: {
|
|
975
|
+
type: 'object',
|
|
976
|
+
props: {
|
|
977
|
+
name: 'string|required',
|
|
978
|
+
email: 'email|required',
|
|
979
|
+
phone: 'string|optional',
|
|
980
|
+
address: {
|
|
981
|
+
type: 'object',
|
|
982
|
+
props: {
|
|
983
|
+
street: 'string|required',
|
|
984
|
+
city: 'string|required',
|
|
985
|
+
country: 'string|required|length:2',
|
|
986
|
+
zipCode: 'string|required'
|
|
987
|
+
}
|
|
988
|
+
}
|
|
989
|
+
}
|
|
990
|
+
},
|
|
1148
991
|
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
if (content.buffer) {
|
|
1154
|
-
console.log('Content type:', content.contentType);
|
|
1155
|
-
console.log('Content size:', content.buffer.length);
|
|
1156
|
-
|
|
1157
|
-
// Save to file
|
|
1158
|
-
fs.writeFileSync('downloaded.jpg', content.buffer);
|
|
1159
|
-
} else {
|
|
1160
|
-
console.log('No content found');
|
|
1161
|
-
}
|
|
1162
|
-
```
|
|
992
|
+
// Arrays
|
|
993
|
+
items: 'array|items:object|min:1',
|
|
994
|
+
tags: 'array|items:string|unique|optional',
|
|
1163
995
|
|
|
1164
|
-
|
|
996
|
+
// Special types
|
|
997
|
+
ipAddress: 'ip4',
|
|
998
|
+
userAgent: 'string|max:500',
|
|
1165
999
|
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
console.log('Has content:', hasContent);
|
|
1000
|
+
// Embeddings for AI/ML
|
|
1001
|
+
orderEmbedding: 'embedding:384'
|
|
1002
|
+
},
|
|
1170
1003
|
|
|
1171
|
-
//
|
|
1172
|
-
|
|
1173
|
-
// User metadata remains, but binary content is removed
|
|
1174
|
-
```
|
|
1004
|
+
// Behavior for large orders
|
|
1005
|
+
behavior: 'body-overflow',
|
|
1175
1006
|
|
|
1176
|
-
|
|
1007
|
+
// Automatic timestamps
|
|
1008
|
+
timestamps: true,
|
|
1177
1009
|
|
|
1178
|
-
|
|
1010
|
+
// Versioning for schema evolution
|
|
1011
|
+
versioningEnabled: true,
|
|
1179
1012
|
|
|
1180
|
-
|
|
1013
|
+
// Custom ID generation
|
|
1014
|
+
idGenerator: () => `ORD-${Date.now()}-${Math.random().toString(36).substr(2, 5)}`,
|
|
1181
1015
|
|
|
1182
|
-
|
|
1183
|
-
// Partition with multiple fields
|
|
1184
|
-
const analytics = await s3db.createResource({
|
|
1185
|
-
name: "analytics",
|
|
1186
|
-
attributes: {
|
|
1187
|
-
userId: "string",
|
|
1188
|
-
event: "string",
|
|
1189
|
-
timestamp: "date",
|
|
1190
|
-
region: "string",
|
|
1191
|
-
device: "string"
|
|
1192
|
-
},
|
|
1016
|
+
// Partitions for efficient queries
|
|
1193
1017
|
partitions: {
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
byEventAndRegion: {
|
|
1199
|
-
fields: {
|
|
1200
|
-
event: "string",
|
|
1201
|
-
region: "string"
|
|
1202
|
-
}
|
|
1203
|
-
},
|
|
1204
|
-
|
|
1205
|
-
// Three field partition
|
|
1206
|
-
byEventRegionDevice: {
|
|
1018
|
+
byStatus: { fields: { status: 'string' } },
|
|
1019
|
+
byUser: { fields: { userId: 'string' } },
|
|
1020
|
+
byCountry: { fields: { 'customer.address.country': 'string' } },
|
|
1021
|
+
byUserAndStatus: {
|
|
1207
1022
|
fields: {
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
device: "string"
|
|
1211
|
-
}
|
|
1212
|
-
}
|
|
1213
|
-
}
|
|
1214
|
-
});
|
|
1215
|
-
```
|
|
1216
|
-
|
|
1217
|
-
#### Nested Field Partitions
|
|
1218
|
-
|
|
1219
|
-
```javascript
|
|
1220
|
-
// Partition by nested object fields
|
|
1221
|
-
const users = await s3db.createResource({
|
|
1222
|
-
name: "users",
|
|
1223
|
-
attributes: {
|
|
1224
|
-
name: "string",
|
|
1225
|
-
profile: {
|
|
1226
|
-
country: "string",
|
|
1227
|
-
city: "string",
|
|
1228
|
-
preferences: {
|
|
1229
|
-
theme: "string"
|
|
1023
|
+
userId: 'string',
|
|
1024
|
+
status: 'string'
|
|
1230
1025
|
}
|
|
1231
1026
|
}
|
|
1232
1027
|
},
|
|
1233
|
-
partitions: {
|
|
1234
|
-
byCountry: { fields: { "profile.country": "string" } },
|
|
1235
|
-
byCity: { fields: { "profile.city": "string" } },
|
|
1236
|
-
byTheme: { fields: { "profile.preferences.theme": "string" } }
|
|
1237
|
-
}
|
|
1238
|
-
});
|
|
1239
1028
|
|
|
1240
|
-
//
|
|
1241
|
-
|
|
1242
|
-
partition: "byCountry",
|
|
1243
|
-
partitionValues: { "profile.country": "US" }
|
|
1244
|
-
});
|
|
1245
|
-
|
|
1246
|
-
// Note: The system automatically manages partition references internally
|
|
1247
|
-
// Users should use standard list() method with partition parameters
|
|
1248
|
-
```
|
|
1249
|
-
|
|
1250
|
-
#### Automatic Timestamp Partitions
|
|
1029
|
+
// Async partitions for faster writes
|
|
1030
|
+
asyncPartitions: true,
|
|
1251
1031
|
|
|
1252
|
-
|
|
1253
|
-
// Enable automatic timestamp partitions
|
|
1254
|
-
const events = await s3db.createResource({
|
|
1255
|
-
name: "events",
|
|
1256
|
-
attributes: {
|
|
1257
|
-
name: "string",
|
|
1258
|
-
data: "object"
|
|
1259
|
-
},
|
|
1260
|
-
timestamps: true // Automatically adds byCreatedDate and byUpdatedDate
|
|
1261
|
-
});
|
|
1262
|
-
|
|
1263
|
-
// Query by creation date
|
|
1264
|
-
const todayEvents = await events.list({
|
|
1265
|
-
partition: "byCreatedDate",
|
|
1266
|
-
partitionValues: { createdAt: "2024-01-15" }
|
|
1267
|
-
});
|
|
1268
|
-
|
|
1269
|
-
// Query by update date
|
|
1270
|
-
const recentlyUpdated = await events.list({
|
|
1271
|
-
partition: "byUpdatedDate",
|
|
1272
|
-
partitionValues: { updatedAt: "2024-01-15" }
|
|
1273
|
-
});
|
|
1274
|
-
```
|
|
1275
|
-
|
|
1276
|
-
#### Partition Validation
|
|
1277
|
-
|
|
1278
|
-
```javascript
|
|
1279
|
-
// Partitions are automatically validated against attributes
|
|
1280
|
-
const users = await s3db.createResource({
|
|
1281
|
-
name: "users",
|
|
1282
|
-
attributes: {
|
|
1283
|
-
name: "string",
|
|
1284
|
-
email: "string",
|
|
1285
|
-
status: "string"
|
|
1286
|
-
},
|
|
1287
|
-
partitions: {
|
|
1288
|
-
byStatus: { fields: { status: "string" } }, // ✅ Valid
|
|
1289
|
-
byEmail: { fields: { email: "string" } } // ✅ Valid
|
|
1290
|
-
// byInvalid: { fields: { invalid: "string" } } // ❌ Would throw error
|
|
1291
|
-
}
|
|
1292
|
-
});
|
|
1293
|
-
```
|
|
1294
|
-
|
|
1295
|
-
### 🎣 Advanced Hooks System
|
|
1296
|
-
|
|
1297
|
-
Extend functionality with comprehensive hook system:
|
|
1298
|
-
|
|
1299
|
-
#### Hook Execution Order
|
|
1300
|
-
|
|
1301
|
-
```javascript
|
|
1302
|
-
const users = await s3db.createResource({
|
|
1303
|
-
name: "users",
|
|
1304
|
-
attributes: { name: "string", email: "string" },
|
|
1032
|
+
// Hooks for business logic
|
|
1305
1033
|
hooks: {
|
|
1306
1034
|
beforeInsert: [
|
|
1307
|
-
async (data)
|
|
1308
|
-
|
|
1309
|
-
|
|
1035
|
+
async function(data) {
|
|
1036
|
+
// Validate stock availability
|
|
1037
|
+
const available = await this.validateStock(data.items);
|
|
1038
|
+
if (!available) throw new Error('Insufficient stock');
|
|
1039
|
+
|
|
1040
|
+
// Calculate total
|
|
1041
|
+
data.total = data.items.reduce((sum, item) => sum + item.price * item.quantity, 0);
|
|
1042
|
+
|
|
1310
1043
|
return data;
|
|
1311
1044
|
},
|
|
1312
1045
|
async (data) => {
|
|
1313
|
-
|
|
1314
|
-
data.
|
|
1046
|
+
// Add metadata
|
|
1047
|
+
data.processedAt = new Date().toISOString();
|
|
1315
1048
|
return data;
|
|
1316
1049
|
}
|
|
1317
1050
|
],
|
|
1051
|
+
|
|
1318
1052
|
afterInsert: [
|
|
1319
1053
|
async (data) => {
|
|
1320
|
-
|
|
1321
|
-
await
|
|
1054
|
+
// Send confirmation email
|
|
1055
|
+
await sendOrderConfirmation(data.customer.email, data.orderId);
|
|
1322
1056
|
},
|
|
1323
1057
|
async (data) => {
|
|
1324
|
-
|
|
1325
|
-
await
|
|
1058
|
+
// Update inventory
|
|
1059
|
+
await updateInventory(data.items);
|
|
1326
1060
|
}
|
|
1327
|
-
]
|
|
1328
|
-
}
|
|
1329
|
-
});
|
|
1330
|
-
|
|
1331
|
-
// Execution order: beforeInsert hooks → insert → afterInsert hooks
|
|
1332
|
-
```
|
|
1333
|
-
|
|
1334
|
-
#### Version-Specific Hooks
|
|
1061
|
+
],
|
|
1335
1062
|
|
|
1336
|
-
|
|
1337
|
-
|
|
1338
|
-
|
|
1339
|
-
|
|
1340
|
-
|
|
1341
|
-
|
|
1342
|
-
hooks: {
|
|
1343
|
-
beforeInsert: [
|
|
1344
|
-
async (data) => {
|
|
1345
|
-
// Access resource context
|
|
1346
|
-
console.log('Current version:', this.version);
|
|
1063
|
+
beforeUpdate: [
|
|
1064
|
+
async function(data) {
|
|
1065
|
+
// Prevent status rollback
|
|
1066
|
+
if (data.status === 'cancelled' && this.previousStatus === 'completed') {
|
|
1067
|
+
throw new Error('Cannot cancel completed order');
|
|
1068
|
+
}
|
|
1347
1069
|
return data;
|
|
1348
1070
|
}
|
|
1349
|
-
]
|
|
1350
|
-
}
|
|
1351
|
-
});
|
|
1352
|
-
|
|
1353
|
-
// Listen for version updates
|
|
1354
|
-
users.on('versionUpdated', ({ oldVersion, newVersion }) => {
|
|
1355
|
-
console.log(`Resource updated from ${oldVersion} to ${newVersion}`);
|
|
1356
|
-
});
|
|
1357
|
-
```
|
|
1358
|
-
|
|
1359
|
-
#### Error Handling in Hooks
|
|
1071
|
+
],
|
|
1360
1072
|
|
|
1361
|
-
|
|
1362
|
-
const users = await s3db.createResource({
|
|
1363
|
-
name: "users",
|
|
1364
|
-
attributes: { name: "string", email: "string" },
|
|
1365
|
-
hooks: {
|
|
1366
|
-
beforeInsert: [
|
|
1367
|
-
async (data) => {
|
|
1368
|
-
try {
|
|
1369
|
-
// Validate external service
|
|
1370
|
-
await validateEmail(data.email);
|
|
1371
|
-
return data;
|
|
1372
|
-
} catch (error) {
|
|
1373
|
-
// Transform error or add context
|
|
1374
|
-
throw new Error(`Email validation failed: ${error.message}`);
|
|
1375
|
-
}
|
|
1376
|
-
}
|
|
1377
|
-
],
|
|
1378
|
-
afterInsert: [
|
|
1073
|
+
afterUpdate: [
|
|
1379
1074
|
async (data) => {
|
|
1380
|
-
|
|
1381
|
-
|
|
1382
|
-
|
|
1383
|
-
// Log but don't fail the operation
|
|
1384
|
-
console.error('Failed to send welcome email:', error);
|
|
1075
|
+
// Notify customer of status change
|
|
1076
|
+
if (data.$before.status !== data.$after.status) {
|
|
1077
|
+
await notifyStatusChange(data.customer.email, data.status);
|
|
1385
1078
|
}
|
|
1386
1079
|
}
|
|
1387
1080
|
]
|
|
1388
|
-
}
|
|
1389
|
-
});
|
|
1390
|
-
```
|
|
1081
|
+
},
|
|
1391
1082
|
|
|
1392
|
-
|
|
1083
|
+
// Events for monitoring
|
|
1084
|
+
events: {
|
|
1085
|
+
insert: (event) => {
|
|
1086
|
+
console.log(`Order ${event.orderId} created - Total: $${event.total}`);
|
|
1087
|
+
metrics.increment('orders.created');
|
|
1088
|
+
},
|
|
1393
1089
|
|
|
1394
|
-
|
|
1395
|
-
|
|
1396
|
-
|
|
1397
|
-
|
|
1398
|
-
|
|
1399
|
-
beforeInsert: [
|
|
1400
|
-
async function(data) {
|
|
1401
|
-
// 'this' is bound to the resource instance
|
|
1402
|
-
console.log('Resource name:', this.name);
|
|
1403
|
-
console.log('Resource version:', this.version);
|
|
1404
|
-
|
|
1405
|
-
// Access resource methods
|
|
1406
|
-
const exists = await this.exists(data.id);
|
|
1407
|
-
if (exists) {
|
|
1408
|
-
throw new Error('User already exists');
|
|
1090
|
+
update: [
|
|
1091
|
+
(event) => {
|
|
1092
|
+
if (event.$before.status !== event.$after.status) {
|
|
1093
|
+
console.log(`Order ${event.orderId}: ${event.$before.status} → ${event.$after.status}`);
|
|
1094
|
+
metrics.increment(`orders.status.${event.$after.status}`);
|
|
1409
1095
|
}
|
|
1410
|
-
|
|
1411
|
-
return data;
|
|
1412
1096
|
}
|
|
1413
|
-
]
|
|
1414
|
-
}
|
|
1415
|
-
});
|
|
1416
|
-
```
|
|
1417
|
-
|
|
1418
|
-
### 🧩 Resource Middlewares
|
|
1419
|
-
|
|
1420
|
-
The Resource class supports a powerful middleware system, similar to Express/Koa, allowing you to intercept, modify, or extend the behavior of core methods like `insert`, `get`, `update`, `delete`, `list`, and more.
|
|
1421
|
-
|
|
1422
|
-
**Supported methods for middleware:**
|
|
1423
|
-
- `get`
|
|
1424
|
-
- `list`
|
|
1425
|
-
- `listIds`
|
|
1426
|
-
- `getAll`
|
|
1427
|
-
- `count`
|
|
1428
|
-
- `page`
|
|
1429
|
-
- `insert`
|
|
1430
|
-
- `update`
|
|
1431
|
-
- `delete`
|
|
1432
|
-
- `deleteMany`
|
|
1433
|
-
- `exists`
|
|
1434
|
-
- `getMany`
|
|
1435
|
-
|
|
1436
|
-
#### Middleware Signature
|
|
1437
|
-
```js
|
|
1438
|
-
async function middleware(ctx, next) {
|
|
1439
|
-
// ctx.resource: Resource instance
|
|
1440
|
-
// ctx.args: arguments array (for the method)
|
|
1441
|
-
// ctx.method: method name (e.g., 'insert')
|
|
1442
|
-
// next(): calls the next middleware or the original method
|
|
1443
|
-
}
|
|
1444
|
-
```
|
|
1097
|
+
],
|
|
1445
1098
|
|
|
1446
|
-
|
|
1447
|
-
|
|
1448
|
-
|
|
1449
|
-
|
|
1450
|
-
|
|
1099
|
+
delete: (event) => {
|
|
1100
|
+
console.warn(`Order ${event.orderId} deleted`);
|
|
1101
|
+
metrics.increment('orders.deleted');
|
|
1102
|
+
}
|
|
1103
|
+
}
|
|
1451
1104
|
});
|
|
1452
1105
|
|
|
1453
|
-
|
|
1454
|
-
|
|
1455
|
-
//
|
|
1456
|
-
|
|
1457
|
-
|
|
1458
|
-
console.log('After insert:', result);
|
|
1459
|
-
return result;
|
|
1106
|
+
// Add middlewares for cross-cutting concerns
|
|
1107
|
+
orders.useMiddleware('insert', async (ctx, next) => {
|
|
1108
|
+
// Rate limiting
|
|
1109
|
+
await checkRateLimit(ctx.args[0].userId);
|
|
1110
|
+
return await next();
|
|
1460
1111
|
});
|
|
1461
1112
|
|
|
1462
|
-
|
|
1463
|
-
//
|
|
1464
|
-
// Before insert: { name: 'john', email: 'john@example.com' }
|
|
1465
|
-
// After insert: { id: '...', name: 'JOHN', email: 'john@example.com', ... }
|
|
1466
|
-
```
|
|
1467
|
-
|
|
1468
|
-
#### Example: Validation or Metrics Middleware
|
|
1469
|
-
```js
|
|
1470
|
-
users.useMiddleware('update', async (ctx, next) => {
|
|
1471
|
-
if (!ctx.args[1].email) throw new Error('Email is required for update!');
|
|
1113
|
+
orders.useMiddleware('update', async (ctx, next) => {
|
|
1114
|
+
// Audit logging
|
|
1472
1115
|
const start = Date.now();
|
|
1473
1116
|
const result = await next();
|
|
1474
|
-
|
|
1475
|
-
|
|
1117
|
+
await auditLog.write({
|
|
1118
|
+
action: 'order.update',
|
|
1119
|
+
orderId: ctx.args[0],
|
|
1120
|
+
duration: Date.now() - start,
|
|
1121
|
+
timestamp: new Date()
|
|
1122
|
+
});
|
|
1476
1123
|
return result;
|
|
1477
1124
|
});
|
|
1478
1125
|
```
|
|
1479
1126
|
|
|
1480
|
-
|
|
1127
|
+
**Complete documentation**: [**docs/resources.md**](./docs/resources.md)
|
|
1481
1128
|
|
|
1482
|
-
|
|
1129
|
+
---
|
|
1483
1130
|
|
|
1484
|
-
|
|
1485
|
-
import { S3db } from 's3db.js';
|
|
1131
|
+
## 🔌 Plugins
|
|
1486
1132
|
|
|
1487
|
-
|
|
1488
|
-
const database = new S3db({ connectionString: 's3://my-bucket/my-app' });
|
|
1489
|
-
await database.connect();
|
|
1133
|
+
Extend s3db.js with powerful plugins for caching, monitoring, replication, relationships, and more.
|
|
1490
1134
|
|
|
1491
|
-
|
|
1492
|
-
name: 'orders',
|
|
1493
|
-
attributes: {
|
|
1494
|
-
id: 'string|required',
|
|
1495
|
-
customerId: 'string|required',
|
|
1496
|
-
amount: 'number|required',
|
|
1497
|
-
status: 'string|required'
|
|
1498
|
-
}
|
|
1499
|
-
});
|
|
1500
|
-
|
|
1501
|
-
// Authentication middleware - runs on all operations
|
|
1502
|
-
['insert', 'update', 'delete', 'get'].forEach(method => {
|
|
1503
|
-
orders.useMiddleware(method, async (ctx, next) => {
|
|
1504
|
-
// Extract user from context (e.g., from JWT token)
|
|
1505
|
-
const user = ctx.user || ctx.args.find(arg => arg?.userId);
|
|
1506
|
-
|
|
1507
|
-
if (!user || !user.userId) {
|
|
1508
|
-
throw new Error(`Authentication required for ${method} operation`);
|
|
1509
|
-
}
|
|
1510
|
-
|
|
1511
|
-
// Add user info to context for other middlewares
|
|
1512
|
-
ctx.authenticatedUser = user;
|
|
1513
|
-
|
|
1514
|
-
return await next();
|
|
1515
|
-
});
|
|
1516
|
-
});
|
|
1135
|
+
### Available Plugins
|
|
1517
1136
|
|
|
1518
|
-
|
|
1519
|
-
|
|
1520
|
-
|
|
1521
|
-
|
|
1522
|
-
|
|
1523
|
-
|
|
1524
|
-
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
|
|
1528
|
-
|
|
1529
|
-
|
|
1530
|
-
userId: user.userId,
|
|
1531
|
-
method,
|
|
1532
|
-
args: ctx.args,
|
|
1533
|
-
duration: Date.now() - startTime,
|
|
1534
|
-
timestamp: new Date().toISOString(),
|
|
1535
|
-
success: true
|
|
1536
|
-
});
|
|
1537
|
-
|
|
1538
|
-
return result;
|
|
1539
|
-
} catch (error) {
|
|
1540
|
-
// Log failed operation
|
|
1541
|
-
console.log(`[AUDIT] ${method.toUpperCase()} FAILED`, {
|
|
1542
|
-
resource: 'orders',
|
|
1543
|
-
userId: user.userId,
|
|
1544
|
-
method,
|
|
1545
|
-
error: error.message,
|
|
1546
|
-
duration: Date.now() - startTime,
|
|
1547
|
-
timestamp: new Date().toISOString(),
|
|
1548
|
-
success: false
|
|
1549
|
-
});
|
|
1550
|
-
|
|
1551
|
-
throw error;
|
|
1552
|
-
}
|
|
1553
|
-
});
|
|
1554
|
-
});
|
|
1137
|
+
| Plugin | Description | Dependencies |
|
|
1138
|
+
|--------|-------------|--------------|
|
|
1139
|
+
| [**CachePlugin**](./docs/plugins/cache.md) | Memory/S3/filesystem caching with compression | None |
|
|
1140
|
+
| [**CostsPlugin**](./docs/plugins/costs.md) | AWS cost tracking and optimization | None |
|
|
1141
|
+
| [**MetricsPlugin**](./docs/plugins/metrics.md) | Performance monitoring and Prometheus export | None |
|
|
1142
|
+
| [**AuditPlugin**](./docs/plugins/audit.md) | Complete audit trail for all operations | None |
|
|
1143
|
+
| [**TTLPlugin**](./docs/plugins/ttl.md) | Auto-cleanup expired records (O(1) partition-based) | None |
|
|
1144
|
+
| [**RelationPlugin**](./docs/plugins/relation.md) | ORM-like relationships (10-100x faster queries) | None |
|
|
1145
|
+
| [**ReplicatorPlugin**](./docs/plugins/replicator.md) | Real-time replication to BigQuery, PostgreSQL, SQS | `pg`, `@google-cloud/bigquery`, `@aws-sdk/client-sqs` |
|
|
1146
|
+
| [**FullTextPlugin**](./docs/plugins/fulltext.md) | Full-text search with indexing | None |
|
|
1147
|
+
| [**EventualConsistencyPlugin**](./docs/plugins/eventual-consistency.md) | Eventually consistent counters and analytics | None |
|
|
1148
|
+
| [**GeoPlugin**](./docs/plugins/geo.md) | Geospatial queries and distance calculations | None |
|
|
1555
1149
|
|
|
1556
|
-
|
|
1557
|
-
orders.useMiddleware('delete', async (ctx, next) => {
|
|
1558
|
-
const user = ctx.authenticatedUser;
|
|
1559
|
-
|
|
1560
|
-
if (user.role !== 'admin') {
|
|
1561
|
-
throw new Error('Only admins can delete orders');
|
|
1562
|
-
}
|
|
1563
|
-
|
|
1564
|
-
return await next();
|
|
1565
|
-
});
|
|
1150
|
+
### Plugin Installation
|
|
1566
1151
|
|
|
1567
|
-
|
|
1568
|
-
|
|
1569
|
-
|
|
1570
|
-
const order = await orders.insert(
|
|
1571
|
-
{
|
|
1572
|
-
id: 'order-123',
|
|
1573
|
-
customerId: 'cust-456',
|
|
1574
|
-
amount: 99.99,
|
|
1575
|
-
status: 'pending'
|
|
1576
|
-
},
|
|
1577
|
-
{ user: { userId: 'user-789', role: 'customer' } }
|
|
1578
|
-
);
|
|
1579
|
-
|
|
1580
|
-
// This will fail - only admins can delete
|
|
1581
|
-
await orders.delete('order-123', {
|
|
1582
|
-
user: { userId: 'user-789', role: 'customer' }
|
|
1583
|
-
});
|
|
1584
|
-
|
|
1585
|
-
} catch (error) {
|
|
1586
|
-
console.error('Operation failed:', error.message);
|
|
1587
|
-
}
|
|
1588
|
-
|
|
1589
|
-
/*
|
|
1590
|
-
Expected output:
|
|
1591
|
-
[AUDIT] INSERT {
|
|
1592
|
-
resource: 'orders',
|
|
1593
|
-
userId: 'user-789',
|
|
1594
|
-
method: 'insert',
|
|
1595
|
-
args: [{ id: 'order-123', customerId: 'cust-456', amount: 99.99, status: 'pending' }],
|
|
1596
|
-
duration: 245,
|
|
1597
|
-
timestamp: '2024-01-15T10:30:45.123Z',
|
|
1598
|
-
success: true
|
|
1599
|
-
}
|
|
1152
|
+
```bash
|
|
1153
|
+
# Core plugins (no dependencies)
|
|
1154
|
+
# Included in s3db.js package
|
|
1600
1155
|
|
|
1601
|
-
|
|
1602
|
-
|
|
1603
|
-
|
|
1604
|
-
|
|
1605
|
-
method: 'delete',
|
|
1606
|
-
error: 'Only admins can delete orders',
|
|
1607
|
-
duration: 12,
|
|
1608
|
-
timestamp: '2024-01-15T10:30:45.456Z',
|
|
1609
|
-
success: false
|
|
1610
|
-
}
|
|
1611
|
-
*/
|
|
1156
|
+
# External dependencies (install only what you need)
|
|
1157
|
+
pnpm add pg # PostgreSQL replication
|
|
1158
|
+
pnpm add @google-cloud/bigquery # BigQuery replication
|
|
1159
|
+
pnpm add @aws-sdk/client-sqs # SQS replication/consumption
|
|
1612
1160
|
```
|
|
1613
1161
|
|
|
1614
|
-
|
|
1615
|
-
- 🔐 **Centralized Authentication**: One middleware handles auth for all operations
|
|
1616
|
-
- 📊 **Comprehensive Auditing**: All operations are logged with timing and user info
|
|
1617
|
-
- 🛡️ **Granular Permissions**: Different rules for different operations
|
|
1618
|
-
- ⚡ **Performance Tracking**: Built-in timing for operation monitoring
|
|
1619
|
-
- 🔧 **Easy to Maintain**: Add/remove middlewares without changing business logic
|
|
1162
|
+
### Quick Example
|
|
1620
1163
|
|
|
1621
|
-
|
|
1622
|
-
|
|
1623
|
-
|
|
1624
|
-
This system is ideal for cross-cutting concerns like logging, access control, custom validation, metrics, or request shaping.
|
|
1625
|
-
|
|
1626
|
-
---
|
|
1627
|
-
|
|
1628
|
-
### 🧩 Hooks vs Middlewares: Differences, Usage, and Coexistence
|
|
1629
|
-
|
|
1630
|
-
s3db.js supports **both hooks and middlewares** for resources. They are complementary tools for customizing and extending resource behavior.
|
|
1164
|
+
```javascript
|
|
1165
|
+
import { S3db, CachePlugin, MetricsPlugin, TTLPlugin } from 's3db.js';
|
|
1631
1166
|
|
|
1632
|
-
|
|
1633
|
-
|
|
1634
|
-
|
|
1635
|
-
|
|
1636
|
-
|
|
1167
|
+
const db = new S3db({
|
|
1168
|
+
connectionString: 's3://bucket/databases/myapp',
|
|
1169
|
+
plugins: [
|
|
1170
|
+
// Cache frequently accessed data
|
|
1171
|
+
new CachePlugin({
|
|
1172
|
+
driver: 'memory',
|
|
1173
|
+
ttl: 300000, // 5 minutes
|
|
1174
|
+
config: {
|
|
1175
|
+
maxMemoryPercent: 0.1, // 10% of system memory
|
|
1176
|
+
enableCompression: true
|
|
1177
|
+
}
|
|
1178
|
+
}),
|
|
1637
1179
|
|
|
1638
|
-
|
|
1180
|
+
// Track performance metrics
|
|
1181
|
+
new MetricsPlugin({
|
|
1182
|
+
enablePrometheus: true,
|
|
1183
|
+
port: 9090
|
|
1184
|
+
}),
|
|
1639
1185
|
|
|
1640
|
-
|
|
1641
|
-
|
|
1642
|
-
|
|
1643
|
-
|
|
1644
|
-
|
|
1186
|
+
// Auto-cleanup expired sessions
|
|
1187
|
+
new TTLPlugin({
|
|
1188
|
+
resources: {
|
|
1189
|
+
sessions: { ttl: 86400, onExpire: 'soft-delete' } // 24h
|
|
1190
|
+
}
|
|
1191
|
+
})
|
|
1192
|
+
]
|
|
1645
1193
|
});
|
|
1646
1194
|
```
|
|
1647
1195
|
|
|
1648
|
-
|
|
1649
|
-
- Middlewares are functions that **wrap** the entire method call (like Express/Koa middlewares).
|
|
1650
|
-
- They can **intercept, modify, block, or replace** the operation.
|
|
1651
|
-
- Middlewares can transform arguments, short-circuit the call, or modify the result.
|
|
1652
|
-
- Middlewares are registered with `useMiddleware(method, fn)`.
|
|
1196
|
+
### Creating Custom Plugins
|
|
1653
1197
|
|
|
1654
|
-
|
|
1655
|
-
```js
|
|
1656
|
-
users.useMiddleware('insert', async (ctx, next) => {
|
|
1657
|
-
if (!ctx.args[0].email) throw new Error('Email required');
|
|
1658
|
-
ctx.args[0].name = ctx.args[0].name.toUpperCase();
|
|
1659
|
-
const result = await next();
|
|
1660
|
-
return result;
|
|
1661
|
-
});
|
|
1662
|
-
```
|
|
1198
|
+
Simple plugin example:
|
|
1663
1199
|
|
|
1664
|
-
|
|
1665
|
-
|
|
1666
|
-
|----------------|------------------------------|------------------------------|
|
|
1667
|
-
| Placement | Before/after operation | Wraps the entire method |
|
|
1668
|
-
| Control | Cannot block/replace op | Can block/replace op |
|
|
1669
|
-
| Use case | Side effects, logging, etc. | Access control, transform |
|
|
1670
|
-
| Registration | `addHook(hookName, fn)` | `useMiddleware(method, fn)` |
|
|
1671
|
-
| Data access | Receives data only | Full context (args, method) |
|
|
1672
|
-
| Chaining | Runs in order, always passes | Runs in order, can short-circuit |
|
|
1673
|
-
|
|
1674
|
-
#### **How They Work Together**
|
|
1675
|
-
Hooks and middlewares can be used **together** on the same resource and method. The order of execution is:
|
|
1676
|
-
|
|
1677
|
-
1. **Middlewares** (before the operation)
|
|
1678
|
-
2. **Hooks** (`beforeX`)
|
|
1679
|
-
3. **Original operation**
|
|
1680
|
-
4. **Hooks** (`afterX`)
|
|
1681
|
-
5. **Middlewares** (after the operation, as the call stack unwinds)
|
|
1682
|
-
|
|
1683
|
-
**Example: Using Both**
|
|
1684
|
-
```js
|
|
1685
|
-
// Middleware: transforms input and checks permissions
|
|
1686
|
-
users.useMiddleware('insert', async (ctx, next) => {
|
|
1687
|
-
if (!userHasPermission(ctx.args[0])) throw new Error('Unauthorized');
|
|
1688
|
-
ctx.args[0].name = ctx.args[0].name.toUpperCase();
|
|
1689
|
-
const result = await next();
|
|
1690
|
-
return result;
|
|
1691
|
-
});
|
|
1200
|
+
```javascript
|
|
1201
|
+
import { Plugin } from 's3db.js';
|
|
1692
1202
|
|
|
1693
|
-
|
|
1694
|
-
|
|
1695
|
-
|
|
1696
|
-
|
|
1697
|
-
}
|
|
1203
|
+
export class MyPlugin extends Plugin {
|
|
1204
|
+
constructor(options = {}) {
|
|
1205
|
+
super(options);
|
|
1206
|
+
this.name = 'MyPlugin';
|
|
1207
|
+
}
|
|
1698
1208
|
|
|
1699
|
-
|
|
1700
|
-
|
|
1701
|
-
// Middleware runs (transforms/checks)
|
|
1702
|
-
// Hook runs (sends email)
|
|
1703
|
-
```
|
|
1209
|
+
async initialize(database) {
|
|
1210
|
+
console.log('Plugin initialized!');
|
|
1704
1211
|
|
|
1705
|
-
|
|
1706
|
-
|
|
1707
|
-
|
|
1708
|
-
|
|
1212
|
+
// Wrap methods
|
|
1213
|
+
this.wrapMethod('Resource', 'insert', async (original, resource, args) => {
|
|
1214
|
+
console.log(`Inserting into ${resource.name}`);
|
|
1215
|
+
const result = await original(...args);
|
|
1216
|
+
console.log(`Inserted: ${result.id}`);
|
|
1217
|
+
return result;
|
|
1218
|
+
});
|
|
1219
|
+
}
|
|
1220
|
+
}
|
|
1221
|
+
```
|
|
1709
1222
|
|
|
1710
|
-
|
|
1711
|
-
- Hooks are lightweight and ideal for observing or reacting to events.
|
|
1223
|
+
**Complete documentation**: [**docs/plugins/README.md**](./docs/plugins/README.md)
|
|
1712
1224
|
|
|
1713
1225
|
---
|
|
1714
1226
|
|
|
1715
|
-
|
|
1227
|
+
## 🤖 MCP & Integrations
|
|
1716
1228
|
|
|
1717
|
-
|
|
1229
|
+
### Model Context Protocol (MCP) Server
|
|
1718
1230
|
|
|
1719
|
-
|
|
1231
|
+
S3DB includes a powerful MCP server with **28 specialized tools** for database operations, debugging, and monitoring.
|
|
1720
1232
|
|
|
1721
|
-
|
|
1722
|
-
// Async events (default) - Non-blocking, better performance
|
|
1723
|
-
const asyncResource = await s3db.createResource({
|
|
1724
|
-
name: "users",
|
|
1725
|
-
attributes: { name: "string", email: "string" },
|
|
1726
|
-
asyncEvents: true // Optional, this is the default
|
|
1727
|
-
});
|
|
1233
|
+
#### Quick Start
|
|
1728
1234
|
|
|
1729
|
-
|
|
1730
|
-
|
|
1731
|
-
|
|
1732
|
-
|
|
1733
|
-
|
|
1734
|
-
});
|
|
1235
|
+
```bash
|
|
1236
|
+
# Claude CLI (one command)
|
|
1237
|
+
claude mcp add s3db \
|
|
1238
|
+
--transport stdio \
|
|
1239
|
+
-- npx -y s3db.js s3db-mcp --transport=stdio
|
|
1735
1240
|
|
|
1736
|
-
|
|
1737
|
-
|
|
1738
|
-
syncResource.setAsyncMode(true); // Switch to async mode
|
|
1241
|
+
# Standalone HTTP server
|
|
1242
|
+
npx s3db.js s3db-mcp --transport=sse
|
|
1739
1243
|
```
|
|
1740
1244
|
|
|
1741
|
-
|
|
1742
|
-
- **Async (default)**: Best for production, logging, analytics, non-critical operations
|
|
1743
|
-
- **Sync**: Testing, critical validations, operations that must complete before continuing
|
|
1245
|
+
#### Features
|
|
1744
1246
|
|
|
1745
|
-
|
|
1247
|
+
- ✅ **28 tools** - CRUD, debugging, partitions, bulk ops, export/import, monitoring
|
|
1248
|
+
- ✅ **Multiple transports** - SSE for web, stdio for CLI
|
|
1249
|
+
- ✅ **Auto-optimization** - Cache and cost tracking enabled by default
|
|
1250
|
+
- ✅ **Partition-aware** - Intelligent caching with partition support
|
|
1746
1251
|
|
|
1747
|
-
####
|
|
1748
|
-
Traditional EventEmitter pattern using `.on()`, `.once()`, or `.off()`:
|
|
1252
|
+
#### Tool Categories
|
|
1749
1253
|
|
|
1750
|
-
|
|
1751
|
-
|
|
1752
|
-
|
|
1753
|
-
|
|
1754
|
-
|
|
1755
|
-
|
|
1756
|
-
|
|
1757
|
-
});
|
|
1758
|
-
|
|
1759
|
-
// Single event listener
|
|
1760
|
-
users.on('insert', (event) => {
|
|
1761
|
-
console.log('User created:', event.name);
|
|
1762
|
-
});
|
|
1763
|
-
|
|
1764
|
-
// Multiple listeners for the same event
|
|
1765
|
-
users.on('update', (event) => {
|
|
1766
|
-
console.log('Update detected:', event.id);
|
|
1767
|
-
});
|
|
1254
|
+
1. **Connection** (3) - `dbConnect`, `dbDisconnect`, `dbStatus`
|
|
1255
|
+
2. **Debugging** (5) - `dbInspectResource`, `dbGetMetadata`, `resourceValidate`, `dbHealthCheck`, `resourceGetRaw`
|
|
1256
|
+
3. **Query** (2) - `resourceQuery`, `resourceSearch`
|
|
1257
|
+
4. **Partitions** (4) - `resourceListPartitions`, `dbFindOrphanedPartitions`, etc.
|
|
1258
|
+
5. **Bulk Ops** (3) - `resourceUpdateMany`, `resourceBulkUpsert`, `resourceDeleteAll`
|
|
1259
|
+
6. **Export/Import** (3) - `resourceExport`, `resourceImport`, `dbBackupMetadata`
|
|
1260
|
+
7. **Monitoring** (4) - `dbGetStats`, `resourceGetStats`, `cacheGetStats`, `dbClearCache`
|
|
1768
1261
|
|
|
1769
|
-
|
|
1770
|
-
if (event.$before.email !== event.$after.email) {
|
|
1771
|
-
console.log('Email changed!');
|
|
1772
|
-
}
|
|
1773
|
-
});
|
|
1774
|
-
```
|
|
1775
|
-
|
|
1776
|
-
#### **Declarative Event Listeners**
|
|
1777
|
-
Configure event listeners directly in the resource configuration for cleaner, more maintainable code:
|
|
1262
|
+
**Complete documentation**: [**docs/mcp.md**](./docs/mcp.md)
|
|
1778
1263
|
|
|
1779
|
-
|
|
1780
|
-
const users = await s3db.createResource({
|
|
1781
|
-
name: "users",
|
|
1782
|
-
attributes: {
|
|
1783
|
-
name: "string|required",
|
|
1784
|
-
email: "string|required"
|
|
1785
|
-
},
|
|
1786
|
-
events: {
|
|
1787
|
-
// Single event listener
|
|
1788
|
-
insert: (event) => {
|
|
1789
|
-
console.log('📝 User created:', {
|
|
1790
|
-
id: event.id,
|
|
1791
|
-
name: event.name,
|
|
1792
|
-
timestamp: new Date().toISOString()
|
|
1793
|
-
});
|
|
1794
|
-
},
|
|
1264
|
+
### Integrations
|
|
1795
1265
|
|
|
1796
|
-
|
|
1797
|
-
update: [
|
|
1798
|
-
(event) => {
|
|
1799
|
-
console.log('⚠️ Update detected for user:', event.id);
|
|
1800
|
-
},
|
|
1801
|
-
(event) => {
|
|
1802
|
-
const changes = [];
|
|
1803
|
-
if (event.$before.name !== event.$after.name) {
|
|
1804
|
-
changes.push(`name: ${event.$before.name} → ${event.$after.name}`);
|
|
1805
|
-
}
|
|
1806
|
-
if (event.$before.email !== event.$after.email) {
|
|
1807
|
-
changes.push(`email: ${event.$before.email} → ${event.$after.email}`);
|
|
1808
|
-
}
|
|
1809
|
-
if (changes.length > 0) {
|
|
1810
|
-
console.log('📝 Changes:', changes.join(', '));
|
|
1811
|
-
}
|
|
1812
|
-
}
|
|
1813
|
-
],
|
|
1266
|
+
s3db.js integrates seamlessly with:
|
|
1814
1267
|
|
|
1815
|
-
|
|
1816
|
-
|
|
1817
|
-
|
|
1818
|
-
|
|
1268
|
+
- **BigQuery** - Real-time data replication via ReplicatorPlugin
|
|
1269
|
+
- **PostgreSQL** - Sync to traditional databases via ReplicatorPlugin
|
|
1270
|
+
- **AWS SQS** - Event streaming and message queues
|
|
1271
|
+
- **RabbitMQ** - Message queue integration
|
|
1272
|
+
- **Prometheus** - Metrics export via MetricsPlugin
|
|
1273
|
+
- **Vector Databases** - Embedding field type with 77% compression
|
|
1819
1274
|
|
|
1820
|
-
|
|
1821
|
-
list: (result) => {
|
|
1822
|
-
console.log(`📋 Listed ${result.count} users, ${result.errors} errors`);
|
|
1823
|
-
}
|
|
1824
|
-
}
|
|
1825
|
-
});
|
|
1826
|
-
```
|
|
1275
|
+
---
|
|
1827
1276
|
|
|
1828
|
-
|
|
1277
|
+
## 🔧 CLI
|
|
1829
1278
|
|
|
1830
|
-
|
|
1831
|
-
|-------|-------------|-------------|
|
|
1832
|
-
| `insert` | Single record inserted | Complete object with all fields |
|
|
1833
|
-
| `update` | Single record updated | Object with `$before` and `$after` states |
|
|
1834
|
-
| `delete` | Single record deleted | Object data before deletion |
|
|
1835
|
-
| `insertMany` | Bulk insert completed | Number of records inserted |
|
|
1836
|
-
| `deleteMany` | Bulk delete completed | Number of records deleted |
|
|
1837
|
-
| `list` | List operation completed | Result object with count and errors |
|
|
1838
|
-
| `count` | Count operation completed | Total count number |
|
|
1839
|
-
| `get` | Single record retrieved | Complete object data |
|
|
1840
|
-
| `getMany` | Multiple records retrieved | Count of records |
|
|
1279
|
+
s3db.js includes a powerful CLI for database management and operations.
|
|
1841
1280
|
|
|
1842
|
-
|
|
1281
|
+
### Installation
|
|
1843
1282
|
|
|
1844
|
-
|
|
1845
|
-
|
|
1846
|
-
|
|
1847
|
-
id: 'user-123',
|
|
1848
|
-
name: 'John Doe',
|
|
1849
|
-
email: 'john@example.com',
|
|
1850
|
-
createdAt: '2023-12-01T10:00:00.000Z',
|
|
1851
|
-
// ... all other fields
|
|
1852
|
-
}
|
|
1853
|
-
```
|
|
1283
|
+
```bash
|
|
1284
|
+
# Global
|
|
1285
|
+
npm install -g s3db.js
|
|
1854
1286
|
|
|
1855
|
-
|
|
1856
|
-
|
|
1857
|
-
|
|
1858
|
-
id: 'user-123',
|
|
1859
|
-
name: 'John Updated',
|
|
1860
|
-
email: 'john.new@example.com',
|
|
1861
|
-
$before: {
|
|
1862
|
-
name: 'John Doe',
|
|
1863
|
-
email: 'john@example.com',
|
|
1864
|
-
// ... previous state
|
|
1865
|
-
},
|
|
1866
|
-
$after: {
|
|
1867
|
-
name: 'John Updated',
|
|
1868
|
-
email: 'john.new@example.com',
|
|
1869
|
-
// ... current state
|
|
1870
|
-
}
|
|
1871
|
-
}
|
|
1287
|
+
# Project
|
|
1288
|
+
npm install s3db.js
|
|
1289
|
+
npx s3db [command]
|
|
1872
1290
|
```
|
|
1873
1291
|
|
|
1874
|
-
|
|
1875
|
-
You can use both declarative and programmatic event listeners together:
|
|
1292
|
+
### Commands
|
|
1876
1293
|
|
|
1877
|
-
```
|
|
1878
|
-
|
|
1879
|
-
|
|
1880
|
-
attributes: { name: "string|required" },
|
|
1881
|
-
events: {
|
|
1882
|
-
insert: (event) => console.log('Config listener:', event.name)
|
|
1883
|
-
}
|
|
1884
|
-
});
|
|
1294
|
+
```bash
|
|
1295
|
+
# List resources
|
|
1296
|
+
s3db list
|
|
1885
1297
|
|
|
1886
|
-
|
|
1887
|
-
|
|
1888
|
-
|
|
1889
|
-
});
|
|
1298
|
+
# Query resources
|
|
1299
|
+
s3db query users
|
|
1300
|
+
s3db query users --filter '{"status":"active"}'
|
|
1890
1301
|
|
|
1891
|
-
|
|
1892
|
-
|
|
1893
|
-
// Config listener: John
|
|
1894
|
-
// Programmatic listener: John
|
|
1895
|
-
```
|
|
1302
|
+
# Insert records
|
|
1303
|
+
s3db insert users --data '{"name":"John","email":"john@example.com"}'
|
|
1896
1304
|
|
|
1897
|
-
|
|
1898
|
-
|
|
1899
|
-
- **Programmatic for conditional/dynamic**: Use `.on()` for listeners that might change at runtime
|
|
1900
|
-
- **Error handling**: Listeners should handle their own errors to avoid breaking operations
|
|
1901
|
-
- **Performance**: Keep listeners lightweight; async events (default) ensure non-blocking operations
|
|
1902
|
-
- **Testing**: Use `asyncEvents: false` in tests when you need predictable synchronous behavior
|
|
1903
|
-
- **Debugging**: Event listeners are excellent for debugging and monitoring
|
|
1904
|
-
- Middlewares are powerful and ideal for controlling or transforming operations.
|
|
1905
|
-
- You can safely combine both for maximum flexibility.
|
|
1305
|
+
# Update records
|
|
1306
|
+
s3db update users user-123 --data '{"age":31}'
|
|
1906
1307
|
|
|
1907
|
-
|
|
1308
|
+
# Delete records
|
|
1309
|
+
s3db delete users user-123
|
|
1908
1310
|
|
|
1909
|
-
|
|
1311
|
+
# Export data
|
|
1312
|
+
s3db export users --format json > users.json
|
|
1313
|
+
s3db export users --format csv > users.csv
|
|
1910
1314
|
|
|
1911
|
-
|
|
1315
|
+
# Import data
|
|
1316
|
+
s3db import users < users.json
|
|
1912
1317
|
|
|
1913
|
-
|
|
1318
|
+
# Stats
|
|
1319
|
+
s3db stats
|
|
1320
|
+
s3db stats users
|
|
1914
1321
|
|
|
1915
|
-
|
|
1916
|
-
|
|
1917
|
-
|
|
1918
|
-
--transport stdio \
|
|
1919
|
-
-- npx -y s3db.js s3db-mcp --transport=stdio
|
|
1322
|
+
# MCP Server
|
|
1323
|
+
s3db s3db-mcp --transport=stdio
|
|
1324
|
+
s3db s3db-mcp --transport=sse --port=17500
|
|
1920
1325
|
```
|
|
1921
1326
|
|
|
1922
|
-
|
|
1923
|
-
```json
|
|
1924
|
-
{
|
|
1925
|
-
"mcpServers": {
|
|
1926
|
-
"s3db": {
|
|
1927
|
-
"command": "npx",
|
|
1928
|
-
"args": ["-y", "s3db.js", "s3db-mcp", "--transport=sse"],
|
|
1929
|
-
"env": {
|
|
1930
|
-
"S3DB_CONNECTION_STRING": "s3://ACCESS_KEY:SECRET_KEY@bucket/databases/myapp",
|
|
1931
|
-
"S3DB_CACHE_ENABLED": "true",
|
|
1932
|
-
"S3DB_COSTS_ENABLED": "true"
|
|
1933
|
-
}
|
|
1934
|
-
}
|
|
1935
|
-
}
|
|
1936
|
-
}
|
|
1937
|
-
```
|
|
1327
|
+
### Environment Variables
|
|
1938
1328
|
|
|
1939
|
-
**Standalone Server:**
|
|
1940
1329
|
```bash
|
|
1941
|
-
|
|
1942
|
-
|
|
1330
|
+
S3DB_CONNECTION_STRING=s3://bucket/databases/myapp
|
|
1331
|
+
S3DB_CACHE_ENABLED=true
|
|
1332
|
+
S3DB_COSTS_ENABLED=true
|
|
1333
|
+
S3DB_VERBOSE=false
|
|
1943
1334
|
```
|
|
1944
1335
|
|
|
1945
|
-
|
|
1946
|
-
- [NPX Setup Guide](./mcp/NPX_SETUP.md) - Use with `npx` (recommended)
|
|
1947
|
-
- [Claude CLI Setup](./mcp/CLAUDE_CLI_SETUP.md) - Detailed configuration
|
|
1948
|
-
|
|
1949
|
-
### Available Tools
|
|
1336
|
+
---
|
|
1950
1337
|
|
|
1951
|
-
|
|
1338
|
+
## 📖 Documentation
|
|
1952
1339
|
|
|
1953
|
-
|
|
1954
|
-
- `dbConnect` - Connect with cache and costs tracking
|
|
1955
|
-
- `dbDisconnect` - Graceful disconnection
|
|
1956
|
-
- `dbStatus` - Check connection status
|
|
1340
|
+
### Core Documentation
|
|
1957
1341
|
|
|
1958
|
-
|
|
1959
|
-
-
|
|
1960
|
-
-
|
|
1961
|
-
-
|
|
1962
|
-
- `dbHealthCheck` - Comprehensive health check
|
|
1963
|
-
- `resourceGetRaw` - View raw S3 object data
|
|
1342
|
+
- [**Resources (Complete Guide)**](./docs/resources.md) - Everything about resources, schemas, behaviors, partitions, hooks, middlewares, events
|
|
1343
|
+
- [**Client Class**](./docs/client.md) - Low-level S3 operations and HTTP configuration
|
|
1344
|
+
- [**Schema Validation**](./docs/schema.md) - Comprehensive schema validation and field types
|
|
1345
|
+
- [**Plugins Overview**](./docs/plugins/README.md) - All available plugins and how to create custom ones
|
|
1964
1346
|
|
|
1965
|
-
|
|
1966
|
-
- `resourceQuery` - Complex queries with filters
|
|
1967
|
-
- `resourceSearch` - Full-text search
|
|
1347
|
+
### Plugin Documentation
|
|
1968
1348
|
|
|
1969
|
-
|
|
1970
|
-
-
|
|
1971
|
-
-
|
|
1972
|
-
-
|
|
1973
|
-
-
|
|
1349
|
+
- [Cache Plugin](./docs/plugins/cache.md)
|
|
1350
|
+
- [Costs Plugin](./docs/plugins/costs.md)
|
|
1351
|
+
- [Metrics Plugin](./docs/plugins/metrics.md)
|
|
1352
|
+
- [Audit Plugin](./docs/plugins/audit.md)
|
|
1353
|
+
- [TTL Plugin](./docs/plugins/ttl.md)
|
|
1354
|
+
- [Relation Plugin](./docs/plugins/relation.md)
|
|
1355
|
+
- [Replicator Plugin](./docs/plugins/replicator.md)
|
|
1974
1356
|
|
|
1975
|
-
|
|
1976
|
-
- `resourceUpdateMany` - Update with filters
|
|
1977
|
-
- `resourceBulkUpsert` - Bulk upsert operations
|
|
1978
|
-
- `resourceDeleteAll` - Delete all documents
|
|
1357
|
+
### MCP & Integrations
|
|
1979
1358
|
|
|
1980
|
-
|
|
1981
|
-
-
|
|
1982
|
-
-
|
|
1983
|
-
- `dbBackupMetadata` - Create metadata backups
|
|
1359
|
+
- [MCP Server Guide](./docs/mcp.md) - Complete MCP documentation with all 28 tools
|
|
1360
|
+
- [NPX Setup Guide](./mcp/NPX_SETUP.md) - Use MCP with npx
|
|
1361
|
+
- [Claude CLI Setup](./mcp/CLAUDE_CLI_SETUP.md) - Detailed Claude CLI configuration
|
|
1984
1362
|
|
|
1985
|
-
|
|
1986
|
-
- `dbGetStats` - Database statistics
|
|
1987
|
-
- `resourceGetStats` - Resource-specific stats
|
|
1988
|
-
- `cacheGetStats` - Cache performance metrics
|
|
1989
|
-
- `dbClearCache` - Clear cache
|
|
1363
|
+
### Benchmarks & Performance
|
|
1990
1364
|
|
|
1991
|
-
|
|
1365
|
+
- [Benchmark Index](./docs/benchmarks/README.md)
|
|
1366
|
+
- [Base62 Encoding](./docs/benchmarks/base62.md)
|
|
1367
|
+
- [All Types Encoding](./docs/benchmarks/all-types-encoding.md)
|
|
1368
|
+
- [String Encoding Optimizations](./docs/benchmarks/STRING-ENCODING-OPTIMIZATIONS.md)
|
|
1369
|
+
- [EventualConsistency Plugin](./docs/benchmarks/eventual-consistency.md)
|
|
1370
|
+
- [Partitions Matrix](./docs/benchmarks/partitions.md)
|
|
1371
|
+
- [Vector Clustering](./docs/benchmarks/vector-clustering.md)
|
|
1992
1372
|
|
|
1993
|
-
###
|
|
1373
|
+
### Examples
|
|
1994
1374
|
|
|
1995
|
-
|
|
1996
|
-
-
|
|
1997
|
-
-
|
|
1998
|
-
-
|
|
1999
|
-
-
|
|
2000
|
-
-
|
|
1375
|
+
Browse [**60+ examples**](./docs/examples/) covering:
|
|
1376
|
+
- Basic CRUD (e01-e07)
|
|
1377
|
+
- Advanced features (e08-e17)
|
|
1378
|
+
- Plugins (e18-e33)
|
|
1379
|
+
- Vectors & RAG (e41-e43)
|
|
1380
|
+
- Testing patterns (e38-e40, e64-e65)
|
|
2001
1381
|
|
|
2002
|
-
###
|
|
1382
|
+
### API Reference
|
|
2003
1383
|
|
|
2004
|
-
|
|
1384
|
+
| Resource | Link |
|
|
1385
|
+
|----------|------|
|
|
1386
|
+
| Resource API | [docs/resources.md](./docs/resources.md) |
|
|
1387
|
+
| Client API | [docs/client.md](./docs/client.md) |
|
|
1388
|
+
| Schema Validation | [docs/schema.md](./docs/schema.md) |
|
|
1389
|
+
| Plugin API | [docs/plugins/README.md](./docs/plugins/README.md) |
|
|
2005
1390
|
|
|
2006
|
-
|
|
2007
|
-
- Tool reference with all 28 tools
|
|
2008
|
-
- Configuration examples (AWS, MinIO, DigitalOcean)
|
|
2009
|
-
- Docker deployment guides
|
|
2010
|
-
- Performance optimization
|
|
2011
|
-
- Troubleshooting
|
|
2012
|
-
- Security best practices
|
|
1391
|
+
---
|
|
2013
1392
|
|
|
2014
|
-
|
|
1393
|
+
## 🔧 Troubleshooting
|
|
2015
1394
|
|
|
2016
|
-
|
|
2017
|
-
// Connect to database
|
|
2018
|
-
await dbConnect({
|
|
2019
|
-
connectionString: "s3://bucket/databases/app",
|
|
2020
|
-
enableCache: true,
|
|
2021
|
-
enableCosts: true
|
|
2022
|
-
})
|
|
2023
|
-
|
|
2024
|
-
// Inspect resource
|
|
2025
|
-
await dbInspectResource({ resourceName: "users" })
|
|
2026
|
-
|
|
2027
|
-
// Query with filters
|
|
2028
|
-
await resourceQuery({
|
|
2029
|
-
resourceName: "users",
|
|
2030
|
-
filters: { status: "active", age: { $gt: 18 } }
|
|
2031
|
-
})
|
|
2032
|
-
|
|
2033
|
-
// Export data
|
|
2034
|
-
await resourceExport({
|
|
2035
|
-
resourceName: "users",
|
|
2036
|
-
format: "csv",
|
|
2037
|
-
filters: { status: "active" }
|
|
2038
|
-
})
|
|
2039
|
-
|
|
2040
|
-
// Check health
|
|
2041
|
-
await dbHealthCheck({ includeOrphanedPartitions: true })
|
|
2042
|
-
```
|
|
1395
|
+
Common issues and solutions:
|
|
2043
1396
|
|
|
2044
|
-
|
|
1397
|
+
<details>
|
|
1398
|
+
<summary><strong>Connection Issues</strong></summary>
|
|
2045
1399
|
|
|
2046
|
-
|
|
2047
|
-
|
|
2048
|
-
### 📚 Core Classes Documentation
|
|
2049
|
-
|
|
2050
|
-
- **[Client Class](./docs/client.md)** - Low-level S3 operations, HTTP client configuration, and advanced object management
|
|
2051
|
-
- **[Database Class](./docs/database.md)** - High-level database interface (coming soon)
|
|
2052
|
-
- **[Resource Class](./docs/resource.md)** - Resource operations and methods (coming soon)
|
|
2053
|
-
|
|
2054
|
-
### 🔌 Database Operations
|
|
2055
|
-
|
|
2056
|
-
| Method | Description | Example |
|
|
2057
|
-
|--------|-------------|---------|
|
|
2058
|
-
| `connect()` | Connect to database | `await s3db.connect()` |
|
|
2059
|
-
| `createResource(config)` | Create new resource | `await s3db.createResource({...})` |
|
|
2060
|
-
| `resources.{name}` | Get resource reference | `const users = s3db.resources.users` |
|
|
2061
|
-
| `resourceExists(name)` | Check if resource exists | `s3db.resourceExists("users")` |
|
|
2062
|
-
|
|
2063
|
-
### ⚙️ Configuration Options
|
|
2064
|
-
|
|
2065
|
-
| Option | Type | Default | Description |
|
|
2066
|
-
|--------|------|---------|-------------|
|
|
2067
|
-
| `connectionString` | string | required | S3 connection string |
|
|
2068
|
-
| `httpClientOptions` | object | optimized | HTTP client configuration |
|
|
2069
|
-
| `verbose` | boolean | false | Enable verbose logging |
|
|
2070
|
-
| `parallelism` | number | 10 | Concurrent operations |
|
|
2071
|
-
| `versioningEnabled` | boolean | false | Enable resource versioning |
|
|
2072
|
-
|
|
2073
|
-
#### HTTP Client Options
|
|
2074
|
-
|
|
2075
|
-
| Option | Type | Default | Description |
|
|
2076
|
-
|--------|------|---------|-------------|
|
|
2077
|
-
| `keepAlive` | boolean | true | Enable connection reuse |
|
|
2078
|
-
| `keepAliveMsecs` | number | 1000 | Keep-alive duration (ms) |
|
|
2079
|
-
| `maxSockets` | number | 50 | Maximum concurrent connections |
|
|
2080
|
-
| `maxFreeSockets` | number | 10 | Free connections in pool |
|
|
2081
|
-
| `timeout` | number | 60000 | Request timeout (ms) |
|
|
2082
|
-
|
|
2083
|
-
### 📝 Resource Operations
|
|
2084
|
-
|
|
2085
|
-
| Method | Description | Example |
|
|
2086
|
-
|--------|-------------|---------|
|
|
2087
|
-
| `insert(data)` | Create document | `await users.insert({name: "John"})` |
|
|
2088
|
-
| `get(id)` | Retrieve document | `await users.get("user-123")` |
|
|
2089
|
-
| `update(id, data)` | Update document | `await users.update("user-123", {age: 31})` |
|
|
2090
|
-
| `upsert(id, data)` | Insert or update | `await users.upsert("user-123", {...})` |
|
|
2091
|
-
| `delete(id)` | Delete document | `await users.delete("user-123")` |
|
|
2092
|
-
| `exists(id)` | Check existence | `await users.exists("user-123")` |
|
|
2093
|
-
| `setContent({id, buffer, contentType})` | Set binary content | `await users.setContent({id: "123", buffer: imageBuffer})` |
|
|
2094
|
-
| `content(id)` | Get binary content | `await users.content("user-123")` |
|
|
2095
|
-
| `hasContent(id)` | Check if has content | `await users.hasContent("user-123")` |
|
|
2096
|
-
| `deleteContent(id)` | Remove content | `await users.deleteContent("user-123")` |
|
|
2097
|
-
|
|
2098
|
-
### 📊 Query Operations
|
|
2099
|
-
|
|
2100
|
-
| Method | Description | Example |
|
|
2101
|
-
|--------|-------------|---------|
|
|
2102
|
-
| `list(options?)` | List documents with pagination & partitions | `await users.list({limit: 10, offset: 0})` |
|
|
2103
|
-
| `listIds(options?)` | List document IDs | `await users.listIds()` |
|
|
2104
|
-
| `count(options?)` | Count documents | `await users.count()` |
|
|
2105
|
-
| `page(options)` | Paginate results | `await users.page({offset: 0, size: 10})` |
|
|
2106
|
-
| `query(filter, options?)` | Filter documents | `await users.query({isActive: true})` |
|
|
2107
|
-
|
|
2108
|
-
#### 📋 List vs GetAll - When to Use Each
|
|
2109
|
-
|
|
2110
|
-
**`list(options?)`** - Advanced listing with full control:
|
|
2111
|
-
```javascript
|
|
2112
|
-
// Simple listing (equivalent to getAll)
|
|
2113
|
-
const allUsers = await users.list();
|
|
1400
|
+
**Problem:** Cannot connect to S3 bucket
|
|
2114
1401
|
|
|
2115
|
-
|
|
2116
|
-
|
|
1402
|
+
**Solutions:**
|
|
1403
|
+
1. Verify credentials in connection string
|
|
1404
|
+
2. Check IAM permissions (s3:ListBucket, s3:GetObject, s3:PutObject)
|
|
1405
|
+
3. Ensure bucket exists
|
|
1406
|
+
4. Check network connectivity
|
|
2117
1407
|
|
|
2118
|
-
|
|
2119
|
-
|
|
2120
|
-
|
|
2121
|
-
|
|
1408
|
+
```javascript
|
|
1409
|
+
// Enable verbose logging
|
|
1410
|
+
const db = new S3db({
|
|
1411
|
+
connectionString: '...',
|
|
1412
|
+
verbose: true
|
|
2122
1413
|
});
|
|
2123
1414
|
```
|
|
1415
|
+
</details>
|
|
2124
1416
|
|
|
2125
|
-
|
|
2126
|
-
|
|
2127
|
-
// Get all documents (no options, no pagination)
|
|
2128
|
-
const allUsers = await users.getAll();
|
|
2129
|
-
console.log(`Total users: ${allUsers.length}`);
|
|
2130
|
-
```
|
|
1417
|
+
<details>
|
|
1418
|
+
<summary><strong>Metadata Size Exceeded</strong></summary>
|
|
2131
1419
|
|
|
2132
|
-
**
|
|
2133
|
-
- ✅ You want all documents without pagination
|
|
2134
|
-
- ✅ You don't need partition filtering
|
|
2135
|
-
- ✅ You prefer simplicity over flexibility
|
|
1420
|
+
**Problem:** Error: "S3 metadata size exceeds 2KB limit"
|
|
2136
1421
|
|
|
2137
|
-
**
|
|
2138
|
-
|
|
2139
|
-
|
|
2140
|
-
|
|
1422
|
+
**Solutions:**
|
|
1423
|
+
1. Change behavior to `body-overflow` or `body-only`
|
|
1424
|
+
2. Reduce field sizes or use truncation
|
|
1425
|
+
3. Move large content to separate fields
|
|
2141
1426
|
|
|
2142
|
-
|
|
1427
|
+
```javascript
|
|
1428
|
+
const resource = await db.createResource({
|
|
1429
|
+
name: 'blogs',
|
|
1430
|
+
behavior: 'body-overflow', // Automatically handle overflow
|
|
1431
|
+
attributes: { title: 'string', content: 'string' }
|
|
1432
|
+
});
|
|
1433
|
+
```
|
|
1434
|
+
</details>
|
|
2143
1435
|
|
|
2144
|
-
|
|
2145
|
-
|
|
2146
|
-
| `insertMany(docs)` | Insert multiple | `await users.insertMany([{...}, {...}])` |
|
|
2147
|
-
| `getMany(ids)` | Get multiple | `await users.getMany(["id1", "id2"])` |
|
|
2148
|
-
| `deleteMany(ids)` | Delete multiple | `await users.deleteMany(["id1", "id2"])` |
|
|
2149
|
-
| `getAll()` | Get all documents | `await users.getAll()` |
|
|
2150
|
-
| `deleteAll()` | Delete all documents | `await users.deleteAll()` |
|
|
1436
|
+
<details>
|
|
1437
|
+
<summary><strong>Performance Issues</strong></summary>
|
|
2151
1438
|
|
|
2152
|
-
|
|
1439
|
+
**Problem:** Slow queries or operations
|
|
2153
1440
|
|
|
2154
|
-
|
|
2155
|
-
|
|
2156
|
-
|
|
2157
|
-
|
|
1441
|
+
**Solutions:**
|
|
1442
|
+
1. Use partitions for frequently queried fields
|
|
1443
|
+
2. Enable caching with CachePlugin
|
|
1444
|
+
3. Increase HTTP client concurrency
|
|
1445
|
+
4. Use bulk operations instead of loops
|
|
2158
1446
|
|
|
2159
|
-
|
|
1447
|
+
```javascript
|
|
1448
|
+
// Add partitions
|
|
1449
|
+
const resource = await db.createResource({
|
|
1450
|
+
name: 'analytics',
|
|
1451
|
+
attributes: { event: 'string', region: 'string' },
|
|
1452
|
+
partitions: {
|
|
1453
|
+
byEvent: { fields: { event: 'string' } }
|
|
1454
|
+
},
|
|
1455
|
+
asyncPartitions: true // 70-100% faster writes
|
|
1456
|
+
});
|
|
2160
1457
|
|
|
2161
|
-
|
|
1458
|
+
// Enable caching
|
|
1459
|
+
const db = new S3db({
|
|
1460
|
+
connectionString: '...',
|
|
1461
|
+
plugins: [new CachePlugin({ ttl: 300000 })]
|
|
1462
|
+
});
|
|
1463
|
+
```
|
|
1464
|
+
</details>
|
|
2162
1465
|
|
|
2163
|
-
>
|
|
1466
|
+
<details>
|
|
1467
|
+
<summary><strong>Orphaned Partitions</strong></summary>
|
|
2164
1468
|
|
|
2165
|
-
|
|
1469
|
+
**Problem:** Partition references deleted field
|
|
2166
1470
|
|
|
2167
|
-
|
|
1471
|
+
**Solutions:**
|
|
2168
1472
|
|
|
2169
|
-
|
|
2170
|
-
|
|
2171
|
-
|
|
2172
|
-
|
|
1473
|
+
```javascript
|
|
1474
|
+
const resource = await db.getResource('users', { strictValidation: false });
|
|
1475
|
+
const orphaned = resource.findOrphanedPartitions();
|
|
1476
|
+
console.log('Orphaned:', orphaned);
|
|
2173
1477
|
|
|
2174
|
-
|
|
2175
|
-
|
|
2176
|
-
|
|
2177
|
-
|
|
2178
|
-
|
|
1478
|
+
// Remove them
|
|
1479
|
+
resource.removeOrphanedPartitions();
|
|
1480
|
+
await db.uploadMetadataFile();
|
|
1481
|
+
```
|
|
1482
|
+
</details>
|
|
2179
1483
|
|
|
2180
|
-
|
|
2181
|
-
- **Automatic type detection** and optimal encoding selection
|
|
2182
|
-
- **2-3x faster** UTF-8 byte calculations with caching
|
|
1484
|
+
---
|
|
2183
1485
|
|
|
2184
|
-
|
|
2185
|
-
- **13.5% average savings** for IPv4 addresses (up to 47% for longer IPs)
|
|
2186
|
-
- **38.5% savings** for full-length IPv6 addresses
|
|
2187
|
-
- **2.7M+ ops/s** for IPv4 encoding, **595K+ ops/s** for IPv6
|
|
2188
|
-
- **Ideal for metadata-constrained storage** (S3 2KB limit)
|
|
1486
|
+
## 📊 Performance Benchmarks
|
|
2189
1487
|
|
|
2190
|
-
|
|
1488
|
+
> **⚠️ Important**: All benchmark results documented were generated using **Node.js v22.6.0**. Performance may vary with different Node.js versions.
|
|
2191
1489
|
|
|
2192
|
-
|
|
2193
|
-
- **70-100% faster writes** with async partitions
|
|
2194
|
-
- **Parallel analytics updates** for high-throughput scenarios
|
|
2195
|
-
- **O(1) partition queries** vs O(n) full scans
|
|
1490
|
+
s3db.js includes comprehensive benchmarks demonstrating real-world performance optimizations:
|
|
2196
1491
|
|
|
2197
|
-
|
|
1492
|
+
- [**Base62 Encoding**](./docs/benchmarks/base62.md) - 40-46% space savings, 5x faster than Base36
|
|
1493
|
+
- [**All Types Encoding**](./docs/benchmarks/all-types-encoding.md) - Comprehensive encoding across all field types
|
|
1494
|
+
- [**String Encoding Optimizations**](./docs/benchmarks/STRING-ENCODING-OPTIMIZATIONS.md) - 2-3x faster UTF-8 calculations
|
|
1495
|
+
- [**EventualConsistency Plugin**](./docs/benchmarks/eventual-consistency.md) - 70-100% faster writes
|
|
1496
|
+
- [**Partitions Matrix**](./docs/benchmarks/partitions.md) - Test 110 combinations to find optimal config
|
|
1497
|
+
- [**Vector Clustering**](./docs/benchmarks/vector-clustering.md) - Vector similarity and clustering performance
|
|
2198
1498
|
|
|
2199
|
-
**[
|
|
2200
|
-
- **Test matrix**: 0-10 partitions × 1-10 attributes (110 combinations)
|
|
2201
|
-
- **Measurements**: Create, insert, query (partition & full scan)
|
|
2202
|
-
- **Insights**: Find optimal partition configuration for your use case
|
|
2203
|
-
- Run with: `pnpm run benchmark:partitions`
|
|
1499
|
+
**[📋 Complete Benchmark Index](./docs/benchmarks/README.md)**
|
|
2204
1500
|
|
|
2205
|
-
|
|
1501
|
+
---
|
|
2206
1502
|
|
|
2207
|
-
|
|
2208
|
-
- ✅ **TL;DR summary** - Quick results and recommendations
|
|
2209
|
-
- ✅ **Code examples** - Runnable benchmark scripts
|
|
2210
|
-
- ✅ **Performance metrics** - Real numbers with explanations
|
|
2211
|
-
- ✅ **Use cases** - When to apply each optimization
|
|
1503
|
+
## 🤝 Contributing
|
|
2212
1504
|
|
|
2213
|
-
|
|
1505
|
+
Contributions are welcome! Please read our [Contributing Guide](CONTRIBUTING.md) for details on our code of conduct and the process for submitting pull requests.
|
|
2214
1506
|
|
|
2215
1507
|
---
|
|
2216
1508
|
|
|
2217
|
-
##
|
|
2218
|
-
|
|
2219
|
-
s3db.js includes optimized HTTP client settings by default for excellent S3 performance. You can customize these settings based on your specific needs:
|
|
1509
|
+
## 📄 License
|
|
2220
1510
|
|
|
2221
|
-
|
|
1511
|
+
This project is licensed under the [Unlicense](LICENSE) - see the LICENSE file for details.
|
|
2222
1512
|
|
|
2223
|
-
|
|
2224
|
-
const s3db = new S3db({
|
|
2225
|
-
connectionString: "s3://ACCESS_KEY:SECRET_KEY@BUCKET_NAME/databases/myapp",
|
|
2226
|
-
// Default HTTP client options (optimized for most applications):
|
|
2227
|
-
httpClientOptions: {
|
|
2228
|
-
keepAlive: true, // Enable connection reuse
|
|
2229
|
-
keepAliveMsecs: 1000, // Keep connections alive for 1 second
|
|
2230
|
-
maxSockets: 50, // Maximum 50 concurrent connections
|
|
2231
|
-
maxFreeSockets: 10, // Keep 10 free connections in pool
|
|
2232
|
-
timeout: 60000 // 60 second timeout
|
|
2233
|
-
}
|
|
2234
|
-
});
|
|
2235
|
-
```
|
|
1513
|
+
---
|
|
2236
1514
|
|
|
2237
|
-
|
|
1515
|
+
## 🙏 Acknowledgments
|
|
2238
1516
|
|
|
2239
|
-
|
|
2240
|
-
|
|
2241
|
-
|
|
2242
|
-
connectionString: "s3://ACCESS_KEY:SECRET_KEY@BUCKET_NAME/databases/myapp",
|
|
2243
|
-
httpClientOptions: {
|
|
2244
|
-
keepAlive: true,
|
|
2245
|
-
keepAliveMsecs: 1000,
|
|
2246
|
-
maxSockets: 100, // Higher concurrency
|
|
2247
|
-
maxFreeSockets: 20, // More free connections
|
|
2248
|
-
timeout: 60000
|
|
2249
|
-
}
|
|
2250
|
-
});
|
|
2251
|
-
```
|
|
1517
|
+
- Built with [AWS SDK for JavaScript](https://aws.amazon.com/sdk-for-javascript/)
|
|
1518
|
+
- Validation powered by [@icebob/fastest-validator](https://github.com/icebob/fastest-validator)
|
|
1519
|
+
- ID generation using [nanoid](https://github.com/ai/nanoid)
|
|
2252
1520
|
|
|
2253
|
-
|
|
2254
|
-
```javascript
|
|
2255
|
-
const s3db = new S3db({
|
|
2256
|
-
connectionString: "s3://ACCESS_KEY:SECRET_KEY@BUCKET_NAME/databases/myapp",
|
|
2257
|
-
httpClientOptions: {
|
|
2258
|
-
keepAlive: true,
|
|
2259
|
-
keepAliveMsecs: 5000, // Longer keep-alive
|
|
2260
|
-
maxSockets: 200, // High concurrency
|
|
2261
|
-
maxFreeSockets: 50, // Large connection pool
|
|
2262
|
-
timeout: 120000 // 2 minute timeout
|
|
2263
|
-
}
|
|
2264
|
-
});
|
|
2265
|
-
```
|
|
1521
|
+
---
|
|
2266
1522
|
|
|
2267
|
-
|
|
2268
|
-
|
|
2269
|
-
|
|
2270
|
-
connectionString: "s3://ACCESS_KEY:SECRET_KEY@BUCKET_NAME/databases/myapp",
|
|
2271
|
-
httpClientOptions: {
|
|
2272
|
-
keepAlive: true,
|
|
2273
|
-
keepAliveMsecs: 500, // Shorter keep-alive
|
|
2274
|
-
maxSockets: 10, // Lower concurrency
|
|
2275
|
-
maxFreeSockets: 2, // Smaller pool
|
|
2276
|
-
timeout: 15000 // 15 second timeout
|
|
2277
|
-
}
|
|
2278
|
-
});
|
|
2279
|
-
```
|
|
1523
|
+
<p align="center">
|
|
1524
|
+
Made with ❤️ by the s3db.js community
|
|
1525
|
+
</p>
|