s3db.js 3.3.1 → 4.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +99 -857
- package/dist/s3db.cjs.js +5449 -2028
- package/dist/s3db.cjs.min.js +15 -7
- package/dist/s3db.d.ts +133 -0
- package/dist/s3db.es.js +5453 -2032
- package/dist/s3db.es.min.js +15 -7
- package/dist/s3db.iife.js +5450 -2030
- package/dist/s3db.iife.min.js +15 -7
- package/package.json +36 -16
- package/UNLICENSE +0 -24
- package/docker-compose.yml +0 -18
- package/jest.config.js +0 -22
- package/rollup.config.js +0 -77
- package/scripts/prefix-files-istanbul-ignore.js +0 -18
package/README.md
CHANGED
|
@@ -1,908 +1,150 @@
|
|
|
1
1
|
# s3db.js
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
Use AWS S3, the world's most reliable document storage, as a database with this ORM.
|
|
4
4
|
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
<table width="100%">
|
|
8
|
-
<tr>
|
|
9
|
-
<td>
|
|
10
|
-
|
|
11
|
-
1. <a href="#motivation">Motivation</a>
|
|
12
|
-
1. <a href="#usage">Usage</a>
|
|
13
|
-
1. <a href="#install">Install</a>
|
|
14
|
-
1. <a href="#quick-setup">Quick Setup</a>
|
|
15
|
-
1. <a href="#insights">Insights</a>
|
|
16
|
-
1. <a href="#database">Database</a>
|
|
17
|
-
1. <a href="#create-a-resource">Create a resource</a>
|
|
18
|
-
1. <a href="#resource-methods">Resource methods</a>
|
|
19
|
-
1. <a href="#insert-one">Insert one</a>
|
|
20
|
-
1. <a href="#get-one">Get one</a>
|
|
21
|
-
1. <a href="#update-one">Update one</a>
|
|
22
|
-
1. <a href="#delete-one">Delete one</a>
|
|
23
|
-
1. <a href="#count">Count</a>
|
|
24
|
-
1. <a href="#insert-many">Insert many</a>
|
|
25
|
-
1. <a href="#get-many">Get many</a>
|
|
26
|
-
1. <a href="#get-all">Get all</a>
|
|
27
|
-
1. <a href="#delete-many">Delete many</a>
|
|
28
|
-
1. <a href="#delete-all">Delete all</a>
|
|
29
|
-
1. <a href="#list-ids">List ids</a>
|
|
30
|
-
1. <a href="#resource-streams">Resource streams</a>
|
|
31
|
-
1. <a href="#readable-stream">Readable stream</a>
|
|
32
|
-
1. <a href="#writable-stream">Writable stream</a>
|
|
33
|
-
1. <a href="#s3-client">S3 Client</a>
|
|
34
|
-
1. <a href="#events">Events</a>
|
|
35
|
-
1. <a href="#plugins">Plugins</a>
|
|
36
|
-
1. <a href="#cost-simulation">Cost Simulation</a>
|
|
37
|
-
1. <a href="#big-example">Big Example</a>
|
|
38
|
-
1. <a href="#small-example">Small example</a>
|
|
39
|
-
1. <a href="#roadmap">Roadmap</a>
|
|
40
|
-
|
|
41
|
-
</td>
|
|
42
|
-
</tr>
|
|
43
|
-
</table>
|
|
44
|
-
|
|
45
|
-
---
|
|
46
|
-
|
|
47
|
-
## Motivation
|
|
48
|
-
|
|
49
|
-
First of all:
|
|
50
|
-
|
|
51
|
-
1. Nothing is for free, but it can be cheaper.
|
|
52
|
-
2. I'm not responsible for your AWS Costs strategy, use `s3db.js` at your own risk.
|
|
53
|
-
3. Please, do not use in production!
|
|
54
|
-
|
|
55
|
-
**Let's go!**
|
|
56
|
-
|
|
57
|
-
You might know AWS's S3 product for its high availability and its cheap pricing rules. I'll show you another clever and funny way to use S3.
|
|
58
|
-
|
|
59
|
-
AWS allows you define `Metadata` to every single file you upload into your bucket. This attribute must be defined within a **2kb** limit using in `UTF-8` encoding. As this encoding [may vary the bytes width for each symbol](https://en.wikipedia.org/wiki/UTF-8) you may use [500 to 2000] chars of metadata storage. Follow the docs at [AWS S3 User Guide: Using metadata](https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingMetadata.html#object-metadata).
|
|
60
|
-
|
|
61
|
-
There is another management subset of data called `tags` that is used globally as [key, value] params. You can assign 10 tags with the conditions of: the key must be at most 128 unicode chars lengthy and the value up to 256 chars. With those key-values we can use more `2.5kb` of data, unicode will allow you to use up to 2500 more chars. Follow the official docs at [AWS User Guide: Object Tagging](https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-tagging.html).
|
|
62
|
-
|
|
63
|
-
With all this set you may store objects that should be able to store up to `4.5kb` of free space **per object**.
|
|
64
|
-
|
|
65
|
-
Check the <a href="#cost-simulation">cost simulation</a> section below for a deep cost dive!
|
|
66
|
-
|
|
67
|
-
Lets give it a try! :)
|
|
68
|
-
|
|
69
|
-
---
|
|
70
|
-
|
|
71
|
-
## Usage
|
|
72
|
-
|
|
73
|
-
You may check the snippets bellow or go straight to the <a href="#examples">Examples</a> section!
|
|
74
|
-
|
|
75
|
-
### Install
|
|
5
|
+
## Installation
|
|
76
6
|
|
|
77
7
|
```bash
|
|
78
|
-
npm
|
|
79
|
-
|
|
80
|
-
# or
|
|
81
|
-
|
|
82
|
-
yarn add s3db.js
|
|
8
|
+
npm install s3db.js
|
|
83
9
|
```
|
|
84
10
|
|
|
85
|
-
|
|
11
|
+
## Quick Start
|
|
86
12
|
|
|
87
|
-
|
|
13
|
+
### Node.js (ES Modules)
|
|
88
14
|
|
|
89
15
|
```javascript
|
|
90
|
-
import
|
|
16
|
+
import S3db from 's3db.js';
|
|
91
17
|
|
|
92
|
-
const {
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
const s3db = new S3db({
|
|
99
|
-
uri: `s3://${AWS_ACCESS_KEY_ID}:${AWS_SECRET_ACCESS_KEY}@${AWS_BUCKET}/databases/mydatabase`
|
|
18
|
+
const db = new S3db({
|
|
19
|
+
region: 'us-east-1',
|
|
20
|
+
accessKeyId: 'your-access-key',
|
|
21
|
+
secretAccessKey: 'your-secret-key',
|
|
22
|
+
bucket: 'your-bucket-name'
|
|
100
23
|
});
|
|
101
24
|
|
|
102
|
-
|
|
103
|
-
.connect()
|
|
104
|
-
.then(() => console.log('connected!')))
|
|
105
|
-
```
|
|
106
|
-
|
|
107
|
-
If you do use `dotenv` package:
|
|
108
|
-
|
|
109
|
-
```javascript
|
|
110
|
-
import * as dotenv from "dotenv";
|
|
111
|
-
dotenv.config();
|
|
112
|
-
|
|
113
|
-
import { S3db } from "s3db.js";
|
|
114
|
-
```
|
|
115
|
-
|
|
116
|
-
### Insights
|
|
117
|
-
|
|
118
|
-
- This implementation of ORM simulates a document repository. Due to the fact that `s3db.js` uses `aws-sdk`'s' S3 api; all requests are GET/PUT as `key=value` resources. So the best case scenario is to access like a document implementation.
|
|
119
|
-
|
|
120
|
-
- For better use of the <a href="#cache">cache</a> and listing, the best ID format is to use sequential ids with leading zeros (eq: 00001, 00002, 00003) due to S3 internal keys sorting method. But you will need to manage this incremental ID by your own.
|
|
121
|
-
|
|
122
|
-
### Database
|
|
123
|
-
|
|
124
|
-
Your `s3db.js` client can be initiated with options:
|
|
125
|
-
|
|
126
|
-
| option | optional | description | type | default |
|
|
127
|
-
| :---------: | :------: | :-------------------------------------------------: | :-------: | :---------: |
|
|
128
|
-
| cache | true | Persist searched data to reduce repeated requests | `boolean` | `undefined` |
|
|
129
|
-
| parallelism | true | Number of simultaneous tasks | `number` | 10 |
|
|
130
|
-
| passphrase | true | Your encryption secret | `string` | `undefined` |
|
|
131
|
-
| ttl | true | (Coming soon) TTL to your cache duration in seconds | `number` | 86400 |
|
|
132
|
-
| uri | false | A url as your S3 connection string | `string` | `undefined` |
|
|
133
|
-
|
|
134
|
-
Config example:
|
|
135
|
-
|
|
136
|
-
```javascript
|
|
137
|
-
const {
|
|
138
|
-
AWS_BUCKET = "my-bucket",
|
|
139
|
-
AWS_ACCESS_KEY_ID = "secret",
|
|
140
|
-
AWS_SECRET_ACCESS_KEY = "secret",
|
|
141
|
-
AWS_BUCKET_PREFIX = "databases/test-" + Date.now(),
|
|
142
|
-
} = process.env;
|
|
143
|
-
|
|
144
|
-
const uri = `s3://${AWS_ACCESS_KEY_ID}:${AWS_SECRET_ACCESS_KEY}@${AWS_BUCKET}/${AWS_BUCKET_PREFIX}`;
|
|
145
|
-
|
|
146
|
-
const options = {
|
|
147
|
-
uri,
|
|
148
|
-
parallelism: 25,
|
|
149
|
-
passphrase: fs.readFileSync("./cert.pem"),
|
|
150
|
-
};
|
|
151
|
-
```
|
|
152
|
-
|
|
153
|
-
#### s3db.connect()
|
|
154
|
-
|
|
155
|
-
This method must always be invoked before any operation take place. This will interact with AWS' S3 api and check the itens below:
|
|
156
|
-
|
|
157
|
-
1. With current credentials:
|
|
158
|
-
- Check if client has access to the S3 bucket.
|
|
159
|
-
- Check if client has access to bucket life-cycle policies.
|
|
160
|
-
1. With defined database:
|
|
161
|
-
- Check if there is already a database in this connection string.
|
|
162
|
-
- If any database is found, downloads it's medatada and loads each `Resource` definition.
|
|
163
|
-
- Else, it will generate an empty <a href="#metadata-file">`metadata`</a> file into this prefix and mark that this is a new database from scratch.
|
|
25
|
+
await db.connect();
|
|
164
26
|
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
{
|
|
171
|
-
// file version
|
|
172
|
-
"version": "1",
|
|
173
|
-
|
|
174
|
-
// previously defined resources
|
|
175
|
-
"resources": {
|
|
176
|
-
// definition example
|
|
177
|
-
"leads": {
|
|
178
|
-
"name": "leads",
|
|
179
|
-
|
|
180
|
-
// resource options
|
|
181
|
-
"options": {},
|
|
182
|
-
|
|
183
|
-
// resource defined schema
|
|
184
|
-
"schema": {
|
|
185
|
-
"name": "string",
|
|
186
|
-
"token": "secret"
|
|
187
|
-
},
|
|
188
|
-
|
|
189
|
-
// rules to simplify metadata usage
|
|
190
|
-
"mapper": {
|
|
191
|
-
"name": "0",
|
|
192
|
-
"token": "1"
|
|
193
|
-
},
|
|
194
|
-
}
|
|
27
|
+
const users = db.resource('users', {
|
|
28
|
+
schema: {
|
|
29
|
+
name: { type: 'string', required: true },
|
|
30
|
+
email: { type: 'string', required: true },
|
|
31
|
+
age: { type: 'number' }
|
|
195
32
|
}
|
|
196
|
-
}
|
|
197
|
-
```
|
|
198
|
-
|
|
199
|
-
### Create a resource
|
|
200
|
-
|
|
201
|
-
Resources are definitions of data collections.
|
|
202
|
-
|
|
203
|
-
```javascript
|
|
204
|
-
// resource
|
|
205
|
-
const attributes = {
|
|
206
|
-
utm: {
|
|
207
|
-
source: "string|optional",
|
|
208
|
-
medium: "string|optional",
|
|
209
|
-
campaign: "string|optional",
|
|
210
|
-
term: "string|optional",
|
|
211
|
-
},
|
|
212
|
-
lead: {
|
|
213
|
-
fullName: "string",
|
|
214
|
-
mobileNumber: "string",
|
|
215
|
-
personalEmail: "email",
|
|
216
|
-
},
|
|
217
|
-
};
|
|
218
|
-
|
|
219
|
-
const resource = await s3db.createResource({
|
|
220
|
-
name: "leads",
|
|
221
|
-
attributes,
|
|
222
33
|
});
|
|
223
|
-
```
|
|
224
|
-
|
|
225
|
-
Resources' names **cannot** prefix each other, like: `leads` and `leads-copy`! S3's api lists keys using prefix notation, so every time you list `leads`, all keys of `leads-copy` will appear as well.
|
|
226
|
-
|
|
227
|
-
##### Attributes
|
|
228
|
-
|
|
229
|
-
`s3db.js` use the [fastest-validator](https://www.npmjs.com/package/fastest-validator) package to define and validate your resource. Some few examples:
|
|
230
|
-
|
|
231
|
-
```javascript
|
|
232
|
-
const attributes = {
|
|
233
|
-
// few simple examples
|
|
234
|
-
name: "string|min:4|max:64|trim",
|
|
235
|
-
email: "email|nullable",
|
|
236
|
-
mobile: "string|optional",
|
|
237
|
-
count: "number|integer|positive",
|
|
238
|
-
corrency: "corrency|symbol:R$",
|
|
239
|
-
createdAt: "date",
|
|
240
|
-
website: "url",
|
|
241
|
-
id: "uuid",
|
|
242
|
-
ids: "array|items:uuid|unique",
|
|
243
|
-
|
|
244
|
-
// s3db defines a custom type "secret" that is encrypted
|
|
245
|
-
token: "secret",
|
|
246
|
-
|
|
247
|
-
// nested data works aswell
|
|
248
|
-
geo: {
|
|
249
|
-
lat: "number",
|
|
250
|
-
long: "number",
|
|
251
|
-
city: "string",
|
|
252
|
-
},
|
|
253
|
-
|
|
254
|
-
// may have multiple definitions.
|
|
255
|
-
address_number: ["string", "number"],
|
|
256
|
-
};
|
|
257
|
-
```
|
|
258
|
-
|
|
259
|
-
##### Reference:
|
|
260
|
-
|
|
261
|
-
You may just use the reference:
|
|
262
|
-
|
|
263
|
-
```javascript
|
|
264
|
-
const Leads = s3db.resource("leads");
|
|
265
|
-
```
|
|
266
34
|
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
The `fastest-validator` starts with the params below:
|
|
274
|
-
|
|
275
|
-
```javascript
|
|
276
|
-
// fastest-validator params
|
|
277
|
-
{
|
|
278
|
-
useNewCustomCheckerFunction: true,
|
|
279
|
-
defaults: {
|
|
280
|
-
object: {
|
|
281
|
-
strict: "remove",
|
|
282
|
-
},
|
|
283
|
-
},
|
|
284
|
-
}
|
|
285
|
-
```
|
|
286
|
-
|
|
287
|
-
---
|
|
288
|
-
|
|
289
|
-
## Resources methods
|
|
290
|
-
|
|
291
|
-
Consider `resource` as:
|
|
292
|
-
|
|
293
|
-
```javascript
|
|
294
|
-
const resource = s3db.resource("leads");
|
|
295
|
-
```
|
|
296
|
-
|
|
297
|
-
### Insert one
|
|
298
|
-
|
|
299
|
-
```javascript
|
|
300
|
-
// data
|
|
301
|
-
const insertedData = await resource.insert({
|
|
302
|
-
id: "mypersonal@email.com", // if not defined a id will be generated!
|
|
303
|
-
utm: {
|
|
304
|
-
source: "abc",
|
|
305
|
-
},
|
|
306
|
-
lead: {
|
|
307
|
-
fullName: "My Complex Name",
|
|
308
|
-
personalEmail: "mypersonal@email.com",
|
|
309
|
-
mobileNumber: "+5511234567890",
|
|
310
|
-
},
|
|
311
|
-
invalidAttr: "this attribute will disappear",
|
|
35
|
+
// Insert data
|
|
36
|
+
const user = await users.insert({
|
|
37
|
+
name: 'John Doe',
|
|
38
|
+
email: 'john@example.com',
|
|
39
|
+
age: 30
|
|
312
40
|
});
|
|
313
41
|
|
|
314
|
-
//
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
// source: "abc",
|
|
318
|
-
// },
|
|
319
|
-
// lead: {
|
|
320
|
-
// fullName: "My Complex Name",
|
|
321
|
-
// personalEmail: "mypersonal@email.com",
|
|
322
|
-
// mobileNumber: "+5511234567890",
|
|
323
|
-
// },
|
|
324
|
-
// invalidAttr: "this attribute will disappear",
|
|
325
|
-
// }
|
|
42
|
+
// Query data
|
|
43
|
+
const allUsers = await users.find();
|
|
44
|
+
const john = await users.findOne({ name: 'John Doe' });
|
|
326
45
|
```
|
|
327
46
|
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
### Get one
|
|
47
|
+
### Node.js (CommonJS)
|
|
331
48
|
|
|
332
49
|
```javascript
|
|
333
|
-
const
|
|
334
|
-
|
|
335
|
-
// {
|
|
336
|
-
// id: "mypersonal@email.com",
|
|
337
|
-
// utm: {
|
|
338
|
-
// source: "abc",
|
|
339
|
-
// },
|
|
340
|
-
// lead: {
|
|
341
|
-
// fullName: "My Complex Name",
|
|
342
|
-
// personalEmail: "mypersonal@email.com",
|
|
343
|
-
// mobileNumber: "+5511234567890",
|
|
344
|
-
// },
|
|
345
|
-
// }
|
|
346
|
-
```
|
|
347
|
-
|
|
348
|
-
### Update one
|
|
50
|
+
const S3db = require('s3db.js');
|
|
349
51
|
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
lead: {
|
|
353
|
-
fullName: "My New Name",
|
|
354
|
-
mobileNumber: "+5511999999999",
|
|
355
|
-
},
|
|
52
|
+
const db = new S3db({
|
|
53
|
+
connectionString: 's3://access-key:secret-key@bucket-name/prefix?region=us-east-1'
|
|
356
54
|
});
|
|
357
55
|
|
|
358
|
-
|
|
359
|
-
//
|
|
360
|
-
// utm: {
|
|
361
|
-
// source: "abc",
|
|
362
|
-
// },
|
|
363
|
-
// lead: {
|
|
364
|
-
// fullName: "My New Name",
|
|
365
|
-
// personalEmail: "mypersonal@email.com",
|
|
366
|
-
// mobileNumber: "+5511999999999",
|
|
367
|
-
// },
|
|
368
|
-
// }
|
|
56
|
+
await db.connect();
|
|
57
|
+
// ... rest of the code
|
|
369
58
|
```
|
|
370
59
|
|
|
371
|
-
###
|
|
60
|
+
### Browser
|
|
372
61
|
|
|
373
|
-
```
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
You may bulk insert data with a friendly method that receives a list of objects.
|
|
388
|
-
|
|
389
|
-
```javascript
|
|
390
|
-
const objects = new Array(100).fill(0).map((v, k) => ({
|
|
391
|
-
id: `bulk-${k}@mymail.com`,
|
|
392
|
-
lead: {
|
|
393
|
-
fullName: "My Test Name",
|
|
394
|
-
personalEmail: `bulk-${k}@mymail.com`,
|
|
395
|
-
mobileNumber: "+55 11 1234567890",
|
|
396
|
-
},
|
|
397
|
-
}));
|
|
398
|
-
|
|
399
|
-
await resource.insertMany(objects);
|
|
400
|
-
```
|
|
401
|
-
|
|
402
|
-
Keep in mind that we need to send a request to each object to be created. There is an option to change the amount of simultaneos connections that your client will handle.
|
|
403
|
-
|
|
404
|
-
```javascript
|
|
405
|
-
const s3db = new S3db({
|
|
406
|
-
parallelism: 100, // default = 10
|
|
407
|
-
});
|
|
408
|
-
```
|
|
409
|
-
|
|
410
|
-
This method uses [`supercharge/promise-pool`](https://github.com/supercharge/promise-pool) to organize the parallel promises.
|
|
411
|
-
|
|
412
|
-
### Get many
|
|
413
|
-
|
|
414
|
-
```javascript
|
|
415
|
-
await resource.getMany(["id1", "id2", "id3 "]);
|
|
62
|
+
```html
|
|
63
|
+
<!DOCTYPE html>
|
|
64
|
+
<html>
|
|
65
|
+
<head>
|
|
66
|
+
<script src="https://unpkg.com/s3db.js@latest/dist/s3db.iife.min.js"></script>
|
|
67
|
+
</head>
|
|
68
|
+
<body>
|
|
69
|
+
<script>
|
|
70
|
+
const db = new s3db.S3db({
|
|
71
|
+
region: 'us-east-1',
|
|
72
|
+
accessKeyId: 'your-access-key',
|
|
73
|
+
secretAccessKey: 'your-secret-key',
|
|
74
|
+
bucket: 'your-bucket-name'
|
|
75
|
+
});
|
|
416
76
|
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
77
|
+
db.connect().then(() => {
|
|
78
|
+
const users = db.resource('users');
|
|
79
|
+
return users.insert({ name: 'John', email: 'john@example.com' });
|
|
80
|
+
});
|
|
81
|
+
</script>
|
|
82
|
+
</body>
|
|
83
|
+
</html>
|
|
422
84
|
```
|
|
423
85
|
|
|
424
|
-
|
|
86
|
+
## Features
|
|
425
87
|
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
```
|
|
88
|
+
- 🚀 **High Performance**: Optimized for large datasets with streaming support
|
|
89
|
+
- 🔒 **Security**: Built-in encryption and compression
|
|
90
|
+
- 📊 **Schema Validation**: Automatic data validation with customizable schemas
|
|
91
|
+
- 🔄 **Caching**: Intelligent caching with TTL support
|
|
92
|
+
- 📦 **Partitioning**: Automatic data partitioning for better performance
|
|
93
|
+
- 🔌 **Plugin System**: Extensible with custom plugins
|
|
94
|
+
- 🌐 **Universal**: Works in Node.js and browsers
|
|
95
|
+
- 📝 **TypeScript**: Full TypeScript support with autocomplete
|
|
435
96
|
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
```javascript
|
|
439
|
-
await resource.deleteMany(["id1", "id2", "id3 "]);
|
|
440
|
-
```
|
|
97
|
+
## API Reference
|
|
441
98
|
|
|
442
|
-
###
|
|
99
|
+
### S3db Class
|
|
443
100
|
|
|
444
|
-
|
|
445
|
-
await resource.deleteAll();
|
|
446
|
-
```
|
|
101
|
+
The main database class for connecting to S3 and managing resources.
|
|
447
102
|
|
|
448
|
-
|
|
103
|
+
#### Constructor
|
|
449
104
|
|
|
450
105
|
```javascript
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
// [
|
|
454
|
-
// 'id1',
|
|
455
|
-
// 'id2',
|
|
456
|
-
// 'id3',
|
|
457
|
-
// ]
|
|
106
|
+
new S3db(config)
|
|
458
107
|
```
|
|
459
108
|
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
109
|
+
**Config Options:**
|
|
110
|
+
- `connectionString` (string): S3 connection string
|
|
111
|
+
- `region` (string): AWS region
|
|
112
|
+
- `accessKeyId` (string): AWS access key
|
|
113
|
+
- `secretAccessKey` (string): AWS secret key
|
|
114
|
+
- `bucket` (string): S3 bucket name
|
|
115
|
+
- `prefix` (string): Key prefix for all objects
|
|
116
|
+
- `encryption` (boolean): Enable encryption (default: false)
|
|
117
|
+
- `compression` (boolean): Enable compression (default: false)
|
|
118
|
+
- `cache` (boolean): Enable caching (default: true)
|
|
119
|
+
- `cacheTTL` (number): Cache TTL in seconds (default: 300)
|
|
463
120
|
|
|
464
|
-
|
|
121
|
+
#### Methods
|
|
465
122
|
|
|
466
|
-
|
|
123
|
+
- `connect()`: Connect to S3
|
|
124
|
+
- `disconnect()`: Disconnect from S3
|
|
125
|
+
- `resource(name, config)`: Create or get a resource
|
|
126
|
+
- `listResources()`: List all resources
|
|
127
|
+
- `getVersion()`: Get package version
|
|
467
128
|
|
|
468
|
-
|
|
469
|
-
const readableStream = await resource.readable();
|
|
470
|
-
|
|
471
|
-
readableStream.on("id", (id) => console.log("id =", id));
|
|
472
|
-
readableStream.on("data", (lead) => console.log("lead.id =", lead.id));
|
|
473
|
-
readableStream.on("end", console.log("end"));
|
|
474
|
-
```
|
|
475
|
-
|
|
476
|
-
### Writable stream
|
|
477
|
-
|
|
478
|
-
```javascript
|
|
479
|
-
const writableStream = await resource.writable();
|
|
480
|
-
|
|
481
|
-
writableStream.write({
|
|
482
|
-
lead: {
|
|
483
|
-
fullName: "My Test Name",
|
|
484
|
-
personalEmail: `bulk-${k}@mymail.com`,
|
|
485
|
-
mobileNumber: "+55 11 1234567890",
|
|
486
|
-
},
|
|
487
|
-
});
|
|
488
|
-
```
|
|
129
|
+
### Resource Class
|
|
489
130
|
|
|
490
|
-
|
|
131
|
+
Represents a collection of documents in S3.
|
|
491
132
|
|
|
492
|
-
|
|
133
|
+
#### Methods
|
|
493
134
|
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
Each method has a **[:link:](https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html) link** to the official `aws-sdk` docs.
|
|
503
|
-
|
|
504
|
-
##### getObject [:link:](https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#getObject-property)
|
|
505
|
-
|
|
506
|
-
```javascript
|
|
507
|
-
const { Body, Metadata } = await client.getObject({
|
|
508
|
-
key: `my-prefixed-file.csv`,
|
|
509
|
-
});
|
|
510
|
-
|
|
511
|
-
// AWS.Response
|
|
512
|
-
```
|
|
513
|
-
|
|
514
|
-
##### putObject [:link:](https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#putObject-property)
|
|
515
|
-
|
|
516
|
-
```javascript
|
|
517
|
-
const response = await client.putObject({
|
|
518
|
-
key: `my-prefixed-file.csv`,
|
|
519
|
-
contentType: "text/csv",
|
|
520
|
-
metadata: { a: "1", b: "2", c: "3" },
|
|
521
|
-
body: "a;b;c\n1;2;3\n4;5;6",
|
|
522
|
-
});
|
|
523
|
-
|
|
524
|
-
// AWS.Response
|
|
525
|
-
```
|
|
526
|
-
|
|
527
|
-
##### headObject [:link:](https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#headObject-property)
|
|
528
|
-
|
|
529
|
-
```javascript
|
|
530
|
-
const { Metadata } = await client.headObject({
|
|
531
|
-
key: `my-prefixed-file.csv`,
|
|
532
|
-
});
|
|
533
|
-
|
|
534
|
-
// AWS.Response
|
|
535
|
-
```
|
|
536
|
-
|
|
537
|
-
##### deleteObject [:link:](https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#deleteObject-property)
|
|
538
|
-
|
|
539
|
-
```javascript
|
|
540
|
-
const response = await client.deleteObject({
|
|
541
|
-
key: `my-prefixed-file.csv`,
|
|
542
|
-
});
|
|
543
|
-
|
|
544
|
-
// AWS.Response
|
|
545
|
-
```
|
|
546
|
-
|
|
547
|
-
##### deleteObjects [:link:](https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#deleteObjects-property)
|
|
548
|
-
|
|
549
|
-
```javascript
|
|
550
|
-
const response = await client.deleteObjects({
|
|
551
|
-
keys: [`my-prefixed-file.csv`, `my-other-prefixed-file.csv`],
|
|
552
|
-
});
|
|
553
|
-
|
|
554
|
-
// AWS.Response
|
|
555
|
-
```
|
|
556
|
-
|
|
557
|
-
##### listObjects [:link:](https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#listObjects-property)
|
|
558
|
-
|
|
559
|
-
```javascript
|
|
560
|
-
const response = await client.listObjects({
|
|
561
|
-
prefix: `my-subdir`,
|
|
562
|
-
});
|
|
563
|
-
|
|
564
|
-
// AWS.Response
|
|
565
|
-
```
|
|
135
|
+
- `insert(data, options)`: Insert a single document
|
|
136
|
+
- `insertMany(data, options)`: Insert multiple documents
|
|
137
|
+
- `find(query, options)`: Find documents
|
|
138
|
+
- `findOne(query, options)`: Find a single document
|
|
139
|
+
- `update(query, data, options)`: Update documents
|
|
140
|
+
- `delete(query, options)`: Delete documents
|
|
141
|
+
- `createReadStream(query, options)`: Create a read stream
|
|
142
|
+
- `createWriteStream(options)`: Create a write stream
|
|
566
143
|
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
Custom made method to make it easier to count keys within a listObjects loop.
|
|
570
|
-
|
|
571
|
-
```javascript
|
|
572
|
-
const count = await client.count({
|
|
573
|
-
prefix: `my-subdir`,
|
|
574
|
-
});
|
|
575
|
-
|
|
576
|
-
// 10
|
|
577
|
-
```
|
|
578
|
-
|
|
579
|
-
##### getAllKeys
|
|
580
|
-
|
|
581
|
-
Custom made method to make it easier to return all keys in a subpath within a listObjects loop.
|
|
582
|
-
|
|
583
|
-
All returned keys will have the it's fullpath replaced with the current "scope" path.
|
|
584
|
-
|
|
585
|
-
```javascript
|
|
586
|
-
const keys = await client.getAllKeys({
|
|
587
|
-
prefix: `my-subdir`,
|
|
588
|
-
});
|
|
589
|
-
|
|
590
|
-
// [
|
|
591
|
-
// key1,
|
|
592
|
-
// key2,
|
|
593
|
-
// ...
|
|
594
|
-
// ]
|
|
595
|
-
```
|
|
596
|
-
|
|
597
|
-
---
|
|
598
|
-
|
|
599
|
-
## Events
|
|
600
|
-
|
|
601
|
-
The 3 main classes `S3db`, `Resource` and `S3Client` are extensions of Javascript's `EventEmitter`.
|
|
602
|
-
|
|
603
|
-
| S3Database | S3Client | S3Resource | S3Resource Readable Stream |
|
|
604
|
-
| ---------- | ------------- | ---------- | -------------------------- |
|
|
605
|
-
| error | error | error | error |
|
|
606
|
-
| connected | request | insert | id |
|
|
607
|
-
| | response | get | data |
|
|
608
|
-
| | response | update | |
|
|
609
|
-
| | getObject | delete | |
|
|
610
|
-
| | putObject | count | |
|
|
611
|
-
| | headObject | insertMany | |
|
|
612
|
-
| | deleteObject | deleteAll | |
|
|
613
|
-
| | deleteObjects | listIds | |
|
|
614
|
-
| | listObjects | getMany | |
|
|
615
|
-
| | count | getAll | |
|
|
616
|
-
| | getAllKeys | | |
|
|
617
|
-
|
|
618
|
-
### S3Database
|
|
619
|
-
|
|
620
|
-
#### error
|
|
621
|
-
|
|
622
|
-
```javascript
|
|
623
|
-
s3db.on("error", (error) => console.error(error));
|
|
624
|
-
```
|
|
625
|
-
|
|
626
|
-
#### connected
|
|
627
|
-
|
|
628
|
-
```javascript
|
|
629
|
-
s3db.on("connected", () => {});
|
|
630
|
-
```
|
|
631
|
-
|
|
632
|
-
### S3Client
|
|
633
|
-
|
|
634
|
-
Using this reference for the events:
|
|
635
|
-
|
|
636
|
-
```javascript
|
|
637
|
-
const client = s3db.client;
|
|
638
|
-
```
|
|
639
|
-
|
|
640
|
-
#### error
|
|
641
|
-
|
|
642
|
-
```javascript
|
|
643
|
-
client.on("error", (error) => console.error(error));
|
|
644
|
-
```
|
|
645
|
-
|
|
646
|
-
#### request
|
|
647
|
-
|
|
648
|
-
Emitted when a request is generated to AWS.
|
|
649
|
-
|
|
650
|
-
```javascript
|
|
651
|
-
client.on("request", (action, params) => {});
|
|
652
|
-
```
|
|
653
|
-
|
|
654
|
-
#### response
|
|
655
|
-
|
|
656
|
-
Emitted when a response is received from AWS.
|
|
657
|
-
|
|
658
|
-
```javascript
|
|
659
|
-
client.on("response", (action, params, response) => {});
|
|
660
|
-
```
|
|
661
|
-
|
|
662
|
-
#### getObject
|
|
663
|
-
|
|
664
|
-
```javascript
|
|
665
|
-
client.on("getObject", (options, response) => {});
|
|
666
|
-
```
|
|
667
|
-
|
|
668
|
-
#### putObject
|
|
669
|
-
|
|
670
|
-
```javascript
|
|
671
|
-
client.on("putObject", (options, response) => {});
|
|
672
|
-
```
|
|
673
|
-
|
|
674
|
-
#### headObject
|
|
675
|
-
|
|
676
|
-
```javascript
|
|
677
|
-
client.on("headObject", (options, response) => {});
|
|
678
|
-
```
|
|
679
|
-
|
|
680
|
-
#### deleteObject
|
|
681
|
-
|
|
682
|
-
```javascript
|
|
683
|
-
client.on("deleteObject", (options, response) => {});
|
|
684
|
-
```
|
|
685
|
-
|
|
686
|
-
#### deleteObjects
|
|
687
|
-
|
|
688
|
-
```javascript
|
|
689
|
-
client.on("deleteObjects", (options, response) => {});
|
|
690
|
-
```
|
|
691
|
-
|
|
692
|
-
#### listObjects
|
|
693
|
-
|
|
694
|
-
```javascript
|
|
695
|
-
client.on("listObjects", (options, response) => {});
|
|
696
|
-
```
|
|
697
|
-
|
|
698
|
-
#### count
|
|
699
|
-
|
|
700
|
-
```javascript
|
|
701
|
-
client.on("count", (options, response) => {});
|
|
702
|
-
```
|
|
703
|
-
|
|
704
|
-
#### getAllKeys
|
|
705
|
-
|
|
706
|
-
```javascript
|
|
707
|
-
client.on("getAllKeys", (options, response) => {});
|
|
708
|
-
```
|
|
709
|
-
|
|
710
|
-
### S3Resource
|
|
711
|
-
|
|
712
|
-
Using this reference for the events:
|
|
713
|
-
|
|
714
|
-
```javascript
|
|
715
|
-
const resource = s3db.resource("leads");
|
|
716
|
-
```
|
|
717
|
-
|
|
718
|
-
#### error
|
|
719
|
-
|
|
720
|
-
```javascript
|
|
721
|
-
resource.on("error", (err) => console.error(err));
|
|
722
|
-
```
|
|
723
|
-
|
|
724
|
-
#### insert
|
|
725
|
-
|
|
726
|
-
```javascript
|
|
727
|
-
resource.on("insert", (data) => {});
|
|
728
|
-
```
|
|
729
|
-
|
|
730
|
-
#### get
|
|
731
|
-
|
|
732
|
-
```javascript
|
|
733
|
-
resource.on("get", (data) => {});
|
|
734
|
-
```
|
|
735
|
-
|
|
736
|
-
#### update
|
|
737
|
-
|
|
738
|
-
```javascript
|
|
739
|
-
resource.on("update", (attrs, data) => {});
|
|
740
|
-
```
|
|
741
|
-
|
|
742
|
-
#### delete
|
|
743
|
-
|
|
744
|
-
```javascript
|
|
745
|
-
resource.on("delete", (id) => {});
|
|
746
|
-
```
|
|
747
|
-
|
|
748
|
-
#### count
|
|
749
|
-
|
|
750
|
-
```javascript
|
|
751
|
-
resource.on("count", (count) => {});
|
|
752
|
-
```
|
|
753
|
-
|
|
754
|
-
#### insertMany
|
|
755
|
-
|
|
756
|
-
```javascript
|
|
757
|
-
resource.on("insertMany", (count) => {});
|
|
758
|
-
```
|
|
759
|
-
|
|
760
|
-
#### getMany
|
|
761
|
-
|
|
762
|
-
```javascript
|
|
763
|
-
resource.on("getMany", (count) => {});
|
|
764
|
-
```
|
|
765
|
-
|
|
766
|
-
#### getAll
|
|
767
|
-
|
|
768
|
-
```javascript
|
|
769
|
-
resource.on("getAll", (count) => {});
|
|
770
|
-
```
|
|
771
|
-
|
|
772
|
-
#### deleteAll
|
|
773
|
-
|
|
774
|
-
```javascript
|
|
775
|
-
resource.on("deleteAll", (count) => {});
|
|
776
|
-
```
|
|
777
|
-
|
|
778
|
-
#### listIds
|
|
779
|
-
|
|
780
|
-
```javascript
|
|
781
|
-
resource.on("listIds", (count) => {});
|
|
782
|
-
```
|
|
783
|
-
|
|
784
|
-
---
|
|
785
|
-
|
|
786
|
-
## Plugins
|
|
787
|
-
|
|
788
|
-
Anatomy of a plugin:
|
|
789
|
-
|
|
790
|
-
```javascript
|
|
791
|
-
const MyPlugin = {
|
|
792
|
-
setup(s3db: S3db) {},
|
|
793
|
-
start() {},
|
|
794
|
-
};
|
|
795
|
-
```
|
|
796
|
-
|
|
797
|
-
We have an example of a _costs simulator plugin_ [here!](https://github.com/forattini-dev/s3db.js/blob/main/src/plugins/costs.plugin.js)
|
|
798
|
-
|
|
799
|
-
---
|
|
800
|
-
|
|
801
|
-
## Cost simulation
|
|
802
|
-
|
|
803
|
-
S3's pricing deep dive:
|
|
804
|
-
|
|
805
|
-
- Data volume [1 GB x 0.023 USD]: it relates to the total volume of storage used and requests volume but, in this implementation, we just upload `0 bytes` files.
|
|
806
|
-
- GET Requests [1,000 GET requests in a month x 0.0000004 USD per request = 0.0004 USD]: every read requests
|
|
807
|
-
- PUT Requests [1,000 PUT requests for S3 Standard Storage x 0.000005 USD per request = 0.005 USD]: every write request
|
|
808
|
-
- Data transfer [Internet: 1 GB x 0.09 USD per GB = 0.09 USD]:
|
|
809
|
-
|
|
810
|
-
Check by yourself the pricing page details at https://aws.amazon.com/s3/pricing/ and https://calculator.aws/#/addService/S3.
|
|
811
|
-
|
|
812
|
-
### Big example
|
|
813
|
-
|
|
814
|
-
Lets try to simulate a big project where you have a database with a few tables:
|
|
815
|
-
|
|
816
|
-
- pageviews: 100,000,000 lines of 100 bytes each
|
|
817
|
-
- leads: 1,000,000 lines of 200 bytes each
|
|
818
|
-
|
|
819
|
-
```javascript
|
|
820
|
-
const Fakerator = require("fakerator");
|
|
821
|
-
const fake = Fakerator("pt-BR");
|
|
822
|
-
|
|
823
|
-
const pageview = {
|
|
824
|
-
ip: this.faker.internet.ip(),
|
|
825
|
-
domain: this.faker.internet.url(),
|
|
826
|
-
path: this.faker.internet.url(),
|
|
827
|
-
query: `?q=${this.faker.lorem.word()}`,
|
|
828
|
-
};
|
|
829
|
-
|
|
830
|
-
const lead = {
|
|
831
|
-
name: fake.names.name(),
|
|
832
|
-
mobile: fake.phone.number(),
|
|
833
|
-
email: fake.internet.email(),
|
|
834
|
-
country: "Brazil",
|
|
835
|
-
city: fake.address.city(),
|
|
836
|
-
state: fake.address.countryCode(),
|
|
837
|
-
address: fake.address.street(),
|
|
838
|
-
};
|
|
839
|
-
```
|
|
840
|
-
|
|
841
|
-
If you write the whole database of:
|
|
842
|
-
|
|
843
|
-
- pageviews:
|
|
844
|
-
- 100,000,000 PUT requests for S3 Standard Storage x 0.000005 USD per request = 500.00 USD (S3 Standard PUT requests cost)
|
|
845
|
-
- leads:
|
|
846
|
-
- 1,000,000 PUT requests for S3 Standard Storage x 0.000005 USD per request = 5.00 USD (S3 Standard PUT requests cost)
|
|
847
|
-
|
|
848
|
-
It will cost 505.00 USD, once.
|
|
849
|
-
|
|
850
|
-
If you want to read the whole database:
|
|
851
|
-
|
|
852
|
-
- pageviews:
|
|
853
|
-
- 100,000,000 GET requests in a month x 0.0000004 USD per request = 40.00 USD (S3 Standard GET requests cost)
|
|
854
|
-
- (100,000,000 × 100 bytes)÷(1024×1000×1000) ≅ 10 Gb
|
|
855
|
-
Internet: 10 GB x 0.09 USD per GB = 0.90 USD
|
|
856
|
-
- leads:
|
|
857
|
-
- 1,000,000 GET requests in a month x 0.0000004 USD per request = 0.40 USD (S3 Standard GET requests cost)
|
|
858
|
-
- (1,000,000 × 200 bytes)÷(1024×1000×1000) ≅ 0.19 Gb
|
|
859
|
-
Internet: 1 GB x 0.09 USD per GB = 0.09 USD
|
|
860
|
-
|
|
861
|
-
It will cost 41.39 USD, once.
|
|
862
|
-
|
|
863
|
-
### Small example
|
|
864
|
-
|
|
865
|
-
Lets save some JWT tokens using the [RFC:7519](https://www.rfc-editor.org/rfc/rfc7519.html).
|
|
866
|
-
|
|
867
|
-
```javascript
|
|
868
|
-
await s3db.createResource({
|
|
869
|
-
name: "tokens",
|
|
870
|
-
attributes: {
|
|
871
|
-
iss: 'url|max:256',
|
|
872
|
-
sub: 'string',
|
|
873
|
-
aud: 'string',
|
|
874
|
-
exp: 'number',
|
|
875
|
-
email: 'email',
|
|
876
|
-
name: 'string',
|
|
877
|
-
scope: 'string',
|
|
878
|
-
email_verified: 'boolean',
|
|
879
|
-
})
|
|
880
|
-
|
|
881
|
-
function generateToken () {
|
|
882
|
-
const token = createTokenLib(...)
|
|
883
|
-
|
|
884
|
-
await resource.insert({
|
|
885
|
-
id: token.jti || md5(token)
|
|
886
|
-
...token,
|
|
887
|
-
})
|
|
888
|
-
|
|
889
|
-
return token
|
|
890
|
-
}
|
|
891
|
-
|
|
892
|
-
function validateToken (token) {
|
|
893
|
-
const id = token.jti || md5(token)
|
|
894
|
-
|
|
895
|
-
if (!validateTokenSignature(token, ...)) {
|
|
896
|
-
await resource.deleteById(id)
|
|
897
|
-
throw new Error('invalid-token')
|
|
898
|
-
}
|
|
899
|
-
|
|
900
|
-
return resource.getById(id)
|
|
901
|
-
}
|
|
902
|
-
```
|
|
144
|
+
## Examples
|
|
903
145
|
|
|
904
|
-
|
|
146
|
+
See the [examples](./examples) directory for more detailed usage examples.
|
|
905
147
|
|
|
906
|
-
|
|
148
|
+
## License
|
|
907
149
|
|
|
908
|
-
|
|
150
|
+
UNLICENSED
|