s3db.js 6.2.0 → 7.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/PLUGINS.md +2724 -0
- package/README.md +372 -469
- package/UNLICENSE +24 -0
- package/dist/s3db.cjs.js +30057 -18387
- package/dist/s3db.cjs.min.js +1 -1
- package/dist/s3db.d.ts +373 -72
- package/dist/s3db.es.js +30043 -18384
- package/dist/s3db.es.min.js +1 -1
- package/dist/s3db.iife.js +29730 -18061
- package/dist/s3db.iife.min.js +1 -1
- package/package.json +44 -69
- package/src/behaviors/body-only.js +110 -0
- package/src/behaviors/body-overflow.js +153 -0
- package/src/behaviors/enforce-limits.js +195 -0
- package/src/behaviors/index.js +39 -0
- package/src/behaviors/truncate-data.js +204 -0
- package/src/behaviors/user-managed.js +147 -0
- package/src/client.class.js +515 -0
- package/src/concerns/base62.js +61 -0
- package/src/concerns/calculator.js +204 -0
- package/src/concerns/crypto.js +142 -0
- package/src/concerns/id.js +8 -0
- package/src/concerns/index.js +5 -0
- package/src/concerns/try-fn.js +151 -0
- package/src/connection-string.class.js +75 -0
- package/src/database.class.js +599 -0
- package/src/errors.js +261 -0
- package/src/index.js +17 -0
- package/src/plugins/audit.plugin.js +442 -0
- package/src/plugins/cache/cache.class.js +53 -0
- package/src/plugins/cache/index.js +6 -0
- package/src/plugins/cache/memory-cache.class.js +164 -0
- package/src/plugins/cache/s3-cache.class.js +189 -0
- package/src/plugins/cache.plugin.js +275 -0
- package/src/plugins/consumers/index.js +24 -0
- package/src/plugins/consumers/rabbitmq-consumer.js +56 -0
- package/src/plugins/consumers/sqs-consumer.js +102 -0
- package/src/plugins/costs.plugin.js +81 -0
- package/src/plugins/fulltext.plugin.js +473 -0
- package/src/plugins/index.js +12 -0
- package/src/plugins/metrics.plugin.js +603 -0
- package/src/plugins/plugin.class.js +210 -0
- package/src/plugins/plugin.obj.js +13 -0
- package/src/plugins/queue-consumer.plugin.js +134 -0
- package/src/plugins/replicator.plugin.js +769 -0
- package/src/plugins/replicators/base-replicator.class.js +85 -0
- package/src/plugins/replicators/bigquery-replicator.class.js +328 -0
- package/src/plugins/replicators/index.js +44 -0
- package/src/plugins/replicators/postgres-replicator.class.js +427 -0
- package/src/plugins/replicators/s3db-replicator.class.js +352 -0
- package/src/plugins/replicators/sqs-replicator.class.js +427 -0
- package/src/resource.class.js +2626 -0
- package/src/s3db.d.ts +1263 -0
- package/src/schema.class.js +706 -0
- package/src/stream/index.js +16 -0
- package/src/stream/resource-ids-page-reader.class.js +10 -0
- package/src/stream/resource-ids-reader.class.js +63 -0
- package/src/stream/resource-reader.class.js +81 -0
- package/src/stream/resource-writer.class.js +92 -0
- package/src/validator.class.js +97 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "s3db.js",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "7.0.0",
|
|
4
4
|
"description": "Use AWS S3, the world's most reliable document storage, as a database with this ORM.",
|
|
5
5
|
"main": "dist/s3db.cjs.js",
|
|
6
6
|
"module": "dist/s3db.es.js",
|
|
@@ -8,19 +8,7 @@
|
|
|
8
8
|
"types": "dist/s3db.d.ts",
|
|
9
9
|
"unpkg": "dist/s3db.iife.min.js",
|
|
10
10
|
"jsdelivr": "dist/s3db.iife.min.js",
|
|
11
|
-
"
|
|
12
|
-
"exports": {
|
|
13
|
-
".": {
|
|
14
|
-
"import": "./dist/s3db.es.js",
|
|
15
|
-
"require": "./dist/s3db.cjs.js",
|
|
16
|
-
"browser": "./dist/s3db.iife.js",
|
|
17
|
-
"default": "./dist/s3db.es.js"
|
|
18
|
-
}
|
|
19
|
-
},
|
|
20
|
-
"files": [
|
|
21
|
-
"dist"
|
|
22
|
-
],
|
|
23
|
-
"author": "forattini-dev",
|
|
11
|
+
"author": "@stone/martech",
|
|
24
12
|
"license": "UNLICENSED",
|
|
25
13
|
"repository": {
|
|
26
14
|
"type": "git",
|
|
@@ -35,68 +23,55 @@
|
|
|
35
23
|
"aws",
|
|
36
24
|
"database"
|
|
37
25
|
],
|
|
26
|
+
"type": "module",
|
|
27
|
+
"exports": {
|
|
28
|
+
".": {
|
|
29
|
+
"import": "./dist/s3db.es.js",
|
|
30
|
+
"require": "./dist/s3db.cjs.js",
|
|
31
|
+
"types": "./dist/s3db.d.ts"
|
|
32
|
+
}
|
|
33
|
+
},
|
|
34
|
+
"files": [
|
|
35
|
+
"dist",
|
|
36
|
+
"src",
|
|
37
|
+
"README.md",
|
|
38
|
+
"PLUGINS.md",
|
|
39
|
+
"UNLICENSE"
|
|
40
|
+
],
|
|
41
|
+
"peerDependencies": {},
|
|
38
42
|
"dependencies": {
|
|
39
|
-
"@aws-sdk/client-s3": "^3.
|
|
43
|
+
"@aws-sdk/client-s3": "^3.691.0",
|
|
40
44
|
"@supercharge/promise-pool": "^3.2.0",
|
|
41
|
-
"
|
|
42
|
-
"
|
|
43
|
-
"
|
|
44
|
-
"json-stable-stringify": "^1.3.0",
|
|
45
|
+
"fastest-validator": "^1.19.0",
|
|
46
|
+
"hash-wasm": "^4.14.0",
|
|
47
|
+
"json-stable-stringify": "^1.1.1",
|
|
45
48
|
"lodash-es": "^4.17.21",
|
|
46
|
-
"nanoid": "5.
|
|
47
|
-
"zlib": "^1.0.5"
|
|
48
|
-
},
|
|
49
|
-
"peerDependencies": {
|
|
50
|
-
"@aws-sdk/client-sqs": "^3.0.0",
|
|
51
|
-
"@google-cloud/bigquery": "^7.0.0",
|
|
52
|
-
"pg": "^8.0.0",
|
|
53
|
-
"uuid": "^9.0.0"
|
|
54
|
-
},
|
|
55
|
-
"peerDependenciesMeta": {
|
|
56
|
-
"@aws-sdk/client-sqs": {
|
|
57
|
-
"optional": true
|
|
58
|
-
},
|
|
59
|
-
"@google-cloud/bigquery": {
|
|
60
|
-
"optional": true
|
|
61
|
-
},
|
|
62
|
-
"pg": {
|
|
63
|
-
"optional": true
|
|
64
|
-
},
|
|
65
|
-
"uuid": {
|
|
66
|
-
"optional": true
|
|
67
|
-
}
|
|
49
|
+
"nanoid": "^5.0.9"
|
|
68
50
|
},
|
|
69
51
|
"devDependencies": {
|
|
70
|
-
"@
|
|
71
|
-
"@
|
|
72
|
-
"
|
|
73
|
-
"
|
|
74
|
-
"
|
|
75
|
-
"
|
|
76
|
-
"
|
|
77
|
-
"babel-jest": "^30.0.4",
|
|
78
|
-
"cliui": "^9.0.1",
|
|
79
|
-
"coveralls": "^3.1.1",
|
|
80
|
-
"dotenv": "^17.0.1",
|
|
81
|
-
"esbuild": "^0.25.6",
|
|
82
|
-
"fakerator": "^0.3.6",
|
|
83
|
-
"jest": "^30.0.4",
|
|
84
|
-
"multi-progress": "^4.0.0",
|
|
85
|
-
"progress": "^2.0.3",
|
|
86
|
-
"rollup": "^4.44.2",
|
|
87
|
-
"rollup-plugin-esbuild": "^6.2.1",
|
|
88
|
-
"rollup-plugin-polyfill-node": "^0.13.0",
|
|
89
|
-
"uuid": "^11.1.0"
|
|
52
|
+
"@rollup/plugin-replace": "^6.0.1",
|
|
53
|
+
"@types/node": "24.0.14",
|
|
54
|
+
"jest": "^29.7.0",
|
|
55
|
+
"rollup": "^4.27.4",
|
|
56
|
+
"rollup-plugin-copy": "^3.5.0",
|
|
57
|
+
"rollup-plugin-terser": "^7.0.2",
|
|
58
|
+
"typescript": "5.8.3"
|
|
90
59
|
},
|
|
60
|
+
"funding": [
|
|
61
|
+
"https://github.com/sponsors/forattini-dev"
|
|
62
|
+
],
|
|
91
63
|
"scripts": {
|
|
92
64
|
"build": "rollup -c",
|
|
93
|
-
"
|
|
94
|
-
"
|
|
95
|
-
"test
|
|
96
|
-
"
|
|
97
|
-
"
|
|
98
|
-
"test": "node --no-warnings --experimental-vm-modules node_modules/jest/bin/jest.js",
|
|
99
|
-
"test:
|
|
100
|
-
"test:
|
|
65
|
+
"dev": "rollup -c -w",
|
|
66
|
+
"test": "npm run test:js && npm run test:ts",
|
|
67
|
+
"test:js": "node --no-warnings --experimental-vm-modules node_modules/jest/bin/jest.js",
|
|
68
|
+
"test:ts": "tsc --noEmit --project tests/typescript/tsconfig.json",
|
|
69
|
+
"test:js-converage": "node --no-warnings --experimental-vm-modules node_modules/jest/bin/jest.js --detectOpenHandles --coverage",
|
|
70
|
+
"test:js-ai": "node --no-warnings --experimental-vm-modules node_modules/jest/bin/jest.js --detectOpenHandles --runInBand",
|
|
71
|
+
"test:types": "tsc --noEmit --project tests/typescript/tsconfig.json",
|
|
72
|
+
"test:types:basic": "tsc --noEmit tests/typescript/basic-usage.test.ts",
|
|
73
|
+
"test:types:direct": "tsc --noEmit tests/typescript/direct-type-test.ts",
|
|
74
|
+
"test:types:watch": "tsc --noEmit --watch --project tests/typescript/tsconfig.json",
|
|
75
|
+
"validate:types": "npm run test:types && echo 'TypeScript definitions are valid!'"
|
|
101
76
|
}
|
|
102
77
|
}
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
import { calculateTotalSize } from '../concerns/calculator.js';
|
|
2
|
+
import { tryFn, tryFnSync } from '../concerns/try-fn.js';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Body Only Behavior Configuration Documentation
|
|
6
|
+
*
|
|
7
|
+
* The `body-only` behavior stores all data in the S3 object body as JSON, keeping only
|
|
8
|
+
* the version field (`_v`) in metadata. This allows for unlimited data size since S3
|
|
9
|
+
* objects can be up to 5TB, but requires reading the full object body for any operation.
|
|
10
|
+
*
|
|
11
|
+
* ## Purpose & Use Cases
|
|
12
|
+
* - For large objects that exceed S3 metadata limits
|
|
13
|
+
* - When you need to store complex nested data structures
|
|
14
|
+
* - For objects that will be read infrequently (higher latency)
|
|
15
|
+
* - When you want to avoid metadata size constraints entirely
|
|
16
|
+
*
|
|
17
|
+
* ## How It Works
|
|
18
|
+
* - Keeps only the `_v` (version) field in S3 metadata
|
|
19
|
+
* - Serializes all other data as JSON in the object body
|
|
20
|
+
* - Requires full object read for any data access
|
|
21
|
+
* - No size limits on data (only S3 object size limit of 5TB)
|
|
22
|
+
*
|
|
23
|
+
* ## Performance Considerations
|
|
24
|
+
* - Higher latency for read operations (requires full object download)
|
|
25
|
+
* - Higher bandwidth usage for read operations
|
|
26
|
+
* - No metadata-based filtering or querying possible
|
|
27
|
+
* - Best for large, infrequently accessed data
|
|
28
|
+
*
|
|
29
|
+
* @example
|
|
30
|
+
* // Create a resource with body-only behavior
|
|
31
|
+
* const resource = await db.createResource({
|
|
32
|
+
* name: 'large_documents',
|
|
33
|
+
* attributes: { ... },
|
|
34
|
+
* behavior: 'body-only'
|
|
35
|
+
* });
|
|
36
|
+
*
|
|
37
|
+
* // All data goes to body, only _v stays in metadata
|
|
38
|
+
* const doc = await resource.insert({
|
|
39
|
+
* title: 'Large Document',
|
|
40
|
+
* content: 'Very long content...',
|
|
41
|
+
* metadata: { ... }
|
|
42
|
+
* });
|
|
43
|
+
*
|
|
44
|
+
* ## Comparison to Other Behaviors
|
|
45
|
+
* | Behavior | Metadata Usage | Body Usage | Size Limits | Performance |
|
|
46
|
+
* |------------------|----------------|------------|-------------|-------------|
|
|
47
|
+
* | body-only | Minimal (_v) | All data | 5TB | Slower reads |
|
|
48
|
+
* | body-overflow | Optimized | Overflow | 2KB metadata | Balanced |
|
|
49
|
+
* | truncate-data | All (truncated)| None | 2KB metadata | Fast reads |
|
|
50
|
+
* | enforce-limits | All (limited) | None | 2KB metadata | Fast reads |
|
|
51
|
+
* | user-managed | All (unlimited)| None | S3 limit | Fast reads |
|
|
52
|
+
*
|
|
53
|
+
* @typedef {Object} BodyOnlyBehaviorConfig
|
|
54
|
+
* @property {boolean} [enabled=true] - Whether the behavior is active
|
|
55
|
+
*/
|
|
56
|
+
export async function handleInsert({ resource, data, mappedData }) {
|
|
57
|
+
// Keep only the version field in metadata
|
|
58
|
+
const metadataOnly = {
|
|
59
|
+
'_v': mappedData._v || String(resource.version)
|
|
60
|
+
};
|
|
61
|
+
metadataOnly._map = JSON.stringify(resource.schema.map);
|
|
62
|
+
|
|
63
|
+
// Use o objeto original para o body
|
|
64
|
+
const body = JSON.stringify(mappedData);
|
|
65
|
+
|
|
66
|
+
return { mappedData: metadataOnly, body };
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
export async function handleUpdate({ resource, id, data, mappedData }) {
|
|
70
|
+
// For updates, we need to merge with existing data
|
|
71
|
+
// Since we can't easily read the existing body during update,
|
|
72
|
+
// we'll put the update data in the body and let the resource handle merging
|
|
73
|
+
|
|
74
|
+
// Keep only the version field in metadata
|
|
75
|
+
const metadataOnly = {
|
|
76
|
+
'_v': mappedData._v || String(resource.version)
|
|
77
|
+
};
|
|
78
|
+
metadataOnly._map = JSON.stringify(resource.schema.map);
|
|
79
|
+
|
|
80
|
+
// Use o objeto original para o body
|
|
81
|
+
const body = JSON.stringify(mappedData);
|
|
82
|
+
|
|
83
|
+
return { mappedData: metadataOnly, body };
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
export async function handleUpsert({ resource, id, data, mappedData }) {
|
|
87
|
+
// Same as insert for body-only behavior
|
|
88
|
+
return handleInsert({ resource, data, mappedData });
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
export async function handleGet({ resource, metadata, body }) {
|
|
92
|
+
// Parse the body to get the actual data
|
|
93
|
+
let bodyData = {};
|
|
94
|
+
if (body && body.trim() !== '') {
|
|
95
|
+
const [ok, err, parsed] = tryFnSync(() => JSON.parse(body));
|
|
96
|
+
if (ok) {
|
|
97
|
+
bodyData = parsed;
|
|
98
|
+
} else {
|
|
99
|
+
bodyData = {};
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
// Merge metadata (which contains _v) with body data
|
|
104
|
+
const mergedData = {
|
|
105
|
+
...bodyData,
|
|
106
|
+
...metadata // metadata contains _v
|
|
107
|
+
};
|
|
108
|
+
|
|
109
|
+
return { metadata: mergedData, body };
|
|
110
|
+
}
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
import { calculateTotalSize, calculateAttributeSizes, calculateUTF8Bytes } from '../concerns/calculator.js';
|
|
2
|
+
import { calculateEffectiveLimit } from '../concerns/calculator.js';
|
|
3
|
+
import { S3_METADATA_LIMIT_BYTES } from './enforce-limits.js';
|
|
4
|
+
import { tryFn, tryFnSync } from '../concerns/try-fn.js';
|
|
5
|
+
|
|
6
|
+
const OVERFLOW_FLAG = '$overflow';
|
|
7
|
+
const OVERFLOW_FLAG_VALUE = 'true';
|
|
8
|
+
const OVERFLOW_FLAG_BYTES = calculateUTF8Bytes(OVERFLOW_FLAG) + calculateUTF8Bytes(OVERFLOW_FLAG_VALUE);
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Body Overflow Behavior Configuration Documentation
|
|
12
|
+
*
|
|
13
|
+
* The `body-overflow` behavior optimizes metadata usage by sorting attributes by size
|
|
14
|
+
* in ascending order and placing as many small attributes as possible in metadata,
|
|
15
|
+
* while moving larger attributes to the S3 object body. This maximizes metadata
|
|
16
|
+
* utilization while keeping frequently accessed small fields in metadata for fast access.
|
|
17
|
+
*
|
|
18
|
+
* ## Purpose & Use Cases
|
|
19
|
+
* - For objects with mixed field sizes (some small, some large)
|
|
20
|
+
* - When you want to optimize for both metadata efficiency and read performance
|
|
21
|
+
* - For objects that exceed metadata limits but have important small fields
|
|
22
|
+
* - When you need fast access to frequently used small fields
|
|
23
|
+
*
|
|
24
|
+
* ## How It Works
|
|
25
|
+
* 1. Calculates the size of each attribute
|
|
26
|
+
* 2. Sorts attributes by size in ascending order (smallest first)
|
|
27
|
+
* 3. Fills metadata with small attributes until limit is reached
|
|
28
|
+
* 4. Places remaining (larger) attributes in the object body as JSON
|
|
29
|
+
* 5. Adds a `$overflow` flag to metadata to indicate body usage
|
|
30
|
+
*
|
|
31
|
+
* ## Performance Characteristics
|
|
32
|
+
* - Fast access to small fields (in metadata)
|
|
33
|
+
* - Slower access to large fields (requires body read)
|
|
34
|
+
* - Optimized metadata utilization
|
|
35
|
+
* - Balanced approach between performance and size efficiency
|
|
36
|
+
*
|
|
37
|
+
* @example
|
|
38
|
+
* // Create a resource with body-overflow behavior
|
|
39
|
+
* const resource = await db.createResource({
|
|
40
|
+
* name: 'mixed_content',
|
|
41
|
+
* attributes: { ... },
|
|
42
|
+
* behavior: 'body-overflow'
|
|
43
|
+
* });
|
|
44
|
+
*
|
|
45
|
+
* // Small fields go to metadata, large fields go to body
|
|
46
|
+
* const doc = await resource.insert({
|
|
47
|
+
* id: 'doc123', // Small -> metadata
|
|
48
|
+
* title: 'Short Title', // Small -> metadata
|
|
49
|
+
* content: 'Very long...', // Large -> body
|
|
50
|
+
* metadata: { ... } // Large -> body
|
|
51
|
+
* });
|
|
52
|
+
*
|
|
53
|
+
* ## Comparison to Other Behaviors
|
|
54
|
+
* | Behavior | Metadata Usage | Body Usage | Size Limits | Performance |
|
|
55
|
+
* |------------------|----------------|------------|-------------|-------------|
|
|
56
|
+
* | body-overflow | Optimized | Overflow | 2KB metadata | Balanced |
|
|
57
|
+
* | body-only | Minimal (_v) | All data | 5TB | Slower reads |
|
|
58
|
+
* | truncate-data | All (truncated)| None | 2KB metadata | Fast reads |
|
|
59
|
+
* | enforce-limits | All (limited) | None | 2KB metadata | Fast reads |
|
|
60
|
+
* | user-managed | All (unlimited)| None | S3 limit | Fast reads |
|
|
61
|
+
*
|
|
62
|
+
* @typedef {Object} BodyOverflowBehaviorConfig
|
|
63
|
+
* @property {boolean} [enabled=true] - Whether the behavior is active
|
|
64
|
+
* @property {number} [metadataReserve=50] - Reserve bytes for system fields
|
|
65
|
+
* @property {string[]} [priorityFields] - Fields that should be prioritized in metadata
|
|
66
|
+
* @property {boolean} [preserveOrder=false] - Whether to preserve original field order
|
|
67
|
+
*/
|
|
68
|
+
export async function handleInsert({ resource, data, mappedData, originalData }) {
|
|
69
|
+
const effectiveLimit = calculateEffectiveLimit({
|
|
70
|
+
s3Limit: S3_METADATA_LIMIT_BYTES,
|
|
71
|
+
systemConfig: {
|
|
72
|
+
version: resource.version,
|
|
73
|
+
timestamps: resource.config.timestamps,
|
|
74
|
+
id: data.id
|
|
75
|
+
}
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
const attributeSizes = calculateAttributeSizes(mappedData);
|
|
79
|
+
const sortedFields = Object.entries(attributeSizes)
|
|
80
|
+
.sort(([, a], [, b]) => a - b);
|
|
81
|
+
|
|
82
|
+
const metadataFields = {};
|
|
83
|
+
const bodyFields = {};
|
|
84
|
+
let currentSize = 0;
|
|
85
|
+
let willOverflow = false;
|
|
86
|
+
|
|
87
|
+
// Always include version field first
|
|
88
|
+
if (mappedData._v) {
|
|
89
|
+
metadataFields._v = mappedData._v;
|
|
90
|
+
currentSize += attributeSizes._v;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
// Reserve space for $overflow if overflow is possible
|
|
94
|
+
let reservedLimit = effectiveLimit;
|
|
95
|
+
for (const [fieldName, size] of sortedFields) {
|
|
96
|
+
if (fieldName === '_v') continue;
|
|
97
|
+
if (!willOverflow && (currentSize + size > effectiveLimit)) {
|
|
98
|
+
reservedLimit -= OVERFLOW_FLAG_BYTES;
|
|
99
|
+
willOverflow = true;
|
|
100
|
+
}
|
|
101
|
+
if (!willOverflow && (currentSize + size <= reservedLimit)) {
|
|
102
|
+
metadataFields[fieldName] = mappedData[fieldName];
|
|
103
|
+
currentSize += size;
|
|
104
|
+
} else {
|
|
105
|
+
bodyFields[fieldName] = mappedData[fieldName];
|
|
106
|
+
willOverflow = true;
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
if (willOverflow) {
|
|
111
|
+
metadataFields[OVERFLOW_FLAG] = OVERFLOW_FLAG_VALUE;
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
const hasOverflow = Object.keys(bodyFields).length > 0;
|
|
115
|
+
let body = hasOverflow ? JSON.stringify(bodyFields) : "";
|
|
116
|
+
if (!hasOverflow) body = '{}';
|
|
117
|
+
|
|
118
|
+
// FIX: Only return metadataFields as mappedData, not full mappedData
|
|
119
|
+
return { mappedData: metadataFields, body };
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
export async function handleUpdate({ resource, id, data, mappedData, originalData }) {
|
|
123
|
+
// For updates, use the same logic as insert (split fields by size)
|
|
124
|
+
return handleInsert({ resource, data, mappedData, originalData });
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
export async function handleUpsert({ resource, id, data, mappedData }) {
|
|
128
|
+
return handleInsert({ resource, data, mappedData });
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
export async function handleGet({ resource, metadata, body }) {
|
|
132
|
+
// Parse body content if it exists
|
|
133
|
+
let bodyData = {};
|
|
134
|
+
if (body && body.trim() !== '') {
|
|
135
|
+
const [ok, err, parsed] = tryFnSync(() => JSON.parse(body));
|
|
136
|
+
if (ok) {
|
|
137
|
+
bodyData = parsed;
|
|
138
|
+
} else {
|
|
139
|
+
bodyData = {};
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
// Merge metadata and body data, with metadata taking precedence
|
|
144
|
+
const mergedData = {
|
|
145
|
+
...bodyData,
|
|
146
|
+
...metadata
|
|
147
|
+
};
|
|
148
|
+
|
|
149
|
+
// Remove internal flags from the merged result
|
|
150
|
+
delete mergedData.$overflow;
|
|
151
|
+
|
|
152
|
+
return { metadata: mergedData, body };
|
|
153
|
+
}
|
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
import { calculateTotalSize } from '../concerns/calculator.js';
|
|
2
|
+
import { calculateEffectiveLimit } from '../concerns/calculator.js';
|
|
3
|
+
|
|
4
|
+
export const S3_METADATA_LIMIT_BYTES = 2047;
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Enforce Limits Behavior Configuration Documentation
|
|
8
|
+
*
|
|
9
|
+
* This behavior enforces various limits on data operations to prevent abuse and ensure
|
|
10
|
+
* system stability. It can limit body size, metadata size, and other resource constraints.
|
|
11
|
+
*
|
|
12
|
+
* @typedef {Object} EnforceLimitsBehaviorConfig
|
|
13
|
+
* @property {boolean} [enabled=true] - Whether the behavior is active
|
|
14
|
+
* @property {number} [maxBodySize=1024*1024] - Maximum body size in bytes (1MB default)
|
|
15
|
+
* @property {number} [maxMetadataSize=2048] - Maximum metadata size in bytes (2KB default)
|
|
16
|
+
* @property {number} [maxKeySize=1024] - Maximum key size in bytes (1KB default)
|
|
17
|
+
* @property {number} [maxValueSize=1024*1024] - Maximum value size in bytes (1MB default)
|
|
18
|
+
* @property {number} [maxFields=100] - Maximum number of fields in a single object
|
|
19
|
+
* @property {number} [maxNestingDepth=10] - Maximum nesting depth for objects and arrays
|
|
20
|
+
* @property {number} [maxArrayLength=1000] - Maximum length for arrays
|
|
21
|
+
* @property {number} [maxStringLength=10000] - Maximum length for string values
|
|
22
|
+
* @property {number} [maxNumberValue=Number.MAX_SAFE_INTEGER] - Maximum numeric value
|
|
23
|
+
* @property {number} [minNumberValue=Number.MIN_SAFE_INTEGER] - Minimum numeric value
|
|
24
|
+
* @property {string} [enforcementMode='strict'] - Enforcement mode: 'strict', 'warn', 'soft'
|
|
25
|
+
* @property {boolean} [logViolations=true] - Whether to log limit violations
|
|
26
|
+
* @property {boolean} [throwOnViolation=true] - Whether to throw errors on limit violations
|
|
27
|
+
* @property {Function} [customValidator] - Custom function to validate data against limits
|
|
28
|
+
* - Parameters: (data: any, limits: Object, context: Object) => boolean
|
|
29
|
+
* - Return: true if valid, false if invalid
|
|
30
|
+
* @property {Object.<string, number>} [fieldLimits] - Field-specific size limits
|
|
31
|
+
* - Key: field name (e.g., 'content', 'description')
|
|
32
|
+
* - Value: maximum size in bytes
|
|
33
|
+
* @property {string[]} [excludeFields] - Array of field names to exclude from limit enforcement
|
|
34
|
+
* @property {string[]} [includeFields] - Array of field names to include in limit enforcement
|
|
35
|
+
* @property {boolean} [applyToInsert=true] - Whether to apply limits to insert operations
|
|
36
|
+
* @property {boolean} [applyToUpdate=true] - Whether to apply limits to update operations
|
|
37
|
+
* @property {boolean} [applyToUpsert=true] - Whether to apply limits to upsert operations
|
|
38
|
+
* @property {boolean} [applyToRead=false] - Whether to apply limits to read operations
|
|
39
|
+
* @property {number} [warningThreshold=0.8] - Percentage of limit to trigger warnings (0.8 = 80%)
|
|
40
|
+
* @property {Object} [context] - Additional context for custom functions
|
|
41
|
+
* @property {boolean} [validateMetadata=true] - Whether to validate metadata size
|
|
42
|
+
* @property {boolean} [validateBody=true] - Whether to validate body size
|
|
43
|
+
* @property {boolean} [validateKeys=true] - Whether to validate key sizes
|
|
44
|
+
* @property {boolean} [validateValues=true] - Whether to validate value sizes
|
|
45
|
+
*
|
|
46
|
+
* @example
|
|
47
|
+
* // Basic configuration with standard limits
|
|
48
|
+
* {
|
|
49
|
+
* enabled: true,
|
|
50
|
+
* maxBodySize: 2 * 1024 * 1024, // 2MB
|
|
51
|
+
* maxMetadataSize: 4096, // 4KB
|
|
52
|
+
* maxFields: 200,
|
|
53
|
+
* enforcementMode: 'strict',
|
|
54
|
+
* logViolations: true
|
|
55
|
+
* }
|
|
56
|
+
*
|
|
57
|
+
* @example
|
|
58
|
+
* // Configuration with field-specific limits
|
|
59
|
+
* {
|
|
60
|
+
* enabled: true,
|
|
61
|
+
* fieldLimits: {
|
|
62
|
+
* 'content': 5 * 1024 * 1024, // 5MB for content
|
|
63
|
+
* 'description': 1024 * 1024, // 1MB for description
|
|
64
|
+
* 'title': 1024, // 1KB for title
|
|
65
|
+
* 'tags': 512 // 512B for tags
|
|
66
|
+
* },
|
|
67
|
+
* excludeFields: ['id', 'created_at', 'updated_at'],
|
|
68
|
+
* enforcementMode: 'warn',
|
|
69
|
+
* warningThreshold: 0.7
|
|
70
|
+
* }
|
|
71
|
+
*
|
|
72
|
+
* @example
|
|
73
|
+
* // Configuration with custom validation
|
|
74
|
+
* {
|
|
75
|
+
* enabled: true,
|
|
76
|
+
* maxBodySize: 1024 * 1024, // 1MB
|
|
77
|
+
* customValidator: (data, limits, context) => {
|
|
78
|
+
* // Custom validation logic
|
|
79
|
+
* if (data.content && data.content.length > limits.maxBodySize) {
|
|
80
|
+
* return false;
|
|
81
|
+
* }
|
|
82
|
+
* return true;
|
|
83
|
+
* },
|
|
84
|
+
* context: {
|
|
85
|
+
* environment: 'production',
|
|
86
|
+
* userRole: 'admin'
|
|
87
|
+
* },
|
|
88
|
+
* enforcementMode: 'soft',
|
|
89
|
+
* logViolations: true
|
|
90
|
+
* }
|
|
91
|
+
*
|
|
92
|
+
* @example
|
|
93
|
+
* // Configuration with strict limits for API endpoints
|
|
94
|
+
* {
|
|
95
|
+
* enabled: true,
|
|
96
|
+
* maxBodySize: 512 * 1024, // 512KB
|
|
97
|
+
* maxMetadataSize: 1024, // 1KB
|
|
98
|
+
* maxFields: 50,
|
|
99
|
+
* maxNestingDepth: 5,
|
|
100
|
+
* maxArrayLength: 100,
|
|
101
|
+
* maxStringLength: 5000,
|
|
102
|
+
* enforcementMode: 'strict',
|
|
103
|
+
* throwOnViolation: true,
|
|
104
|
+
* applyToInsert: true,
|
|
105
|
+
* applyToUpdate: true,
|
|
106
|
+
* applyToUpsert: true
|
|
107
|
+
* }
|
|
108
|
+
*
|
|
109
|
+
* @example
|
|
110
|
+
* // Minimal configuration using defaults
|
|
111
|
+
* {
|
|
112
|
+
* enabled: true,
|
|
113
|
+
* maxBodySize: 1024 * 1024 // 1MB
|
|
114
|
+
* }
|
|
115
|
+
*
|
|
116
|
+
* @notes
|
|
117
|
+
* - Default body size limit is 1MB (1024*1024 bytes)
|
|
118
|
+
* - Default metadata size limit is 2KB (2048 bytes)
|
|
119
|
+
* - Strict mode throws errors on violations
|
|
120
|
+
* - Warn mode logs violations but allows operations
|
|
121
|
+
* - Soft mode allows violations with warnings
|
|
122
|
+
* - Field-specific limits override global limits
|
|
123
|
+
* - Custom validators allow for specialized logic
|
|
124
|
+
* - Warning threshold helps prevent unexpected violations
|
|
125
|
+
* - Performance impact is minimal for most use cases
|
|
126
|
+
* - Limits help prevent abuse and ensure system stability
|
|
127
|
+
* - Context object is useful for conditional validation
|
|
128
|
+
* - Validation can be selectively applied to different operations
|
|
129
|
+
*/
|
|
130
|
+
|
|
131
|
+
/**
|
|
132
|
+
* Enforce Limits Behavior
|
|
133
|
+
* Throws error when metadata exceeds 2KB limit
|
|
134
|
+
*/
|
|
135
|
+
export async function handleInsert({ resource, data, mappedData, originalData }) {
|
|
136
|
+
const totalSize = calculateTotalSize(mappedData);
|
|
137
|
+
|
|
138
|
+
// Calculate effective limit considering system overhead
|
|
139
|
+
const effectiveLimit = calculateEffectiveLimit({
|
|
140
|
+
s3Limit: S3_METADATA_LIMIT_BYTES,
|
|
141
|
+
systemConfig: {
|
|
142
|
+
version: resource.version,
|
|
143
|
+
timestamps: resource.config.timestamps,
|
|
144
|
+
id: data.id
|
|
145
|
+
}
|
|
146
|
+
});
|
|
147
|
+
|
|
148
|
+
if (totalSize > effectiveLimit) {
|
|
149
|
+
throw new Error(`S3 metadata size exceeds 2KB limit. Current size: ${totalSize} bytes, effective limit: ${effectiveLimit} bytes, absolute limit: ${S3_METADATA_LIMIT_BYTES} bytes`);
|
|
150
|
+
}
|
|
151
|
+
return { mappedData, body: JSON.stringify(mappedData) };
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
export async function handleUpdate({ resource, id, data, mappedData, originalData }) {
|
|
155
|
+
const totalSize = calculateTotalSize(mappedData);
|
|
156
|
+
|
|
157
|
+
// Calculate effective limit considering system overhead
|
|
158
|
+
const effectiveLimit = calculateEffectiveLimit({
|
|
159
|
+
s3Limit: S3_METADATA_LIMIT_BYTES,
|
|
160
|
+
systemConfig: {
|
|
161
|
+
version: resource.version,
|
|
162
|
+
timestamps: resource.config.timestamps,
|
|
163
|
+
id
|
|
164
|
+
}
|
|
165
|
+
});
|
|
166
|
+
|
|
167
|
+
if (totalSize > effectiveLimit) {
|
|
168
|
+
throw new Error(`S3 metadata size exceeds 2KB limit. Current size: ${totalSize} bytes, effective limit: ${effectiveLimit} bytes, absolute limit: ${S3_METADATA_LIMIT_BYTES} bytes`);
|
|
169
|
+
}
|
|
170
|
+
return { mappedData, body: JSON.stringify(mappedData) };
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
export async function handleUpsert({ resource, id, data, mappedData }) {
|
|
174
|
+
const totalSize = calculateTotalSize(mappedData);
|
|
175
|
+
|
|
176
|
+
// Calculate effective limit considering system overhead
|
|
177
|
+
const effectiveLimit = calculateEffectiveLimit({
|
|
178
|
+
s3Limit: S3_METADATA_LIMIT_BYTES,
|
|
179
|
+
systemConfig: {
|
|
180
|
+
version: resource.version,
|
|
181
|
+
timestamps: resource.config.timestamps,
|
|
182
|
+
id
|
|
183
|
+
}
|
|
184
|
+
});
|
|
185
|
+
|
|
186
|
+
if (totalSize > effectiveLimit) {
|
|
187
|
+
throw new Error(`S3 metadata size exceeds 2KB limit. Current size: ${totalSize} bytes, effective limit: ${effectiveLimit} bytes, absolute limit: ${S3_METADATA_LIMIT_BYTES} bytes`);
|
|
188
|
+
}
|
|
189
|
+
return { mappedData, body: "" };
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
export async function handleGet({ resource, metadata, body }) {
|
|
193
|
+
// No special handling needed for enforce-limits behavior
|
|
194
|
+
return { metadata, body };
|
|
195
|
+
}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import * as userManaged from './user-managed.js';
|
|
2
|
+
import * as enforceLimits from './enforce-limits.js';
|
|
3
|
+
import * as dataTruncate from './truncate-data.js';
|
|
4
|
+
import * as bodyOverflow from './body-overflow.js';
|
|
5
|
+
import * as bodyOnly from './body-only.js';
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* Available behaviors for Resource metadata handling
|
|
9
|
+
*/
|
|
10
|
+
export const behaviors = {
|
|
11
|
+
'user-managed': userManaged,
|
|
12
|
+
'enforce-limits': enforceLimits,
|
|
13
|
+
'truncate-data': dataTruncate,
|
|
14
|
+
'body-overflow': bodyOverflow,
|
|
15
|
+
'body-only': bodyOnly
|
|
16
|
+
};
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* Get behavior implementation by name
|
|
20
|
+
* @param {string} behaviorName - Name of the behavior
|
|
21
|
+
* @returns {Object} Behavior implementation with handler functions
|
|
22
|
+
*/
|
|
23
|
+
export function getBehavior(behaviorName) {
|
|
24
|
+
const behavior = behaviors[behaviorName];
|
|
25
|
+
if (!behavior) {
|
|
26
|
+
throw new Error(`Unknown behavior: ${behaviorName}. Available behaviors: ${Object.keys(behaviors).join(', ')}`);
|
|
27
|
+
}
|
|
28
|
+
return behavior;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* List of available behavior names
|
|
33
|
+
*/
|
|
34
|
+
export const AVAILABLE_BEHAVIORS = Object.keys(behaviors);
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Default behavior name
|
|
38
|
+
*/
|
|
39
|
+
export const DEFAULT_BEHAVIOR = 'user-managed';
|