s3db.js 7.3.4 → 7.3.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/PLUGINS.md +1285 -157
- package/dist/s3db.cjs.js +322 -119
- package/dist/s3db.cjs.min.js +1 -1
- package/dist/s3db.es.js +322 -119
- package/dist/s3db.es.min.js +1 -1
- package/dist/s3db.iife.js +323 -120
- package/dist/s3db.iife.min.js +1 -1
- package/mcp/server.js +1410 -0
- package/package.json +30 -24
- package/src/database.class.js +10 -8
- package/src/plugins/cache/filesystem-cache.class.js +9 -0
- package/src/plugins/metrics.plugin.js +18 -8
- package/src/plugins/replicator.plugin.js +130 -72
- package/src/plugins/replicators/bigquery-replicator.class.js +31 -5
- package/src/plugins/replicators/postgres-replicator.class.js +17 -2
- package/src/plugins/replicators/s3db-replicator.class.js +175 -71
- package/src/plugins/replicators/sqs-replicator.class.js +13 -1
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "s3db.js",
|
|
3
|
-
"version": "7.3.
|
|
3
|
+
"version": "7.3.6",
|
|
4
4
|
"description": "Use AWS S3, the world's most reliable document storage, as a database with this ORM.",
|
|
5
5
|
"main": "dist/s3db.cjs.js",
|
|
6
6
|
"module": "dist/s3db.es.js",
|
|
@@ -10,6 +10,9 @@
|
|
|
10
10
|
"jsdelivr": "dist/s3db.iife.min.js",
|
|
11
11
|
"author": "@stone/martech",
|
|
12
12
|
"license": "UNLICENSED",
|
|
13
|
+
"bin": {
|
|
14
|
+
"s3db-mcp": "./mcp/server.js"
|
|
15
|
+
},
|
|
13
16
|
"repository": {
|
|
14
17
|
"type": "git",
|
|
15
18
|
"url": "git+https://github.com/forattini-dev/s3db.js.git"
|
|
@@ -26,6 +29,9 @@
|
|
|
26
29
|
"type": "module",
|
|
27
30
|
"sideEffects": false,
|
|
28
31
|
"imports": {
|
|
32
|
+
"#mcp/*": "./mcp/*",
|
|
33
|
+
"#dist/*": "./dist/*",
|
|
34
|
+
"#examples/*": "./examples/*",
|
|
29
35
|
"#src/*": "./src/*",
|
|
30
36
|
"#tests/*": "./tests/*"
|
|
31
37
|
},
|
|
@@ -43,27 +49,6 @@
|
|
|
43
49
|
"PLUGINS.md",
|
|
44
50
|
"UNLICENSE"
|
|
45
51
|
],
|
|
46
|
-
"scripts": {
|
|
47
|
-
"build": "rollup -c",
|
|
48
|
-
"dev": "rollup -c -w",
|
|
49
|
-
"test": "npm run test:js && npm run test:ts",
|
|
50
|
-
"test:js": "node --no-warnings --experimental-vm-modules node_modules/jest/bin/jest.js --runInBand",
|
|
51
|
-
"test:ts": "tsc --noEmit --project tests/typescript/tsconfig.json",
|
|
52
|
-
"test:js-converage": "node --no-warnings --experimental-vm-modules node_modules/jest/bin/jest.js --detectOpenHandles --coverage --runInBand",
|
|
53
|
-
"test:js-ai": "node --no-warnings --experimental-vm-modules node_modules/jest/bin/jest.js --detectOpenHandles --runInBand",
|
|
54
|
-
"test:full": "npm run test:js && npm run test:ts",
|
|
55
|
-
"test:cache": "node --no-warnings --experimental-vm-modules node_modules/jest/bin/jest.js tests/plugins/plugin-cache*.test.js --runInBand",
|
|
56
|
-
"test:quick": "node --no-warnings --experimental-vm-modules node_modules/jest/bin/jest.js --runInBand --testTimeout=10000",
|
|
57
|
-
"test:batch": "./test-batch.sh",
|
|
58
|
-
"test:plugins": "node --no-warnings --experimental-vm-modules node_modules/jest/bin/jest.js tests/plugins/ --runInBand --testTimeout=60000",
|
|
59
|
-
"test:plugins:fast": "node --no-warnings --experimental-vm-modules node_modules/jest/bin/jest.js tests/plugins/ --runInBand --testTimeout=15000 --testPathIgnorePatterns='plugin-audit.test.js|plugin-replicator-s3db.test.js|plugin-fulltext.test.js'",
|
|
60
|
-
"test:slow": "node --no-warnings --experimental-vm-modules node_modules/jest/bin/jest.js tests/plugins/plugin-audit.test.js tests/plugins/plugin-replicator-s3db.test.js tests/plugins/plugin-fulltext.test.js --runInBand --testTimeout=120000",
|
|
61
|
-
"test:types": "tsc --noEmit --project tests/typescript/tsconfig.json",
|
|
62
|
-
"test:types:basic": "tsc --noEmit tests/typescript/basic-usage.test.ts",
|
|
63
|
-
"test:types:direct": "tsc --noEmit tests/typescript/direct-type-test.ts",
|
|
64
|
-
"test:types:watch": "tsc --noEmit --watch --project tests/typescript/tsconfig.json",
|
|
65
|
-
"validate:types": "npm run test:types && echo 'TypeScript definitions are valid!'"
|
|
66
|
-
},
|
|
67
52
|
"dependencies": {
|
|
68
53
|
"@aws-sdk/client-s3": "^3.848.0",
|
|
69
54
|
"@supercharge/promise-pool": "^3.2.0",
|
|
@@ -115,5 +100,26 @@
|
|
|
115
100
|
},
|
|
116
101
|
"funding": [
|
|
117
102
|
"https://github.com/sponsors/forattini-dev"
|
|
118
|
-
]
|
|
119
|
-
|
|
103
|
+
],
|
|
104
|
+
"scripts": {
|
|
105
|
+
"build": "rollup -c",
|
|
106
|
+
"dev": "rollup -c -w",
|
|
107
|
+
"test": "npm run test:js && npm run test:ts",
|
|
108
|
+
"test:js": "node --no-warnings --experimental-vm-modules node_modules/jest/bin/jest.js --runInBand",
|
|
109
|
+
"test:ts": "tsc --noEmit --project tests/typescript/tsconfig.json",
|
|
110
|
+
"test:js-converage": "node --no-warnings --experimental-vm-modules node_modules/jest/bin/jest.js --detectOpenHandles --coverage --runInBand",
|
|
111
|
+
"test:js-ai": "node --no-warnings --experimental-vm-modules node_modules/jest/bin/jest.js --detectOpenHandles --runInBand",
|
|
112
|
+
"test:full": "npm run test:js && npm run test:ts",
|
|
113
|
+
"test:cache": "node --no-warnings --experimental-vm-modules node_modules/jest/bin/jest.js tests/plugins/plugin-cache*.test.js --runInBand",
|
|
114
|
+
"test:quick": "node --no-warnings --experimental-vm-modules node_modules/jest/bin/jest.js --runInBand --testTimeout=10000",
|
|
115
|
+
"test:batch": "./test-batch.sh",
|
|
116
|
+
"test:plugins": "node --no-warnings --experimental-vm-modules node_modules/jest/bin/jest.js tests/plugins/ --runInBand --testTimeout=60000",
|
|
117
|
+
"test:plugins:fast": "node --no-warnings --experimental-vm-modules node_modules/jest/bin/jest.js tests/plugins/ --runInBand --testTimeout=15000 --testPathIgnorePatterns='plugin-audit.test.js|plugin-replicator-s3db.test.js|plugin-fulltext.test.js'",
|
|
118
|
+
"test:slow": "node --no-warnings --experimental-vm-modules node_modules/jest/bin/jest.js tests/plugins/plugin-audit.test.js tests/plugins/plugin-replicator-s3db.test.js tests/plugins/plugin-fulltext.test.js --runInBand --testTimeout=120000",
|
|
119
|
+
"test:types": "tsc --noEmit --project tests/typescript/tsconfig.json",
|
|
120
|
+
"test:types:basic": "tsc --noEmit tests/typescript/basic-usage.test.ts",
|
|
121
|
+
"test:types:direct": "tsc --noEmit tests/typescript/direct-type-test.ts",
|
|
122
|
+
"test:types:watch": "tsc --noEmit --watch --project tests/typescript/tsconfig.json",
|
|
123
|
+
"validate:types": "npm run test:types && echo 'TypeScript definitions are valid!'"
|
|
124
|
+
}
|
|
125
|
+
}
|
package/src/database.class.js
CHANGED
|
@@ -74,15 +74,17 @@ export class Database extends EventEmitter {
|
|
|
74
74
|
// Add process exit listener for cleanup
|
|
75
75
|
if (!this._exitListenerRegistered) {
|
|
76
76
|
this._exitListenerRegistered = true;
|
|
77
|
-
process
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
77
|
+
if (typeof process !== 'undefined') {
|
|
78
|
+
process.on('exit', async () => {
|
|
79
|
+
if (this.isConnected()) {
|
|
80
|
+
try {
|
|
81
|
+
await this.disconnect();
|
|
82
|
+
} catch (err) {
|
|
83
|
+
// Silently ignore errors on exit
|
|
84
|
+
}
|
|
83
85
|
}
|
|
84
|
-
}
|
|
85
|
-
}
|
|
86
|
+
});
|
|
87
|
+
}
|
|
86
88
|
}
|
|
87
89
|
}
|
|
88
90
|
|
|
@@ -428,6 +428,15 @@ export class FilesystemCache extends Cache {
|
|
|
428
428
|
|
|
429
429
|
async _clear(prefix) {
|
|
430
430
|
try {
|
|
431
|
+
// Check if directory exists before trying to read it
|
|
432
|
+
if (!await this._fileExists(this.directory)) {
|
|
433
|
+
// Directory doesn't exist, nothing to clear
|
|
434
|
+
if (this.enableStats) {
|
|
435
|
+
this.stats.clears++;
|
|
436
|
+
}
|
|
437
|
+
return true;
|
|
438
|
+
}
|
|
439
|
+
|
|
431
440
|
const files = await readdir(this.directory);
|
|
432
441
|
const cacheFiles = files.filter(file => {
|
|
433
442
|
if (!file.startsWith(this.prefix)) return false;
|
|
@@ -33,7 +33,7 @@ export class MetricsPlugin extends Plugin {
|
|
|
33
33
|
|
|
34
34
|
async setup(database) {
|
|
35
35
|
this.database = database;
|
|
36
|
-
if (process.env.NODE_ENV === 'test') return;
|
|
36
|
+
if (typeof process !== 'undefined' && process.env.NODE_ENV === 'test') return;
|
|
37
37
|
|
|
38
38
|
const [ok, err] = await tryFn(async () => {
|
|
39
39
|
const [ok1, err1, metricsResource] = await tryFn(() => database.createResource({
|
|
@@ -90,7 +90,7 @@ export class MetricsPlugin extends Plugin {
|
|
|
90
90
|
this.installMetricsHooks();
|
|
91
91
|
|
|
92
92
|
// Disable flush timer during tests to avoid side effects
|
|
93
|
-
if (process.env.NODE_ENV !== 'test') {
|
|
93
|
+
if (typeof process !== 'undefined' && process.env.NODE_ENV !== 'test') {
|
|
94
94
|
this.startFlushTimer();
|
|
95
95
|
}
|
|
96
96
|
}
|
|
@@ -107,7 +107,7 @@ export class MetricsPlugin extends Plugin {
|
|
|
107
107
|
}
|
|
108
108
|
|
|
109
109
|
// Don't flush metrics during tests
|
|
110
|
-
if (process.env.NODE_ENV !== 'test') {
|
|
110
|
+
if (typeof process !== 'undefined' && process.env.NODE_ENV !== 'test') {
|
|
111
111
|
await this.flushMetrics();
|
|
112
112
|
}
|
|
113
113
|
}
|
|
@@ -328,11 +328,21 @@ export class MetricsPlugin extends Plugin {
|
|
|
328
328
|
if (!this.metricsResource) return;
|
|
329
329
|
|
|
330
330
|
const [ok, err] = await tryFn(async () => {
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
331
|
+
let metadata, perfMetadata, errorMetadata, resourceMetadata;
|
|
332
|
+
|
|
333
|
+
if (typeof process !== 'undefined' && process.env.NODE_ENV === 'test') {
|
|
334
|
+
// Use empty metadata during tests to avoid header issues
|
|
335
|
+
metadata = {};
|
|
336
|
+
perfMetadata = {};
|
|
337
|
+
errorMetadata = {};
|
|
338
|
+
resourceMetadata = {};
|
|
339
|
+
} else {
|
|
340
|
+
// Use empty metadata during tests to avoid header issues
|
|
341
|
+
metadata = { global: 'true' };
|
|
342
|
+
perfMetadata = { perf: 'true' };
|
|
343
|
+
errorMetadata = { error: 'true' };
|
|
344
|
+
resourceMetadata = { resource: 'true' };
|
|
345
|
+
}
|
|
336
346
|
|
|
337
347
|
// Flush operation metrics
|
|
338
348
|
for (const [operation, data] of Object.entries(this.metrics.operations)) {
|
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
import { isPlainObject } from 'lodash-es';
|
|
2
|
-
|
|
3
1
|
import Plugin from "./plugin.class.js";
|
|
4
2
|
import tryFn from "../concerns/try-fn.js";
|
|
5
3
|
import { createReplicator, validateReplicatorConfig } from "./replicators/index.js";
|
|
@@ -63,19 +61,13 @@ function normalizeResourceName(name) {
|
|
|
63
61
|
* 2. Map: source resource → destination resource name:
|
|
64
62
|
* resources: { users: 'people' }
|
|
65
63
|
*
|
|
66
|
-
* 3. Map: source resource →
|
|
67
|
-
* resources: { users:
|
|
68
|
-
*
|
|
69
|
-
* 4. Map: source resource → { resource, transformer }:
|
|
70
|
-
* resources: { users: { resource: 'people', transformer: fn } }
|
|
71
|
-
*
|
|
72
|
-
* 5. Map: source resource → array of objects (multi-destination):
|
|
73
|
-
* resources: { users: [ { resource: 'people', transformer: fn } ] }
|
|
64
|
+
* 3. Map: source resource → { resource, transform }:
|
|
65
|
+
* resources: { users: { resource: 'people', transform: fn } }
|
|
74
66
|
*
|
|
75
|
-
*
|
|
67
|
+
* 4. Map: source resource → function (transformer only):
|
|
76
68
|
* resources: { users: (el) => ({ ...el, fullName: el.name }) }
|
|
77
69
|
*
|
|
78
|
-
*
|
|
70
|
+
* The transform function is optional and applies to data before replication.
|
|
79
71
|
*
|
|
80
72
|
* === Example Plugin Configurations ===
|
|
81
73
|
*
|
|
@@ -95,10 +87,10 @@ function normalizeResourceName(name) {
|
|
|
95
87
|
* ]
|
|
96
88
|
* });
|
|
97
89
|
*
|
|
98
|
-
* // Advanced mapping with
|
|
90
|
+
* // Advanced mapping with transform
|
|
99
91
|
* new ReplicatorPlugin({
|
|
100
92
|
* replicators: [
|
|
101
|
-
* { driver: 's3db', client: dbB, config: { resources: { users:
|
|
93
|
+
* { driver: 's3db', client: dbB, config: { resources: { users: { resource: 'people', transform: (el) => ({ ...el, fullName: el.name }) } } } }
|
|
102
94
|
* ]
|
|
103
95
|
* });
|
|
104
96
|
*
|
|
@@ -179,27 +171,42 @@ export class ReplicatorPlugin extends Plugin {
|
|
|
179
171
|
}
|
|
180
172
|
|
|
181
173
|
resource.on('insert', async (data) => {
|
|
182
|
-
|
|
174
|
+
const [ok, error] = await tryFn(async () => {
|
|
183
175
|
const completeData = { ...data, createdAt: new Date().toISOString() };
|
|
184
176
|
await plugin.processReplicatorEvent('insert', resource.name, completeData.id, completeData);
|
|
185
|
-
}
|
|
177
|
+
});
|
|
178
|
+
|
|
179
|
+
if (!ok) {
|
|
180
|
+
if (this.config.verbose) {
|
|
181
|
+
console.warn(`[ReplicatorPlugin] Insert event failed for resource ${resource.name}: ${error.message}`);
|
|
182
|
+
}
|
|
186
183
|
this.emit('error', { operation: 'insert', error: error.message, resource: resource.name });
|
|
187
184
|
}
|
|
188
185
|
});
|
|
189
186
|
|
|
190
187
|
resource.on('update', async (data, beforeData) => {
|
|
191
|
-
|
|
188
|
+
const [ok, error] = await tryFn(async () => {
|
|
192
189
|
const completeData = { ...data, updatedAt: new Date().toISOString() };
|
|
193
190
|
await plugin.processReplicatorEvent('update', resource.name, completeData.id, completeData, beforeData);
|
|
194
|
-
}
|
|
191
|
+
});
|
|
192
|
+
|
|
193
|
+
if (!ok) {
|
|
194
|
+
if (this.config.verbose) {
|
|
195
|
+
console.warn(`[ReplicatorPlugin] Update event failed for resource ${resource.name}: ${error.message}`);
|
|
196
|
+
}
|
|
195
197
|
this.emit('error', { operation: 'update', error: error.message, resource: resource.name });
|
|
196
198
|
}
|
|
197
199
|
});
|
|
198
200
|
|
|
199
201
|
resource.on('delete', async (data) => {
|
|
200
|
-
|
|
202
|
+
const [ok, error] = await tryFn(async () => {
|
|
201
203
|
await plugin.processReplicatorEvent('delete', resource.name, data.id, data);
|
|
202
|
-
}
|
|
204
|
+
});
|
|
205
|
+
|
|
206
|
+
if (!ok) {
|
|
207
|
+
if (this.config.verbose) {
|
|
208
|
+
console.warn(`[ReplicatorPlugin] Delete event failed for resource ${resource.name}: ${error.message}`);
|
|
209
|
+
}
|
|
203
210
|
this.emit('error', { operation: 'delete', error: error.message, resource: resource.name });
|
|
204
211
|
}
|
|
205
212
|
});
|
|
@@ -221,14 +228,19 @@ export class ReplicatorPlugin extends Plugin {
|
|
|
221
228
|
async setup(database) {
|
|
222
229
|
this.database = database;
|
|
223
230
|
|
|
224
|
-
|
|
231
|
+
const [initOk, initError] = await tryFn(async () => {
|
|
225
232
|
await this.initializeReplicators(database);
|
|
226
|
-
}
|
|
227
|
-
|
|
228
|
-
|
|
233
|
+
});
|
|
234
|
+
|
|
235
|
+
if (!initOk) {
|
|
236
|
+
if (this.config.verbose) {
|
|
237
|
+
console.warn(`[ReplicatorPlugin] Replicator initialization failed: ${initError.message}`);
|
|
238
|
+
}
|
|
239
|
+
this.emit('error', { operation: 'setup', error: initError.message });
|
|
240
|
+
throw initError;
|
|
229
241
|
}
|
|
230
242
|
|
|
231
|
-
|
|
243
|
+
const [logOk, logError] = await tryFn(async () => {
|
|
232
244
|
if (this.config.replicatorLogResource) {
|
|
233
245
|
const logRes = await database.createResource({
|
|
234
246
|
name: this.config.replicatorLogResource,
|
|
@@ -245,8 +257,16 @@ export class ReplicatorPlugin extends Plugin {
|
|
|
245
257
|
}
|
|
246
258
|
});
|
|
247
259
|
}
|
|
248
|
-
}
|
|
249
|
-
|
|
260
|
+
});
|
|
261
|
+
|
|
262
|
+
if (!logOk) {
|
|
263
|
+
if (this.config.verbose) {
|
|
264
|
+
console.warn(`[ReplicatorPlugin] Failed to create log resource ${this.config.replicatorLogResource}: ${logError.message}`);
|
|
265
|
+
}
|
|
266
|
+
this.emit('replicator_log_resource_creation_error', {
|
|
267
|
+
resourceName: this.config.replicatorLogResource,
|
|
268
|
+
error: logError.message
|
|
269
|
+
});
|
|
250
270
|
}
|
|
251
271
|
|
|
252
272
|
await this.uploadMetadataFile(database);
|
|
@@ -300,48 +320,33 @@ export class ReplicatorPlugin extends Plugin {
|
|
|
300
320
|
// await this.processQueue(); // Removed as per edit hint
|
|
301
321
|
}
|
|
302
322
|
|
|
303
|
-
filterInternalFields(data) {
|
|
304
|
-
if (!data || typeof data !== 'object') return data;
|
|
305
|
-
const filtered = {};
|
|
306
|
-
for (const [key, value] of Object.entries(data)) {
|
|
307
|
-
// Filter out internal fields that start with _ or $
|
|
308
|
-
if (!key.startsWith('_') && !key.startsWith('$')) {
|
|
309
|
-
filtered[key] = value;
|
|
310
|
-
}
|
|
311
|
-
}
|
|
312
|
-
return filtered;
|
|
313
|
-
}
|
|
314
|
-
|
|
315
323
|
async uploadMetadataFile(database) {
|
|
316
324
|
if (typeof database.uploadMetadataFile === 'function') {
|
|
317
325
|
await database.uploadMetadataFile();
|
|
318
326
|
}
|
|
319
327
|
}
|
|
320
328
|
|
|
321
|
-
async getCompleteData(resource, data) {
|
|
322
|
-
try {
|
|
323
|
-
const [ok, err, record] = await tryFn(() => resource.get(data.id));
|
|
324
|
-
if (ok && record) {
|
|
325
|
-
return record;
|
|
326
|
-
}
|
|
327
|
-
} catch (error) {
|
|
328
|
-
// Fallback to provided data
|
|
329
|
-
}
|
|
330
|
-
return data;
|
|
331
|
-
}
|
|
332
|
-
|
|
333
329
|
async retryWithBackoff(operation, maxRetries = 3) {
|
|
334
330
|
let lastError;
|
|
335
331
|
for (let attempt = 1; attempt <= maxRetries; attempt++) {
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
332
|
+
const [ok, error] = await tryFn(operation);
|
|
333
|
+
|
|
334
|
+
if (ok) {
|
|
335
|
+
return ok;
|
|
336
|
+
} else {
|
|
339
337
|
lastError = error;
|
|
338
|
+
if (this.config.verbose) {
|
|
339
|
+
console.warn(`[ReplicatorPlugin] Retry attempt ${attempt}/${maxRetries} failed: ${error.message}`);
|
|
340
|
+
}
|
|
341
|
+
|
|
340
342
|
if (attempt === maxRetries) {
|
|
341
343
|
throw error;
|
|
342
344
|
}
|
|
343
345
|
// Simple backoff: wait 1s, 2s, 4s...
|
|
344
346
|
const delay = Math.pow(2, attempt - 1) * 1000;
|
|
347
|
+
if (this.config.verbose) {
|
|
348
|
+
console.warn(`[ReplicatorPlugin] Waiting ${delay}ms before retry...`);
|
|
349
|
+
}
|
|
345
350
|
await new Promise(resolve => setTimeout(resolve, delay));
|
|
346
351
|
}
|
|
347
352
|
}
|
|
@@ -349,7 +354,7 @@ export class ReplicatorPlugin extends Plugin {
|
|
|
349
354
|
}
|
|
350
355
|
|
|
351
356
|
async logError(replicator, resourceName, operation, recordId, data, error) {
|
|
352
|
-
|
|
357
|
+
const [ok, logError] = await tryFn(async () => {
|
|
353
358
|
const logResourceName = this.config.replicatorLogResource;
|
|
354
359
|
if (this.database && this.database.resources && this.database.resources[logResourceName]) {
|
|
355
360
|
const logResource = this.database.resources[logResourceName];
|
|
@@ -364,8 +369,20 @@ export class ReplicatorPlugin extends Plugin {
|
|
|
364
369
|
status: 'error'
|
|
365
370
|
});
|
|
366
371
|
}
|
|
367
|
-
}
|
|
368
|
-
|
|
372
|
+
});
|
|
373
|
+
|
|
374
|
+
if (!ok) {
|
|
375
|
+
if (this.config.verbose) {
|
|
376
|
+
console.warn(`[ReplicatorPlugin] Failed to log error for ${resourceName}: ${logError.message}`);
|
|
377
|
+
}
|
|
378
|
+
this.emit('replicator_log_error', {
|
|
379
|
+
replicator: replicator.name || replicator.id,
|
|
380
|
+
resourceName,
|
|
381
|
+
operation,
|
|
382
|
+
recordId,
|
|
383
|
+
originalError: error.message,
|
|
384
|
+
logError: logError.message
|
|
385
|
+
});
|
|
369
386
|
}
|
|
370
387
|
}
|
|
371
388
|
|
|
@@ -382,7 +399,7 @@ export class ReplicatorPlugin extends Plugin {
|
|
|
382
399
|
}
|
|
383
400
|
|
|
384
401
|
const promises = applicableReplicators.map(async (replicator) => {
|
|
385
|
-
|
|
402
|
+
const [ok, error, result] = await tryFn(async () => {
|
|
386
403
|
const result = await this.retryWithBackoff(
|
|
387
404
|
() => replicator.replicate(resourceName, operation, data, recordId, beforeData),
|
|
388
405
|
this.config.maxRetries
|
|
@@ -398,7 +415,15 @@ export class ReplicatorPlugin extends Plugin {
|
|
|
398
415
|
});
|
|
399
416
|
|
|
400
417
|
return result;
|
|
401
|
-
}
|
|
418
|
+
});
|
|
419
|
+
|
|
420
|
+
if (ok) {
|
|
421
|
+
return result;
|
|
422
|
+
} else {
|
|
423
|
+
if (this.config.verbose) {
|
|
424
|
+
console.warn(`[ReplicatorPlugin] Replication failed for ${replicator.name || replicator.id} on ${resourceName}: ${error.message}`);
|
|
425
|
+
}
|
|
426
|
+
|
|
402
427
|
this.emit('replicator_error', {
|
|
403
428
|
replicator: replicator.name || replicator.id,
|
|
404
429
|
resourceName,
|
|
@@ -429,12 +454,16 @@ export class ReplicatorPlugin extends Plugin {
|
|
|
429
454
|
}
|
|
430
455
|
|
|
431
456
|
const promises = applicableReplicators.map(async (replicator) => {
|
|
432
|
-
|
|
457
|
+
const [wrapperOk, wrapperError] = await tryFn(async () => {
|
|
433
458
|
const [ok, err, result] = await tryFn(() =>
|
|
434
459
|
replicator.replicate(item.resourceName, item.operation, item.data, item.recordId, item.beforeData)
|
|
435
460
|
);
|
|
436
461
|
|
|
437
462
|
if (!ok) {
|
|
463
|
+
if (this.config.verbose) {
|
|
464
|
+
console.warn(`[ReplicatorPlugin] Replicator item processing failed for ${replicator.name || replicator.id} on ${item.resourceName}: ${err.message}`);
|
|
465
|
+
}
|
|
466
|
+
|
|
438
467
|
this.emit('replicator_error', {
|
|
439
468
|
replicator: replicator.name || replicator.id,
|
|
440
469
|
resourceName: item.resourceName,
|
|
@@ -460,20 +489,28 @@ export class ReplicatorPlugin extends Plugin {
|
|
|
460
489
|
});
|
|
461
490
|
|
|
462
491
|
return { success: true, result };
|
|
463
|
-
}
|
|
492
|
+
});
|
|
493
|
+
|
|
494
|
+
if (wrapperOk) {
|
|
495
|
+
return wrapperOk;
|
|
496
|
+
} else {
|
|
497
|
+
if (this.config.verbose) {
|
|
498
|
+
console.warn(`[ReplicatorPlugin] Wrapper processing failed for ${replicator.name || replicator.id} on ${item.resourceName}: ${wrapperError.message}`);
|
|
499
|
+
}
|
|
500
|
+
|
|
464
501
|
this.emit('replicator_error', {
|
|
465
502
|
replicator: replicator.name || replicator.id,
|
|
466
503
|
resourceName: item.resourceName,
|
|
467
504
|
operation: item.operation,
|
|
468
505
|
recordId: item.recordId,
|
|
469
|
-
error:
|
|
506
|
+
error: wrapperError.message
|
|
470
507
|
});
|
|
471
508
|
|
|
472
509
|
if (this.config.logErrors && this.database) {
|
|
473
|
-
await this.logError(replicator, item.resourceName, item.operation, item.recordId, item.data,
|
|
510
|
+
await this.logError(replicator, item.resourceName, item.operation, item.recordId, item.data, wrapperError);
|
|
474
511
|
}
|
|
475
512
|
|
|
476
|
-
return { success: false, error:
|
|
513
|
+
return { success: false, error: wrapperError.message };
|
|
477
514
|
}
|
|
478
515
|
});
|
|
479
516
|
|
|
@@ -500,9 +537,14 @@ export class ReplicatorPlugin extends Plugin {
|
|
|
500
537
|
timestamp: typeof item.timestamp === 'number' ? item.timestamp : Date.now(),
|
|
501
538
|
createdAt: item.createdAt || new Date().toISOString().slice(0, 10),
|
|
502
539
|
};
|
|
503
|
-
|
|
540
|
+
const [ok, err] = await tryFn(async () => {
|
|
504
541
|
await logRes.insert(logItem);
|
|
505
|
-
}
|
|
542
|
+
});
|
|
543
|
+
|
|
544
|
+
if (!ok) {
|
|
545
|
+
if (this.config.verbose) {
|
|
546
|
+
console.warn(`[ReplicatorPlugin] Failed to log replicator item: ${err.message}`);
|
|
547
|
+
}
|
|
506
548
|
this.emit('replicator.log.failed', { error: err, item });
|
|
507
549
|
}
|
|
508
550
|
}
|
|
@@ -637,15 +679,24 @@ export class ReplicatorPlugin extends Plugin {
|
|
|
637
679
|
}
|
|
638
680
|
|
|
639
681
|
async cleanup() {
|
|
640
|
-
|
|
682
|
+
const [ok, error] = await tryFn(async () => {
|
|
641
683
|
if (this.replicators && this.replicators.length > 0) {
|
|
642
684
|
const cleanupPromises = this.replicators.map(async (replicator) => {
|
|
643
|
-
|
|
685
|
+
const [replicatorOk, replicatorError] = await tryFn(async () => {
|
|
644
686
|
if (replicator && typeof replicator.cleanup === 'function') {
|
|
645
687
|
await replicator.cleanup();
|
|
646
688
|
}
|
|
647
|
-
}
|
|
648
|
-
|
|
689
|
+
});
|
|
690
|
+
|
|
691
|
+
if (!replicatorOk) {
|
|
692
|
+
if (this.config.verbose) {
|
|
693
|
+
console.warn(`[ReplicatorPlugin] Failed to cleanup replicator ${replicator.name || replicator.id}: ${replicatorError.message}`);
|
|
694
|
+
}
|
|
695
|
+
this.emit('replicator_cleanup_error', {
|
|
696
|
+
replicator: replicator.name || replicator.id || 'unknown',
|
|
697
|
+
driver: replicator.driver || 'unknown',
|
|
698
|
+
error: replicatorError.message
|
|
699
|
+
});
|
|
649
700
|
}
|
|
650
701
|
});
|
|
651
702
|
|
|
@@ -657,8 +708,15 @@ export class ReplicatorPlugin extends Plugin {
|
|
|
657
708
|
this.eventListenersInstalled.clear();
|
|
658
709
|
|
|
659
710
|
this.removeAllListeners();
|
|
660
|
-
}
|
|
661
|
-
|
|
711
|
+
});
|
|
712
|
+
|
|
713
|
+
if (!ok) {
|
|
714
|
+
if (this.config.verbose) {
|
|
715
|
+
console.warn(`[ReplicatorPlugin] Failed to cleanup plugin: ${error.message}`);
|
|
716
|
+
}
|
|
717
|
+
this.emit('replicator_plugin_cleanup_error', {
|
|
718
|
+
error: error.message
|
|
719
|
+
});
|
|
662
720
|
}
|
|
663
721
|
}
|
|
664
722
|
}
|
|
@@ -115,6 +115,9 @@ class BigqueryReplicator extends BaseReplicator {
|
|
|
115
115
|
await super.initialize(database);
|
|
116
116
|
const [ok, err, sdk] = await tryFn(() => import('@google-cloud/bigquery'));
|
|
117
117
|
if (!ok) {
|
|
118
|
+
if (this.config.verbose) {
|
|
119
|
+
console.warn(`[BigqueryReplicator] Failed to import BigQuery SDK: ${err.message}`);
|
|
120
|
+
}
|
|
118
121
|
this.emit('initialization_error', { replicator: this.name, error: err.message });
|
|
119
122
|
throw err;
|
|
120
123
|
}
|
|
@@ -206,21 +209,32 @@ class BigqueryReplicator extends BaseReplicator {
|
|
|
206
209
|
let lastError = null;
|
|
207
210
|
|
|
208
211
|
for (let attempt = 1; attempt <= maxRetries; attempt++) {
|
|
209
|
-
|
|
212
|
+
const [ok, error] = await tryFn(async () => {
|
|
210
213
|
const [updateJob] = await this.bigqueryClient.createQueryJob({
|
|
211
214
|
query,
|
|
212
215
|
params,
|
|
213
216
|
location: this.location
|
|
214
217
|
});
|
|
215
218
|
await updateJob.getQueryResults();
|
|
216
|
-
|
|
219
|
+
return [updateJob];
|
|
220
|
+
});
|
|
221
|
+
|
|
222
|
+
if (ok) {
|
|
223
|
+
job = ok;
|
|
217
224
|
break;
|
|
218
|
-
}
|
|
225
|
+
} else {
|
|
219
226
|
lastError = error;
|
|
220
227
|
|
|
228
|
+
if (this.config.verbose) {
|
|
229
|
+
console.warn(`[BigqueryReplicator] Update attempt ${attempt} failed: ${error.message}`);
|
|
230
|
+
}
|
|
231
|
+
|
|
221
232
|
// If it's streaming buffer error and not the last attempt
|
|
222
233
|
if (error?.message?.includes('streaming buffer') && attempt < maxRetries) {
|
|
223
234
|
const delaySeconds = 30;
|
|
235
|
+
if (this.config.verbose) {
|
|
236
|
+
console.warn(`[BigqueryReplicator] Retrying in ${delaySeconds} seconds due to streaming buffer issue`);
|
|
237
|
+
}
|
|
224
238
|
await new Promise(resolve => setTimeout(resolve, delaySeconds * 1000));
|
|
225
239
|
continue;
|
|
226
240
|
}
|
|
@@ -298,6 +312,9 @@ class BigqueryReplicator extends BaseReplicator {
|
|
|
298
312
|
|
|
299
313
|
if (ok) return result;
|
|
300
314
|
|
|
315
|
+
if (this.config.verbose) {
|
|
316
|
+
console.warn(`[BigqueryReplicator] Replication failed for ${resourceName}: ${err.message}`);
|
|
317
|
+
}
|
|
301
318
|
this.emit('replicator_error', {
|
|
302
319
|
replicator: this.name,
|
|
303
320
|
resourceName,
|
|
@@ -321,8 +338,14 @@ class BigqueryReplicator extends BaseReplicator {
|
|
|
321
338
|
record.id,
|
|
322
339
|
record.beforeData
|
|
323
340
|
));
|
|
324
|
-
if (ok)
|
|
325
|
-
|
|
341
|
+
if (ok) {
|
|
342
|
+
results.push(res);
|
|
343
|
+
} else {
|
|
344
|
+
if (this.config.verbose) {
|
|
345
|
+
console.warn(`[BigqueryReplicator] Batch replication failed for record ${record.id}: ${err.message}`);
|
|
346
|
+
}
|
|
347
|
+
errors.push({ id: record.id, error: err.message });
|
|
348
|
+
}
|
|
326
349
|
}
|
|
327
350
|
|
|
328
351
|
return {
|
|
@@ -340,6 +363,9 @@ class BigqueryReplicator extends BaseReplicator {
|
|
|
340
363
|
return true;
|
|
341
364
|
});
|
|
342
365
|
if (ok) return true;
|
|
366
|
+
if (this.config.verbose) {
|
|
367
|
+
console.warn(`[BigqueryReplicator] Connection test failed: ${err.message}`);
|
|
368
|
+
}
|
|
343
369
|
this.emit('connection_error', { replicator: this.name, error: err.message });
|
|
344
370
|
return false;
|
|
345
371
|
}
|
|
@@ -113,6 +113,9 @@ class PostgresReplicator extends BaseReplicator {
|
|
|
113
113
|
await super.initialize(database);
|
|
114
114
|
const [ok, err, sdk] = await tryFn(() => import('pg'));
|
|
115
115
|
if (!ok) {
|
|
116
|
+
if (this.config.verbose) {
|
|
117
|
+
console.warn(`[PostgresReplicator] Failed to import pg SDK: ${err.message}`);
|
|
118
|
+
}
|
|
116
119
|
this.emit('initialization_error', {
|
|
117
120
|
replicator: this.name,
|
|
118
121
|
error: err.message
|
|
@@ -276,6 +279,9 @@ class PostgresReplicator extends BaseReplicator {
|
|
|
276
279
|
};
|
|
277
280
|
});
|
|
278
281
|
if (ok) return result;
|
|
282
|
+
if (this.config.verbose) {
|
|
283
|
+
console.warn(`[PostgresReplicator] Replication failed for ${resourceName}: ${err.message}`);
|
|
284
|
+
}
|
|
279
285
|
this.emit('replicator_error', {
|
|
280
286
|
replicator: this.name,
|
|
281
287
|
resourceName,
|
|
@@ -298,8 +304,14 @@ class PostgresReplicator extends BaseReplicator {
|
|
|
298
304
|
record.id,
|
|
299
305
|
record.beforeData
|
|
300
306
|
));
|
|
301
|
-
if (ok)
|
|
302
|
-
|
|
307
|
+
if (ok) {
|
|
308
|
+
results.push(res);
|
|
309
|
+
} else {
|
|
310
|
+
if (this.config.verbose) {
|
|
311
|
+
console.warn(`[PostgresReplicator] Batch replication failed for record ${record.id}: ${err.message}`);
|
|
312
|
+
}
|
|
313
|
+
errors.push({ id: record.id, error: err.message });
|
|
314
|
+
}
|
|
303
315
|
}
|
|
304
316
|
|
|
305
317
|
return {
|
|
@@ -316,6 +328,9 @@ class PostgresReplicator extends BaseReplicator {
|
|
|
316
328
|
return true;
|
|
317
329
|
});
|
|
318
330
|
if (ok) return true;
|
|
331
|
+
if (this.config.verbose) {
|
|
332
|
+
console.warn(`[PostgresReplicator] Connection test failed: ${err.message}`);
|
|
333
|
+
}
|
|
319
334
|
this.emit('connection_error', { replicator: this.name, error: err.message });
|
|
320
335
|
return false;
|
|
321
336
|
}
|