s3db.js 7.3.5 → 7.3.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/PLUGINS.md +1285 -157
- package/dist/s3db.cjs.js +314 -83
- package/dist/s3db.cjs.min.js +1 -1
- package/dist/s3db.es.js +314 -83
- package/dist/s3db.es.min.js +1 -1
- package/dist/s3db.iife.js +314 -83
- package/dist/s3db.iife.min.js +1 -1
- package/mcp/server.js +1410 -0
- package/package.json +7 -1
- package/src/plugins/cache/filesystem-cache.class.js +9 -0
- package/src/plugins/replicator.plugin.js +130 -46
- package/src/plugins/replicators/bigquery-replicator.class.js +42 -5
- package/src/plugins/replicators/postgres-replicator.class.js +28 -2
- package/src/plugins/replicators/s3db-replicator.class.js +177 -68
- package/src/plugins/replicators/sqs-replicator.class.js +18 -1
package/dist/s3db.iife.js
CHANGED
|
@@ -7070,6 +7070,12 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
7070
7070
|
}
|
|
7071
7071
|
async _clear(prefix) {
|
|
7072
7072
|
try {
|
|
7073
|
+
if (!await this._fileExists(this.directory)) {
|
|
7074
|
+
if (this.enableStats) {
|
|
7075
|
+
this.stats.clears++;
|
|
7076
|
+
}
|
|
7077
|
+
return true;
|
|
7078
|
+
}
|
|
7073
7079
|
const files = await promises.readdir(this.directory);
|
|
7074
7080
|
const cacheFiles = files.filter((file) => {
|
|
7075
7081
|
if (!file.startsWith(this.prefix)) return false;
|
|
@@ -8941,6 +8947,9 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
8941
8947
|
await super.initialize(database);
|
|
8942
8948
|
const [ok, err, sdk] = await try_fn_default(() => import('@google-cloud/bigquery'));
|
|
8943
8949
|
if (!ok) {
|
|
8950
|
+
if (this.config.verbose) {
|
|
8951
|
+
console.warn(`[BigqueryReplicator] Failed to import BigQuery SDK: ${err.message}`);
|
|
8952
|
+
}
|
|
8944
8953
|
this.emit("initialization_error", { replicator: this.name, error: err.message });
|
|
8945
8954
|
throw err;
|
|
8946
8955
|
}
|
|
@@ -9010,19 +9019,28 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
9010
9019
|
const maxRetries = 2;
|
|
9011
9020
|
let lastError = null;
|
|
9012
9021
|
for (let attempt = 1; attempt <= maxRetries; attempt++) {
|
|
9013
|
-
|
|
9022
|
+
const [ok2, error] = await try_fn_default(async () => {
|
|
9014
9023
|
const [updateJob] = await this.bigqueryClient.createQueryJob({
|
|
9015
9024
|
query,
|
|
9016
9025
|
params,
|
|
9017
9026
|
location: this.location
|
|
9018
9027
|
});
|
|
9019
9028
|
await updateJob.getQueryResults();
|
|
9020
|
-
|
|
9029
|
+
return [updateJob];
|
|
9030
|
+
});
|
|
9031
|
+
if (ok2) {
|
|
9032
|
+
job = ok2;
|
|
9021
9033
|
break;
|
|
9022
|
-
}
|
|
9034
|
+
} else {
|
|
9023
9035
|
lastError = error;
|
|
9036
|
+
if (this.config.verbose) {
|
|
9037
|
+
console.warn(`[BigqueryReplicator] Update attempt ${attempt} failed: ${error.message}`);
|
|
9038
|
+
}
|
|
9024
9039
|
if (error?.message?.includes("streaming buffer") && attempt < maxRetries) {
|
|
9025
9040
|
const delaySeconds = 30;
|
|
9041
|
+
if (this.config.verbose) {
|
|
9042
|
+
console.warn(`[BigqueryReplicator] Retrying in ${delaySeconds} seconds due to streaming buffer issue`);
|
|
9043
|
+
}
|
|
9026
9044
|
await new Promise((resolve) => setTimeout(resolve, delaySeconds * 1e3));
|
|
9027
9045
|
continue;
|
|
9028
9046
|
}
|
|
@@ -9071,6 +9089,9 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
9071
9089
|
}
|
|
9072
9090
|
}
|
|
9073
9091
|
const success = errors.length === 0;
|
|
9092
|
+
if (errors.length > 0) {
|
|
9093
|
+
console.warn(`[BigqueryReplicator] Replication completed with errors for ${resourceName}:`, errors);
|
|
9094
|
+
}
|
|
9074
9095
|
this.emit("replicated", {
|
|
9075
9096
|
replicator: this.name,
|
|
9076
9097
|
resourceName,
|
|
@@ -9089,6 +9110,9 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
9089
9110
|
};
|
|
9090
9111
|
});
|
|
9091
9112
|
if (ok) return result;
|
|
9113
|
+
if (this.config.verbose) {
|
|
9114
|
+
console.warn(`[BigqueryReplicator] Replication failed for ${resourceName}: ${err.message}`);
|
|
9115
|
+
}
|
|
9092
9116
|
this.emit("replicator_error", {
|
|
9093
9117
|
replicator: this.name,
|
|
9094
9118
|
resourceName,
|
|
@@ -9109,8 +9133,17 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
9109
9133
|
record.id,
|
|
9110
9134
|
record.beforeData
|
|
9111
9135
|
));
|
|
9112
|
-
if (ok)
|
|
9113
|
-
|
|
9136
|
+
if (ok) {
|
|
9137
|
+
results.push(res);
|
|
9138
|
+
} else {
|
|
9139
|
+
if (this.config.verbose) {
|
|
9140
|
+
console.warn(`[BigqueryReplicator] Batch replication failed for record ${record.id}: ${err.message}`);
|
|
9141
|
+
}
|
|
9142
|
+
errors.push({ id: record.id, error: err.message });
|
|
9143
|
+
}
|
|
9144
|
+
}
|
|
9145
|
+
if (errors.length > 0) {
|
|
9146
|
+
console.warn(`[BigqueryReplicator] Batch replication completed with ${errors.length} error(s) for ${resourceName}:`, errors);
|
|
9114
9147
|
}
|
|
9115
9148
|
return {
|
|
9116
9149
|
success: errors.length === 0,
|
|
@@ -9126,6 +9159,9 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
9126
9159
|
return true;
|
|
9127
9160
|
});
|
|
9128
9161
|
if (ok) return true;
|
|
9162
|
+
if (this.config.verbose) {
|
|
9163
|
+
console.warn(`[BigqueryReplicator] Connection test failed: ${err.message}`);
|
|
9164
|
+
}
|
|
9129
9165
|
this.emit("connection_error", { replicator: this.name, error: err.message });
|
|
9130
9166
|
return false;
|
|
9131
9167
|
}
|
|
@@ -9213,6 +9249,9 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
9213
9249
|
await super.initialize(database);
|
|
9214
9250
|
const [ok, err, sdk] = await try_fn_default(() => import('pg'));
|
|
9215
9251
|
if (!ok) {
|
|
9252
|
+
if (this.config.verbose) {
|
|
9253
|
+
console.warn(`[PostgresReplicator] Failed to import pg SDK: ${err.message}`);
|
|
9254
|
+
}
|
|
9216
9255
|
this.emit("initialization_error", {
|
|
9217
9256
|
replicator: this.name,
|
|
9218
9257
|
error: err.message
|
|
@@ -9336,6 +9375,9 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
9336
9375
|
}
|
|
9337
9376
|
}
|
|
9338
9377
|
const success = errors.length === 0;
|
|
9378
|
+
if (errors.length > 0) {
|
|
9379
|
+
console.warn(`[PostgresReplicator] Replication completed with errors for ${resourceName}:`, errors);
|
|
9380
|
+
}
|
|
9339
9381
|
this.emit("replicated", {
|
|
9340
9382
|
replicator: this.name,
|
|
9341
9383
|
resourceName,
|
|
@@ -9354,6 +9396,9 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
9354
9396
|
};
|
|
9355
9397
|
});
|
|
9356
9398
|
if (ok) return result;
|
|
9399
|
+
if (this.config.verbose) {
|
|
9400
|
+
console.warn(`[PostgresReplicator] Replication failed for ${resourceName}: ${err.message}`);
|
|
9401
|
+
}
|
|
9357
9402
|
this.emit("replicator_error", {
|
|
9358
9403
|
replicator: this.name,
|
|
9359
9404
|
resourceName,
|
|
@@ -9374,8 +9419,17 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
9374
9419
|
record.id,
|
|
9375
9420
|
record.beforeData
|
|
9376
9421
|
));
|
|
9377
|
-
if (ok)
|
|
9378
|
-
|
|
9422
|
+
if (ok) {
|
|
9423
|
+
results.push(res);
|
|
9424
|
+
} else {
|
|
9425
|
+
if (this.config.verbose) {
|
|
9426
|
+
console.warn(`[PostgresReplicator] Batch replication failed for record ${record.id}: ${err.message}`);
|
|
9427
|
+
}
|
|
9428
|
+
errors.push({ id: record.id, error: err.message });
|
|
9429
|
+
}
|
|
9430
|
+
}
|
|
9431
|
+
if (errors.length > 0) {
|
|
9432
|
+
console.warn(`[PostgresReplicator] Batch replication completed with ${errors.length} error(s) for ${resourceName}:`, errors);
|
|
9379
9433
|
}
|
|
9380
9434
|
return {
|
|
9381
9435
|
success: errors.length === 0,
|
|
@@ -9390,6 +9444,9 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
9390
9444
|
return true;
|
|
9391
9445
|
});
|
|
9392
9446
|
if (ok) return true;
|
|
9447
|
+
if (this.config.verbose) {
|
|
9448
|
+
console.warn(`[PostgresReplicator] Connection test failed: ${err.message}`);
|
|
9449
|
+
}
|
|
9393
9450
|
this.emit("connection_error", { replicator: this.name, error: err.message });
|
|
9394
9451
|
return false;
|
|
9395
9452
|
}
|
|
@@ -13124,7 +13181,7 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
13124
13181
|
super();
|
|
13125
13182
|
this.version = "1";
|
|
13126
13183
|
this.s3dbVersion = (() => {
|
|
13127
|
-
const [ok, err, version] = try_fn_default(() => true ? "7.3.
|
|
13184
|
+
const [ok, err, version] = try_fn_default(() => true ? "7.3.7" : "latest");
|
|
13128
13185
|
return ok ? version : "latest";
|
|
13129
13186
|
})();
|
|
13130
13187
|
this.resources = {};
|
|
@@ -13628,9 +13685,8 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
13628
13685
|
const map = {};
|
|
13629
13686
|
for (const res of resources) {
|
|
13630
13687
|
if (typeof res === "string") map[normalizeResourceName$1(res)] = res;
|
|
13631
|
-
else if (Array.isArray(res) && typeof res[0] === "string") map[normalizeResourceName$1(res[0])] = res;
|
|
13632
13688
|
else if (typeof res === "object" && res.resource) {
|
|
13633
|
-
map[normalizeResourceName$1(res.resource)] =
|
|
13689
|
+
map[normalizeResourceName$1(res.resource)] = res;
|
|
13634
13690
|
}
|
|
13635
13691
|
}
|
|
13636
13692
|
return map;
|
|
@@ -13643,15 +13699,14 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
13643
13699
|
else if (Array.isArray(dest)) {
|
|
13644
13700
|
map[normSrc] = dest.map((item) => {
|
|
13645
13701
|
if (typeof item === "string") return item;
|
|
13646
|
-
if (typeof item === "function") return item;
|
|
13647
13702
|
if (typeof item === "object" && item.resource) {
|
|
13648
|
-
return
|
|
13703
|
+
return item;
|
|
13649
13704
|
}
|
|
13650
13705
|
return item;
|
|
13651
13706
|
});
|
|
13652
13707
|
} else if (typeof dest === "function") map[normSrc] = dest;
|
|
13653
13708
|
else if (typeof dest === "object" && dest.resource) {
|
|
13654
|
-
map[normSrc] =
|
|
13709
|
+
map[normSrc] = dest;
|
|
13655
13710
|
}
|
|
13656
13711
|
}
|
|
13657
13712
|
return map;
|
|
@@ -13659,10 +13714,6 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
13659
13714
|
if (typeof resources === "function") {
|
|
13660
13715
|
return resources;
|
|
13661
13716
|
}
|
|
13662
|
-
if (typeof resources === "string") {
|
|
13663
|
-
const map = { [normalizeResourceName$1(resources)]: resources };
|
|
13664
|
-
return map;
|
|
13665
|
-
}
|
|
13666
13717
|
return {};
|
|
13667
13718
|
}
|
|
13668
13719
|
validateConfig() {
|
|
@@ -13676,8 +13727,8 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
13676
13727
|
return { isValid: errors.length === 0, errors };
|
|
13677
13728
|
}
|
|
13678
13729
|
async initialize(database) {
|
|
13679
|
-
|
|
13680
|
-
|
|
13730
|
+
await super.initialize(database);
|
|
13731
|
+
const [ok, err] = await try_fn_default(async () => {
|
|
13681
13732
|
if (this.client) {
|
|
13682
13733
|
this.targetDatabase = this.client;
|
|
13683
13734
|
} else if (this.connectionString) {
|
|
@@ -13696,7 +13747,11 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
13696
13747
|
replicator: this.name,
|
|
13697
13748
|
target: this.connectionString || "client-provided"
|
|
13698
13749
|
});
|
|
13699
|
-
}
|
|
13750
|
+
});
|
|
13751
|
+
if (!ok) {
|
|
13752
|
+
if (this.config.verbose) {
|
|
13753
|
+
console.warn(`[S3dbReplicator] Initialization failed: ${err.message}`);
|
|
13754
|
+
}
|
|
13700
13755
|
throw err;
|
|
13701
13756
|
}
|
|
13702
13757
|
}
|
|
@@ -13715,18 +13770,77 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
13715
13770
|
id = recordId;
|
|
13716
13771
|
}
|
|
13717
13772
|
const normResource = normalizeResourceName$1(resource);
|
|
13718
|
-
const
|
|
13719
|
-
|
|
13720
|
-
|
|
13773
|
+
const entry = this.resourcesMap[normResource];
|
|
13774
|
+
if (!entry) {
|
|
13775
|
+
throw new Error(`[S3dbReplicator] Resource not configured: ${resource}`);
|
|
13776
|
+
}
|
|
13777
|
+
if (Array.isArray(entry)) {
|
|
13778
|
+
const results = [];
|
|
13779
|
+
for (const destConfig of entry) {
|
|
13780
|
+
const [ok, error, result] = await try_fn_default(async () => {
|
|
13781
|
+
return await this._replicateToSingleDestination(destConfig, normResource, op, payload, id);
|
|
13782
|
+
});
|
|
13783
|
+
if (!ok) {
|
|
13784
|
+
if (this.config && this.config.verbose) {
|
|
13785
|
+
console.warn(`[S3dbReplicator] Failed to replicate to destination ${JSON.stringify(destConfig)}: ${error.message}`);
|
|
13786
|
+
}
|
|
13787
|
+
throw error;
|
|
13788
|
+
}
|
|
13789
|
+
results.push(result);
|
|
13790
|
+
}
|
|
13791
|
+
return results;
|
|
13792
|
+
} else {
|
|
13793
|
+
const [ok, error, result] = await try_fn_default(async () => {
|
|
13794
|
+
return await this._replicateToSingleDestination(entry, normResource, op, payload, id);
|
|
13795
|
+
});
|
|
13796
|
+
if (!ok) {
|
|
13797
|
+
if (this.config && this.config.verbose) {
|
|
13798
|
+
console.warn(`[S3dbReplicator] Failed to replicate to destination ${JSON.stringify(entry)}: ${error.message}`);
|
|
13799
|
+
}
|
|
13800
|
+
throw error;
|
|
13801
|
+
}
|
|
13802
|
+
return result;
|
|
13803
|
+
}
|
|
13804
|
+
}
|
|
13805
|
+
async _replicateToSingleDestination(destConfig, sourceResource, operation, data, recordId) {
|
|
13806
|
+
let destResourceName;
|
|
13807
|
+
if (typeof destConfig === "string") {
|
|
13808
|
+
destResourceName = destConfig;
|
|
13809
|
+
} else if (typeof destConfig === "object" && destConfig.resource) {
|
|
13810
|
+
destResourceName = destConfig.resource;
|
|
13811
|
+
} else {
|
|
13812
|
+
destResourceName = sourceResource;
|
|
13813
|
+
}
|
|
13814
|
+
if (typeof destConfig === "object" && destConfig.actions && Array.isArray(destConfig.actions)) {
|
|
13815
|
+
if (!destConfig.actions.includes(operation)) {
|
|
13816
|
+
return { skipped: true, reason: "action_not_supported", action: operation, destination: destResourceName };
|
|
13817
|
+
}
|
|
13818
|
+
}
|
|
13819
|
+
const destResourceObj = this._getDestResourceObj(destResourceName);
|
|
13820
|
+
let transformedData;
|
|
13821
|
+
if (typeof destConfig === "object" && destConfig.transform && typeof destConfig.transform === "function") {
|
|
13822
|
+
transformedData = destConfig.transform(data);
|
|
13823
|
+
if (transformedData && data && data.id && !transformedData.id) {
|
|
13824
|
+
transformedData.id = data.id;
|
|
13825
|
+
}
|
|
13826
|
+
} else if (typeof destConfig === "object" && destConfig.transformer && typeof destConfig.transformer === "function") {
|
|
13827
|
+
transformedData = destConfig.transformer(data);
|
|
13828
|
+
if (transformedData && data && data.id && !transformedData.id) {
|
|
13829
|
+
transformedData.id = data.id;
|
|
13830
|
+
}
|
|
13831
|
+
} else {
|
|
13832
|
+
transformedData = data;
|
|
13833
|
+
}
|
|
13834
|
+
if (!transformedData && data) transformedData = data;
|
|
13721
13835
|
let result;
|
|
13722
|
-
if (
|
|
13836
|
+
if (operation === "insert") {
|
|
13723
13837
|
result = await destResourceObj.insert(transformedData);
|
|
13724
|
-
} else if (
|
|
13725
|
-
result = await destResourceObj.update(
|
|
13726
|
-
} else if (
|
|
13727
|
-
result = await destResourceObj.delete(
|
|
13838
|
+
} else if (operation === "update") {
|
|
13839
|
+
result = await destResourceObj.update(recordId, transformedData);
|
|
13840
|
+
} else if (operation === "delete") {
|
|
13841
|
+
result = await destResourceObj.delete(recordId);
|
|
13728
13842
|
} else {
|
|
13729
|
-
throw new Error(`Invalid operation: ${
|
|
13843
|
+
throw new Error(`Invalid operation: ${operation}. Supported operations are: insert, update, delete`);
|
|
13730
13844
|
}
|
|
13731
13845
|
return result;
|
|
13732
13846
|
}
|
|
@@ -13735,13 +13849,25 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
13735
13849
|
const entry = this.resourcesMap[normResource];
|
|
13736
13850
|
let result;
|
|
13737
13851
|
if (!entry) return data;
|
|
13738
|
-
if (Array.isArray(entry)
|
|
13739
|
-
|
|
13852
|
+
if (Array.isArray(entry)) {
|
|
13853
|
+
for (const item of entry) {
|
|
13854
|
+
if (typeof item === "object" && item.transform && typeof item.transform === "function") {
|
|
13855
|
+
result = item.transform(data);
|
|
13856
|
+
break;
|
|
13857
|
+
} else if (typeof item === "object" && item.transformer && typeof item.transformer === "function") {
|
|
13858
|
+
result = item.transformer(data);
|
|
13859
|
+
break;
|
|
13860
|
+
}
|
|
13861
|
+
}
|
|
13862
|
+
if (!result) result = data;
|
|
13863
|
+
} else if (typeof entry === "object") {
|
|
13864
|
+
if (typeof entry.transform === "function") {
|
|
13865
|
+
result = entry.transform(data);
|
|
13866
|
+
} else if (typeof entry.transformer === "function") {
|
|
13867
|
+
result = entry.transformer(data);
|
|
13868
|
+
}
|
|
13740
13869
|
} else if (typeof entry === "function") {
|
|
13741
13870
|
result = entry(data);
|
|
13742
|
-
} else if (typeof entry === "object") {
|
|
13743
|
-
if (typeof entry.transform === "function") result = entry.transform(data);
|
|
13744
|
-
else if (typeof entry.transformer === "function") result = entry.transformer(data);
|
|
13745
13871
|
} else {
|
|
13746
13872
|
result = data;
|
|
13747
13873
|
}
|
|
@@ -13754,9 +13880,11 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
13754
13880
|
const entry = this.resourcesMap[normResource];
|
|
13755
13881
|
if (!entry) return resource;
|
|
13756
13882
|
if (Array.isArray(entry)) {
|
|
13757
|
-
|
|
13758
|
-
|
|
13759
|
-
|
|
13883
|
+
for (const item of entry) {
|
|
13884
|
+
if (typeof item === "string") return item;
|
|
13885
|
+
if (typeof item === "object" && item.resource) return item.resource;
|
|
13886
|
+
}
|
|
13887
|
+
return resource;
|
|
13760
13888
|
}
|
|
13761
13889
|
if (typeof entry === "string") return entry;
|
|
13762
13890
|
if (typeof entry === "function") return resource;
|
|
@@ -13764,8 +13892,7 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
13764
13892
|
return resource;
|
|
13765
13893
|
}
|
|
13766
13894
|
_getDestResourceObj(resource) {
|
|
13767
|
-
|
|
13768
|
-
const available = Object.keys(this.client.resources);
|
|
13895
|
+
const available = Object.keys(this.client.resources || {});
|
|
13769
13896
|
const norm = normalizeResourceName$1(resource);
|
|
13770
13897
|
const found = available.find((r) => normalizeResourceName$1(r) === norm);
|
|
13771
13898
|
if (!found) {
|
|
@@ -13787,8 +13914,17 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
13787
13914
|
data: record.data,
|
|
13788
13915
|
beforeData: record.beforeData
|
|
13789
13916
|
}));
|
|
13790
|
-
if (ok)
|
|
13791
|
-
|
|
13917
|
+
if (ok) {
|
|
13918
|
+
results.push(result);
|
|
13919
|
+
} else {
|
|
13920
|
+
if (this.config.verbose) {
|
|
13921
|
+
console.warn(`[S3dbReplicator] Batch replication failed for record ${record.id}: ${err.message}`);
|
|
13922
|
+
}
|
|
13923
|
+
errors.push({ id: record.id, error: err.message });
|
|
13924
|
+
}
|
|
13925
|
+
}
|
|
13926
|
+
if (errors.length > 0) {
|
|
13927
|
+
console.warn(`[S3dbReplicator] Batch replication completed with ${errors.length} error(s) for ${resourceName}:`, errors);
|
|
13792
13928
|
}
|
|
13793
13929
|
this.emit("batch_replicated", {
|
|
13794
13930
|
replicator: this.name,
|
|
@@ -13806,18 +13942,20 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
13806
13942
|
}
|
|
13807
13943
|
async testConnection() {
|
|
13808
13944
|
const [ok, err] = await try_fn_default(async () => {
|
|
13809
|
-
if (!this.targetDatabase)
|
|
13810
|
-
|
|
13945
|
+
if (!this.targetDatabase) throw new Error("No target database configured");
|
|
13946
|
+
if (typeof this.targetDatabase.connect === "function") {
|
|
13947
|
+
await this.targetDatabase.connect();
|
|
13811
13948
|
}
|
|
13812
|
-
await this.targetDatabase.listResources();
|
|
13813
13949
|
return true;
|
|
13814
13950
|
});
|
|
13815
|
-
if (ok)
|
|
13816
|
-
|
|
13817
|
-
|
|
13818
|
-
|
|
13819
|
-
|
|
13820
|
-
|
|
13951
|
+
if (!ok) {
|
|
13952
|
+
if (this.config.verbose) {
|
|
13953
|
+
console.warn(`[S3dbReplicator] Connection test failed: ${err.message}`);
|
|
13954
|
+
}
|
|
13955
|
+
this.emit("connection_error", { replicator: this.name, error: err.message });
|
|
13956
|
+
return false;
|
|
13957
|
+
}
|
|
13958
|
+
return true;
|
|
13821
13959
|
}
|
|
13822
13960
|
async getStatus() {
|
|
13823
13961
|
const baseStatus = await super.getStatus();
|
|
@@ -13849,7 +13987,7 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
13849
13987
|
} else {
|
|
13850
13988
|
return true;
|
|
13851
13989
|
}
|
|
13852
|
-
} else if (typeof item === "string"
|
|
13990
|
+
} else if (typeof item === "string") {
|
|
13853
13991
|
return true;
|
|
13854
13992
|
}
|
|
13855
13993
|
}
|
|
@@ -13976,6 +14114,9 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
13976
14114
|
if (!this.sqsClient) {
|
|
13977
14115
|
const [ok, err, sdk] = await try_fn_default(() => import('@aws-sdk/client-sqs'));
|
|
13978
14116
|
if (!ok) {
|
|
14117
|
+
if (this.config.verbose) {
|
|
14118
|
+
console.warn(`[SqsReplicator] Failed to import SQS SDK: ${err.message}`);
|
|
14119
|
+
}
|
|
13979
14120
|
this.emit("initialization_error", {
|
|
13980
14121
|
replicator: this.name,
|
|
13981
14122
|
error: err.message
|
|
@@ -14027,6 +14168,9 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
14027
14168
|
return { success: true, results };
|
|
14028
14169
|
});
|
|
14029
14170
|
if (ok) return result;
|
|
14171
|
+
if (this.config.verbose) {
|
|
14172
|
+
console.warn(`[SqsReplicator] Replication failed for ${resource}: ${err.message}`);
|
|
14173
|
+
}
|
|
14030
14174
|
this.emit("replicator_error", {
|
|
14031
14175
|
replicator: this.name,
|
|
14032
14176
|
resource,
|
|
@@ -14079,6 +14223,9 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
14079
14223
|
}
|
|
14080
14224
|
}
|
|
14081
14225
|
}
|
|
14226
|
+
if (errors.length > 0) {
|
|
14227
|
+
console.warn(`[SqsReplicator] Batch replication completed with ${errors.length} error(s) for ${resource}:`, errors);
|
|
14228
|
+
}
|
|
14082
14229
|
this.emit("batch_replicated", {
|
|
14083
14230
|
replicator: this.name,
|
|
14084
14231
|
resource,
|
|
@@ -14099,6 +14246,9 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
14099
14246
|
});
|
|
14100
14247
|
if (ok) return result;
|
|
14101
14248
|
const errorMessage = err?.message || err || "Unknown error";
|
|
14249
|
+
if (this.config.verbose) {
|
|
14250
|
+
console.warn(`[SqsReplicator] Batch replication failed for ${resource}: ${errorMessage}`);
|
|
14251
|
+
}
|
|
14102
14252
|
this.emit("batch_replicator_error", {
|
|
14103
14253
|
replicator: this.name,
|
|
14104
14254
|
resource,
|
|
@@ -14120,6 +14270,9 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
14120
14270
|
return true;
|
|
14121
14271
|
});
|
|
14122
14272
|
if (ok) return true;
|
|
14273
|
+
if (this.config.verbose) {
|
|
14274
|
+
console.warn(`[SqsReplicator] Connection test failed: ${err.message}`);
|
|
14275
|
+
}
|
|
14123
14276
|
this.emit("connection_error", {
|
|
14124
14277
|
replicator: this.name,
|
|
14125
14278
|
error: err.message
|
|
@@ -14216,25 +14369,37 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
14216
14369
|
return;
|
|
14217
14370
|
}
|
|
14218
14371
|
resource.on("insert", async (data) => {
|
|
14219
|
-
|
|
14372
|
+
const [ok, error] = await try_fn_default(async () => {
|
|
14220
14373
|
const completeData = { ...data, createdAt: (/* @__PURE__ */ new Date()).toISOString() };
|
|
14221
14374
|
await plugin.processReplicatorEvent("insert", resource.name, completeData.id, completeData);
|
|
14222
|
-
}
|
|
14375
|
+
});
|
|
14376
|
+
if (!ok) {
|
|
14377
|
+
if (this.config.verbose) {
|
|
14378
|
+
console.warn(`[ReplicatorPlugin] Insert event failed for resource ${resource.name}: ${error.message}`);
|
|
14379
|
+
}
|
|
14223
14380
|
this.emit("error", { operation: "insert", error: error.message, resource: resource.name });
|
|
14224
14381
|
}
|
|
14225
14382
|
});
|
|
14226
14383
|
resource.on("update", async (data, beforeData) => {
|
|
14227
|
-
|
|
14384
|
+
const [ok, error] = await try_fn_default(async () => {
|
|
14228
14385
|
const completeData = { ...data, updatedAt: (/* @__PURE__ */ new Date()).toISOString() };
|
|
14229
14386
|
await plugin.processReplicatorEvent("update", resource.name, completeData.id, completeData, beforeData);
|
|
14230
|
-
}
|
|
14387
|
+
});
|
|
14388
|
+
if (!ok) {
|
|
14389
|
+
if (this.config.verbose) {
|
|
14390
|
+
console.warn(`[ReplicatorPlugin] Update event failed for resource ${resource.name}: ${error.message}`);
|
|
14391
|
+
}
|
|
14231
14392
|
this.emit("error", { operation: "update", error: error.message, resource: resource.name });
|
|
14232
14393
|
}
|
|
14233
14394
|
});
|
|
14234
14395
|
resource.on("delete", async (data) => {
|
|
14235
|
-
|
|
14396
|
+
const [ok, error] = await try_fn_default(async () => {
|
|
14236
14397
|
await plugin.processReplicatorEvent("delete", resource.name, data.id, data);
|
|
14237
|
-
}
|
|
14398
|
+
});
|
|
14399
|
+
if (!ok) {
|
|
14400
|
+
if (this.config.verbose) {
|
|
14401
|
+
console.warn(`[ReplicatorPlugin] Delete event failed for resource ${resource.name}: ${error.message}`);
|
|
14402
|
+
}
|
|
14238
14403
|
this.emit("error", { operation: "delete", error: error.message, resource: resource.name });
|
|
14239
14404
|
}
|
|
14240
14405
|
});
|
|
@@ -14250,13 +14415,17 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
14250
14415
|
}
|
|
14251
14416
|
async setup(database) {
|
|
14252
14417
|
this.database = database;
|
|
14253
|
-
|
|
14418
|
+
const [initOk, initError] = await try_fn_default(async () => {
|
|
14254
14419
|
await this.initializeReplicators(database);
|
|
14255
|
-
}
|
|
14256
|
-
|
|
14257
|
-
|
|
14420
|
+
});
|
|
14421
|
+
if (!initOk) {
|
|
14422
|
+
if (this.config.verbose) {
|
|
14423
|
+
console.warn(`[ReplicatorPlugin] Replicator initialization failed: ${initError.message}`);
|
|
14424
|
+
}
|
|
14425
|
+
this.emit("error", { operation: "setup", error: initError.message });
|
|
14426
|
+
throw initError;
|
|
14258
14427
|
}
|
|
14259
|
-
|
|
14428
|
+
const [logOk, logError] = await try_fn_default(async () => {
|
|
14260
14429
|
if (this.config.replicatorLogResource) {
|
|
14261
14430
|
const logRes = await database.createResource({
|
|
14262
14431
|
name: this.config.replicatorLogResource,
|
|
@@ -14273,7 +14442,15 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
14273
14442
|
}
|
|
14274
14443
|
});
|
|
14275
14444
|
}
|
|
14276
|
-
}
|
|
14445
|
+
});
|
|
14446
|
+
if (!logOk) {
|
|
14447
|
+
if (this.config.verbose) {
|
|
14448
|
+
console.warn(`[ReplicatorPlugin] Failed to create log resource ${this.config.replicatorLogResource}: ${logError.message}`);
|
|
14449
|
+
}
|
|
14450
|
+
this.emit("replicator_log_resource_creation_error", {
|
|
14451
|
+
resourceName: this.config.replicatorLogResource,
|
|
14452
|
+
error: logError.message
|
|
14453
|
+
});
|
|
14277
14454
|
}
|
|
14278
14455
|
await this.uploadMetadataFile(database);
|
|
14279
14456
|
const originalCreateResource = database.createResource.bind(database);
|
|
@@ -14316,21 +14493,28 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
14316
14493
|
async retryWithBackoff(operation, maxRetries = 3) {
|
|
14317
14494
|
let lastError;
|
|
14318
14495
|
for (let attempt = 1; attempt <= maxRetries; attempt++) {
|
|
14319
|
-
|
|
14320
|
-
|
|
14321
|
-
|
|
14496
|
+
const [ok, error] = await try_fn_default(operation);
|
|
14497
|
+
if (ok) {
|
|
14498
|
+
return ok;
|
|
14499
|
+
} else {
|
|
14322
14500
|
lastError = error;
|
|
14501
|
+
if (this.config.verbose) {
|
|
14502
|
+
console.warn(`[ReplicatorPlugin] Retry attempt ${attempt}/${maxRetries} failed: ${error.message}`);
|
|
14503
|
+
}
|
|
14323
14504
|
if (attempt === maxRetries) {
|
|
14324
14505
|
throw error;
|
|
14325
14506
|
}
|
|
14326
14507
|
const delay = Math.pow(2, attempt - 1) * 1e3;
|
|
14508
|
+
if (this.config.verbose) {
|
|
14509
|
+
console.warn(`[ReplicatorPlugin] Waiting ${delay}ms before retry...`);
|
|
14510
|
+
}
|
|
14327
14511
|
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
14328
14512
|
}
|
|
14329
14513
|
}
|
|
14330
14514
|
throw lastError;
|
|
14331
14515
|
}
|
|
14332
14516
|
async logError(replicator, resourceName, operation, recordId, data, error) {
|
|
14333
|
-
|
|
14517
|
+
const [ok, logError] = await try_fn_default(async () => {
|
|
14334
14518
|
const logResourceName = this.config.replicatorLogResource;
|
|
14335
14519
|
if (this.database && this.database.resources && this.database.resources[logResourceName]) {
|
|
14336
14520
|
const logResource = this.database.resources[logResourceName];
|
|
@@ -14345,7 +14529,19 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
14345
14529
|
status: "error"
|
|
14346
14530
|
});
|
|
14347
14531
|
}
|
|
14348
|
-
}
|
|
14532
|
+
});
|
|
14533
|
+
if (!ok) {
|
|
14534
|
+
if (this.config.verbose) {
|
|
14535
|
+
console.warn(`[ReplicatorPlugin] Failed to log error for ${resourceName}: ${logError.message}`);
|
|
14536
|
+
}
|
|
14537
|
+
this.emit("replicator_log_error", {
|
|
14538
|
+
replicator: replicator.name || replicator.id,
|
|
14539
|
+
resourceName,
|
|
14540
|
+
operation,
|
|
14541
|
+
recordId,
|
|
14542
|
+
originalError: error.message,
|
|
14543
|
+
logError: logError.message
|
|
14544
|
+
});
|
|
14349
14545
|
}
|
|
14350
14546
|
}
|
|
14351
14547
|
async processReplicatorEvent(operation, resourceName, recordId, data, beforeData = null) {
|
|
@@ -14358,8 +14554,8 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
14358
14554
|
return;
|
|
14359
14555
|
}
|
|
14360
14556
|
const promises = applicableReplicators.map(async (replicator) => {
|
|
14361
|
-
|
|
14362
|
-
const
|
|
14557
|
+
const [ok, error, result] = await try_fn_default(async () => {
|
|
14558
|
+
const result2 = await this.retryWithBackoff(
|
|
14363
14559
|
() => replicator.replicate(resourceName, operation, data, recordId, beforeData),
|
|
14364
14560
|
this.config.maxRetries
|
|
14365
14561
|
);
|
|
@@ -14368,11 +14564,17 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
14368
14564
|
resourceName,
|
|
14369
14565
|
operation,
|
|
14370
14566
|
recordId,
|
|
14371
|
-
result,
|
|
14567
|
+
result: result2,
|
|
14372
14568
|
success: true
|
|
14373
14569
|
});
|
|
14570
|
+
return result2;
|
|
14571
|
+
});
|
|
14572
|
+
if (ok) {
|
|
14374
14573
|
return result;
|
|
14375
|
-
}
|
|
14574
|
+
} else {
|
|
14575
|
+
if (this.config.verbose) {
|
|
14576
|
+
console.warn(`[ReplicatorPlugin] Replication failed for ${replicator.name || replicator.id} on ${resourceName}: ${error.message}`);
|
|
14577
|
+
}
|
|
14376
14578
|
this.emit("replicator_error", {
|
|
14377
14579
|
replicator: replicator.name || replicator.id,
|
|
14378
14580
|
resourceName,
|
|
@@ -14397,11 +14599,14 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
14397
14599
|
return;
|
|
14398
14600
|
}
|
|
14399
14601
|
const promises = applicableReplicators.map(async (replicator) => {
|
|
14400
|
-
|
|
14602
|
+
const [wrapperOk, wrapperError] = await try_fn_default(async () => {
|
|
14401
14603
|
const [ok, err, result] = await try_fn_default(
|
|
14402
14604
|
() => replicator.replicate(item.resourceName, item.operation, item.data, item.recordId, item.beforeData)
|
|
14403
14605
|
);
|
|
14404
14606
|
if (!ok) {
|
|
14607
|
+
if (this.config.verbose) {
|
|
14608
|
+
console.warn(`[ReplicatorPlugin] Replicator item processing failed for ${replicator.name || replicator.id} on ${item.resourceName}: ${err.message}`);
|
|
14609
|
+
}
|
|
14405
14610
|
this.emit("replicator_error", {
|
|
14406
14611
|
replicator: replicator.name || replicator.id,
|
|
14407
14612
|
resourceName: item.resourceName,
|
|
@@ -14423,18 +14628,24 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
14423
14628
|
success: true
|
|
14424
14629
|
});
|
|
14425
14630
|
return { success: true, result };
|
|
14426
|
-
}
|
|
14631
|
+
});
|
|
14632
|
+
if (wrapperOk) {
|
|
14633
|
+
return wrapperOk;
|
|
14634
|
+
} else {
|
|
14635
|
+
if (this.config.verbose) {
|
|
14636
|
+
console.warn(`[ReplicatorPlugin] Wrapper processing failed for ${replicator.name || replicator.id} on ${item.resourceName}: ${wrapperError.message}`);
|
|
14637
|
+
}
|
|
14427
14638
|
this.emit("replicator_error", {
|
|
14428
14639
|
replicator: replicator.name || replicator.id,
|
|
14429
14640
|
resourceName: item.resourceName,
|
|
14430
14641
|
operation: item.operation,
|
|
14431
14642
|
recordId: item.recordId,
|
|
14432
|
-
error:
|
|
14643
|
+
error: wrapperError.message
|
|
14433
14644
|
});
|
|
14434
14645
|
if (this.config.logErrors && this.database) {
|
|
14435
|
-
await this.logError(replicator, item.resourceName, item.operation, item.recordId, item.data,
|
|
14646
|
+
await this.logError(replicator, item.resourceName, item.operation, item.recordId, item.data, wrapperError);
|
|
14436
14647
|
}
|
|
14437
|
-
return { success: false, error:
|
|
14648
|
+
return { success: false, error: wrapperError.message };
|
|
14438
14649
|
}
|
|
14439
14650
|
});
|
|
14440
14651
|
return Promise.allSettled(promises);
|
|
@@ -14456,9 +14667,13 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
14456
14667
|
timestamp: typeof item.timestamp === "number" ? item.timestamp : Date.now(),
|
|
14457
14668
|
createdAt: item.createdAt || (/* @__PURE__ */ new Date()).toISOString().slice(0, 10)
|
|
14458
14669
|
};
|
|
14459
|
-
|
|
14670
|
+
const [ok, err] = await try_fn_default(async () => {
|
|
14460
14671
|
await logRes.insert(logItem);
|
|
14461
|
-
}
|
|
14672
|
+
});
|
|
14673
|
+
if (!ok) {
|
|
14674
|
+
if (this.config.verbose) {
|
|
14675
|
+
console.warn(`[ReplicatorPlugin] Failed to log replicator item: ${err.message}`);
|
|
14676
|
+
}
|
|
14462
14677
|
this.emit("replicator.log.failed", { error: err, item });
|
|
14463
14678
|
}
|
|
14464
14679
|
}
|
|
@@ -14564,14 +14779,23 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
14564
14779
|
this.emit("replicator.sync.completed", { replicatorId, stats: this.stats });
|
|
14565
14780
|
}
|
|
14566
14781
|
async cleanup() {
|
|
14567
|
-
|
|
14782
|
+
const [ok, error] = await try_fn_default(async () => {
|
|
14568
14783
|
if (this.replicators && this.replicators.length > 0) {
|
|
14569
14784
|
const cleanupPromises = this.replicators.map(async (replicator) => {
|
|
14570
|
-
|
|
14785
|
+
const [replicatorOk, replicatorError] = await try_fn_default(async () => {
|
|
14571
14786
|
if (replicator && typeof replicator.cleanup === "function") {
|
|
14572
14787
|
await replicator.cleanup();
|
|
14573
14788
|
}
|
|
14574
|
-
}
|
|
14789
|
+
});
|
|
14790
|
+
if (!replicatorOk) {
|
|
14791
|
+
if (this.config.verbose) {
|
|
14792
|
+
console.warn(`[ReplicatorPlugin] Failed to cleanup replicator ${replicator.name || replicator.id}: ${replicatorError.message}`);
|
|
14793
|
+
}
|
|
14794
|
+
this.emit("replicator_cleanup_error", {
|
|
14795
|
+
replicator: replicator.name || replicator.id || "unknown",
|
|
14796
|
+
driver: replicator.driver || "unknown",
|
|
14797
|
+
error: replicatorError.message
|
|
14798
|
+
});
|
|
14575
14799
|
}
|
|
14576
14800
|
});
|
|
14577
14801
|
await Promise.allSettled(cleanupPromises);
|
|
@@ -14580,7 +14804,14 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
14580
14804
|
this.database = null;
|
|
14581
14805
|
this.eventListenersInstalled.clear();
|
|
14582
14806
|
this.removeAllListeners();
|
|
14583
|
-
}
|
|
14807
|
+
});
|
|
14808
|
+
if (!ok) {
|
|
14809
|
+
if (this.config.verbose) {
|
|
14810
|
+
console.warn(`[ReplicatorPlugin] Failed to cleanup plugin: ${error.message}`);
|
|
14811
|
+
}
|
|
14812
|
+
this.emit("replicator_plugin_cleanup_error", {
|
|
14813
|
+
error: error.message
|
|
14814
|
+
});
|
|
14584
14815
|
}
|
|
14585
14816
|
}
|
|
14586
14817
|
}
|