s3db.js 7.3.5 → 7.3.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/PLUGINS.md +1285 -157
- package/dist/s3db.cjs.js +296 -83
- package/dist/s3db.cjs.min.js +1 -1
- package/dist/s3db.es.js +296 -83
- package/dist/s3db.es.min.js +1 -1
- package/dist/s3db.iife.js +296 -83
- package/dist/s3db.iife.min.js +1 -1
- package/mcp/server.js +1410 -0
- package/package.json +7 -1
- package/src/plugins/cache/filesystem-cache.class.js +9 -0
- package/src/plugins/replicator.plugin.js +130 -46
- package/src/plugins/replicators/bigquery-replicator.class.js +31 -5
- package/src/plugins/replicators/postgres-replicator.class.js +17 -2
- package/src/plugins/replicators/s3db-replicator.class.js +172 -68
- package/src/plugins/replicators/sqs-replicator.class.js +13 -1
package/dist/s3db.iife.js
CHANGED
|
@@ -7070,6 +7070,12 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
7070
7070
|
}
|
|
7071
7071
|
async _clear(prefix) {
|
|
7072
7072
|
try {
|
|
7073
|
+
if (!await this._fileExists(this.directory)) {
|
|
7074
|
+
if (this.enableStats) {
|
|
7075
|
+
this.stats.clears++;
|
|
7076
|
+
}
|
|
7077
|
+
return true;
|
|
7078
|
+
}
|
|
7073
7079
|
const files = await promises.readdir(this.directory);
|
|
7074
7080
|
const cacheFiles = files.filter((file) => {
|
|
7075
7081
|
if (!file.startsWith(this.prefix)) return false;
|
|
@@ -8941,6 +8947,9 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
8941
8947
|
await super.initialize(database);
|
|
8942
8948
|
const [ok, err, sdk] = await try_fn_default(() => import('@google-cloud/bigquery'));
|
|
8943
8949
|
if (!ok) {
|
|
8950
|
+
if (this.config.verbose) {
|
|
8951
|
+
console.warn(`[BigqueryReplicator] Failed to import BigQuery SDK: ${err.message}`);
|
|
8952
|
+
}
|
|
8944
8953
|
this.emit("initialization_error", { replicator: this.name, error: err.message });
|
|
8945
8954
|
throw err;
|
|
8946
8955
|
}
|
|
@@ -9010,19 +9019,28 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
9010
9019
|
const maxRetries = 2;
|
|
9011
9020
|
let lastError = null;
|
|
9012
9021
|
for (let attempt = 1; attempt <= maxRetries; attempt++) {
|
|
9013
|
-
|
|
9022
|
+
const [ok2, error] = await try_fn_default(async () => {
|
|
9014
9023
|
const [updateJob] = await this.bigqueryClient.createQueryJob({
|
|
9015
9024
|
query,
|
|
9016
9025
|
params,
|
|
9017
9026
|
location: this.location
|
|
9018
9027
|
});
|
|
9019
9028
|
await updateJob.getQueryResults();
|
|
9020
|
-
|
|
9029
|
+
return [updateJob];
|
|
9030
|
+
});
|
|
9031
|
+
if (ok2) {
|
|
9032
|
+
job = ok2;
|
|
9021
9033
|
break;
|
|
9022
|
-
}
|
|
9034
|
+
} else {
|
|
9023
9035
|
lastError = error;
|
|
9036
|
+
if (this.config.verbose) {
|
|
9037
|
+
console.warn(`[BigqueryReplicator] Update attempt ${attempt} failed: ${error.message}`);
|
|
9038
|
+
}
|
|
9024
9039
|
if (error?.message?.includes("streaming buffer") && attempt < maxRetries) {
|
|
9025
9040
|
const delaySeconds = 30;
|
|
9041
|
+
if (this.config.verbose) {
|
|
9042
|
+
console.warn(`[BigqueryReplicator] Retrying in ${delaySeconds} seconds due to streaming buffer issue`);
|
|
9043
|
+
}
|
|
9026
9044
|
await new Promise((resolve) => setTimeout(resolve, delaySeconds * 1e3));
|
|
9027
9045
|
continue;
|
|
9028
9046
|
}
|
|
@@ -9089,6 +9107,9 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
9089
9107
|
};
|
|
9090
9108
|
});
|
|
9091
9109
|
if (ok) return result;
|
|
9110
|
+
if (this.config.verbose) {
|
|
9111
|
+
console.warn(`[BigqueryReplicator] Replication failed for ${resourceName}: ${err.message}`);
|
|
9112
|
+
}
|
|
9092
9113
|
this.emit("replicator_error", {
|
|
9093
9114
|
replicator: this.name,
|
|
9094
9115
|
resourceName,
|
|
@@ -9109,8 +9130,14 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
9109
9130
|
record.id,
|
|
9110
9131
|
record.beforeData
|
|
9111
9132
|
));
|
|
9112
|
-
if (ok)
|
|
9113
|
-
|
|
9133
|
+
if (ok) {
|
|
9134
|
+
results.push(res);
|
|
9135
|
+
} else {
|
|
9136
|
+
if (this.config.verbose) {
|
|
9137
|
+
console.warn(`[BigqueryReplicator] Batch replication failed for record ${record.id}: ${err.message}`);
|
|
9138
|
+
}
|
|
9139
|
+
errors.push({ id: record.id, error: err.message });
|
|
9140
|
+
}
|
|
9114
9141
|
}
|
|
9115
9142
|
return {
|
|
9116
9143
|
success: errors.length === 0,
|
|
@@ -9126,6 +9153,9 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
9126
9153
|
return true;
|
|
9127
9154
|
});
|
|
9128
9155
|
if (ok) return true;
|
|
9156
|
+
if (this.config.verbose) {
|
|
9157
|
+
console.warn(`[BigqueryReplicator] Connection test failed: ${err.message}`);
|
|
9158
|
+
}
|
|
9129
9159
|
this.emit("connection_error", { replicator: this.name, error: err.message });
|
|
9130
9160
|
return false;
|
|
9131
9161
|
}
|
|
@@ -9213,6 +9243,9 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
9213
9243
|
await super.initialize(database);
|
|
9214
9244
|
const [ok, err, sdk] = await try_fn_default(() => import('pg'));
|
|
9215
9245
|
if (!ok) {
|
|
9246
|
+
if (this.config.verbose) {
|
|
9247
|
+
console.warn(`[PostgresReplicator] Failed to import pg SDK: ${err.message}`);
|
|
9248
|
+
}
|
|
9216
9249
|
this.emit("initialization_error", {
|
|
9217
9250
|
replicator: this.name,
|
|
9218
9251
|
error: err.message
|
|
@@ -9354,6 +9387,9 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
9354
9387
|
};
|
|
9355
9388
|
});
|
|
9356
9389
|
if (ok) return result;
|
|
9390
|
+
if (this.config.verbose) {
|
|
9391
|
+
console.warn(`[PostgresReplicator] Replication failed for ${resourceName}: ${err.message}`);
|
|
9392
|
+
}
|
|
9357
9393
|
this.emit("replicator_error", {
|
|
9358
9394
|
replicator: this.name,
|
|
9359
9395
|
resourceName,
|
|
@@ -9374,8 +9410,14 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
9374
9410
|
record.id,
|
|
9375
9411
|
record.beforeData
|
|
9376
9412
|
));
|
|
9377
|
-
if (ok)
|
|
9378
|
-
|
|
9413
|
+
if (ok) {
|
|
9414
|
+
results.push(res);
|
|
9415
|
+
} else {
|
|
9416
|
+
if (this.config.verbose) {
|
|
9417
|
+
console.warn(`[PostgresReplicator] Batch replication failed for record ${record.id}: ${err.message}`);
|
|
9418
|
+
}
|
|
9419
|
+
errors.push({ id: record.id, error: err.message });
|
|
9420
|
+
}
|
|
9379
9421
|
}
|
|
9380
9422
|
return {
|
|
9381
9423
|
success: errors.length === 0,
|
|
@@ -9390,6 +9432,9 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
9390
9432
|
return true;
|
|
9391
9433
|
});
|
|
9392
9434
|
if (ok) return true;
|
|
9435
|
+
if (this.config.verbose) {
|
|
9436
|
+
console.warn(`[PostgresReplicator] Connection test failed: ${err.message}`);
|
|
9437
|
+
}
|
|
9393
9438
|
this.emit("connection_error", { replicator: this.name, error: err.message });
|
|
9394
9439
|
return false;
|
|
9395
9440
|
}
|
|
@@ -13124,7 +13169,7 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
13124
13169
|
super();
|
|
13125
13170
|
this.version = "1";
|
|
13126
13171
|
this.s3dbVersion = (() => {
|
|
13127
|
-
const [ok, err, version] = try_fn_default(() => true ? "7.3.
|
|
13172
|
+
const [ok, err, version] = try_fn_default(() => true ? "7.3.6" : "latest");
|
|
13128
13173
|
return ok ? version : "latest";
|
|
13129
13174
|
})();
|
|
13130
13175
|
this.resources = {};
|
|
@@ -13628,9 +13673,8 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
13628
13673
|
const map = {};
|
|
13629
13674
|
for (const res of resources) {
|
|
13630
13675
|
if (typeof res === "string") map[normalizeResourceName$1(res)] = res;
|
|
13631
|
-
else if (Array.isArray(res) && typeof res[0] === "string") map[normalizeResourceName$1(res[0])] = res;
|
|
13632
13676
|
else if (typeof res === "object" && res.resource) {
|
|
13633
|
-
map[normalizeResourceName$1(res.resource)] =
|
|
13677
|
+
map[normalizeResourceName$1(res.resource)] = res;
|
|
13634
13678
|
}
|
|
13635
13679
|
}
|
|
13636
13680
|
return map;
|
|
@@ -13643,15 +13687,14 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
13643
13687
|
else if (Array.isArray(dest)) {
|
|
13644
13688
|
map[normSrc] = dest.map((item) => {
|
|
13645
13689
|
if (typeof item === "string") return item;
|
|
13646
|
-
if (typeof item === "function") return item;
|
|
13647
13690
|
if (typeof item === "object" && item.resource) {
|
|
13648
|
-
return
|
|
13691
|
+
return item;
|
|
13649
13692
|
}
|
|
13650
13693
|
return item;
|
|
13651
13694
|
});
|
|
13652
13695
|
} else if (typeof dest === "function") map[normSrc] = dest;
|
|
13653
13696
|
else if (typeof dest === "object" && dest.resource) {
|
|
13654
|
-
map[normSrc] =
|
|
13697
|
+
map[normSrc] = dest;
|
|
13655
13698
|
}
|
|
13656
13699
|
}
|
|
13657
13700
|
return map;
|
|
@@ -13659,10 +13702,6 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
13659
13702
|
if (typeof resources === "function") {
|
|
13660
13703
|
return resources;
|
|
13661
13704
|
}
|
|
13662
|
-
if (typeof resources === "string") {
|
|
13663
|
-
const map = { [normalizeResourceName$1(resources)]: resources };
|
|
13664
|
-
return map;
|
|
13665
|
-
}
|
|
13666
13705
|
return {};
|
|
13667
13706
|
}
|
|
13668
13707
|
validateConfig() {
|
|
@@ -13676,8 +13715,8 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
13676
13715
|
return { isValid: errors.length === 0, errors };
|
|
13677
13716
|
}
|
|
13678
13717
|
async initialize(database) {
|
|
13679
|
-
|
|
13680
|
-
|
|
13718
|
+
await super.initialize(database);
|
|
13719
|
+
const [ok, err] = await try_fn_default(async () => {
|
|
13681
13720
|
if (this.client) {
|
|
13682
13721
|
this.targetDatabase = this.client;
|
|
13683
13722
|
} else if (this.connectionString) {
|
|
@@ -13696,7 +13735,11 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
13696
13735
|
replicator: this.name,
|
|
13697
13736
|
target: this.connectionString || "client-provided"
|
|
13698
13737
|
});
|
|
13699
|
-
}
|
|
13738
|
+
});
|
|
13739
|
+
if (!ok) {
|
|
13740
|
+
if (this.config.verbose) {
|
|
13741
|
+
console.warn(`[S3dbReplicator] Initialization failed: ${err.message}`);
|
|
13742
|
+
}
|
|
13700
13743
|
throw err;
|
|
13701
13744
|
}
|
|
13702
13745
|
}
|
|
@@ -13715,18 +13758,77 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
13715
13758
|
id = recordId;
|
|
13716
13759
|
}
|
|
13717
13760
|
const normResource = normalizeResourceName$1(resource);
|
|
13718
|
-
const
|
|
13719
|
-
|
|
13720
|
-
|
|
13761
|
+
const entry = this.resourcesMap[normResource];
|
|
13762
|
+
if (!entry) {
|
|
13763
|
+
throw new Error(`[S3dbReplicator] Resource not configured: ${resource}`);
|
|
13764
|
+
}
|
|
13765
|
+
if (Array.isArray(entry)) {
|
|
13766
|
+
const results = [];
|
|
13767
|
+
for (const destConfig of entry) {
|
|
13768
|
+
const [ok, error, result] = await try_fn_default(async () => {
|
|
13769
|
+
return await this._replicateToSingleDestination(destConfig, normResource, op, payload, id);
|
|
13770
|
+
});
|
|
13771
|
+
if (!ok) {
|
|
13772
|
+
if (this.config && this.config.verbose) {
|
|
13773
|
+
console.warn(`[S3dbReplicator] Failed to replicate to destination ${JSON.stringify(destConfig)}: ${error.message}`);
|
|
13774
|
+
}
|
|
13775
|
+
throw error;
|
|
13776
|
+
}
|
|
13777
|
+
results.push(result);
|
|
13778
|
+
}
|
|
13779
|
+
return results;
|
|
13780
|
+
} else {
|
|
13781
|
+
const [ok, error, result] = await try_fn_default(async () => {
|
|
13782
|
+
return await this._replicateToSingleDestination(entry, normResource, op, payload, id);
|
|
13783
|
+
});
|
|
13784
|
+
if (!ok) {
|
|
13785
|
+
if (this.config && this.config.verbose) {
|
|
13786
|
+
console.warn(`[S3dbReplicator] Failed to replicate to destination ${JSON.stringify(entry)}: ${error.message}`);
|
|
13787
|
+
}
|
|
13788
|
+
throw error;
|
|
13789
|
+
}
|
|
13790
|
+
return result;
|
|
13791
|
+
}
|
|
13792
|
+
}
|
|
13793
|
+
async _replicateToSingleDestination(destConfig, sourceResource, operation, data, recordId) {
|
|
13794
|
+
let destResourceName;
|
|
13795
|
+
if (typeof destConfig === "string") {
|
|
13796
|
+
destResourceName = destConfig;
|
|
13797
|
+
} else if (typeof destConfig === "object" && destConfig.resource) {
|
|
13798
|
+
destResourceName = destConfig.resource;
|
|
13799
|
+
} else {
|
|
13800
|
+
destResourceName = sourceResource;
|
|
13801
|
+
}
|
|
13802
|
+
if (typeof destConfig === "object" && destConfig.actions && Array.isArray(destConfig.actions)) {
|
|
13803
|
+
if (!destConfig.actions.includes(operation)) {
|
|
13804
|
+
return { skipped: true, reason: "action_not_supported", action: operation, destination: destResourceName };
|
|
13805
|
+
}
|
|
13806
|
+
}
|
|
13807
|
+
const destResourceObj = this._getDestResourceObj(destResourceName);
|
|
13808
|
+
let transformedData;
|
|
13809
|
+
if (typeof destConfig === "object" && destConfig.transform && typeof destConfig.transform === "function") {
|
|
13810
|
+
transformedData = destConfig.transform(data);
|
|
13811
|
+
if (transformedData && data && data.id && !transformedData.id) {
|
|
13812
|
+
transformedData.id = data.id;
|
|
13813
|
+
}
|
|
13814
|
+
} else if (typeof destConfig === "object" && destConfig.transformer && typeof destConfig.transformer === "function") {
|
|
13815
|
+
transformedData = destConfig.transformer(data);
|
|
13816
|
+
if (transformedData && data && data.id && !transformedData.id) {
|
|
13817
|
+
transformedData.id = data.id;
|
|
13818
|
+
}
|
|
13819
|
+
} else {
|
|
13820
|
+
transformedData = data;
|
|
13821
|
+
}
|
|
13822
|
+
if (!transformedData && data) transformedData = data;
|
|
13721
13823
|
let result;
|
|
13722
|
-
if (
|
|
13824
|
+
if (operation === "insert") {
|
|
13723
13825
|
result = await destResourceObj.insert(transformedData);
|
|
13724
|
-
} else if (
|
|
13725
|
-
result = await destResourceObj.update(
|
|
13726
|
-
} else if (
|
|
13727
|
-
result = await destResourceObj.delete(
|
|
13826
|
+
} else if (operation === "update") {
|
|
13827
|
+
result = await destResourceObj.update(recordId, transformedData);
|
|
13828
|
+
} else if (operation === "delete") {
|
|
13829
|
+
result = await destResourceObj.delete(recordId);
|
|
13728
13830
|
} else {
|
|
13729
|
-
throw new Error(`Invalid operation: ${
|
|
13831
|
+
throw new Error(`Invalid operation: ${operation}. Supported operations are: insert, update, delete`);
|
|
13730
13832
|
}
|
|
13731
13833
|
return result;
|
|
13732
13834
|
}
|
|
@@ -13735,13 +13837,25 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
13735
13837
|
const entry = this.resourcesMap[normResource];
|
|
13736
13838
|
let result;
|
|
13737
13839
|
if (!entry) return data;
|
|
13738
|
-
if (Array.isArray(entry)
|
|
13739
|
-
|
|
13840
|
+
if (Array.isArray(entry)) {
|
|
13841
|
+
for (const item of entry) {
|
|
13842
|
+
if (typeof item === "object" && item.transform && typeof item.transform === "function") {
|
|
13843
|
+
result = item.transform(data);
|
|
13844
|
+
break;
|
|
13845
|
+
} else if (typeof item === "object" && item.transformer && typeof item.transformer === "function") {
|
|
13846
|
+
result = item.transformer(data);
|
|
13847
|
+
break;
|
|
13848
|
+
}
|
|
13849
|
+
}
|
|
13850
|
+
if (!result) result = data;
|
|
13851
|
+
} else if (typeof entry === "object") {
|
|
13852
|
+
if (typeof entry.transform === "function") {
|
|
13853
|
+
result = entry.transform(data);
|
|
13854
|
+
} else if (typeof entry.transformer === "function") {
|
|
13855
|
+
result = entry.transformer(data);
|
|
13856
|
+
}
|
|
13740
13857
|
} else if (typeof entry === "function") {
|
|
13741
13858
|
result = entry(data);
|
|
13742
|
-
} else if (typeof entry === "object") {
|
|
13743
|
-
if (typeof entry.transform === "function") result = entry.transform(data);
|
|
13744
|
-
else if (typeof entry.transformer === "function") result = entry.transformer(data);
|
|
13745
13859
|
} else {
|
|
13746
13860
|
result = data;
|
|
13747
13861
|
}
|
|
@@ -13754,9 +13868,11 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
13754
13868
|
const entry = this.resourcesMap[normResource];
|
|
13755
13869
|
if (!entry) return resource;
|
|
13756
13870
|
if (Array.isArray(entry)) {
|
|
13757
|
-
|
|
13758
|
-
|
|
13759
|
-
|
|
13871
|
+
for (const item of entry) {
|
|
13872
|
+
if (typeof item === "string") return item;
|
|
13873
|
+
if (typeof item === "object" && item.resource) return item.resource;
|
|
13874
|
+
}
|
|
13875
|
+
return resource;
|
|
13760
13876
|
}
|
|
13761
13877
|
if (typeof entry === "string") return entry;
|
|
13762
13878
|
if (typeof entry === "function") return resource;
|
|
@@ -13764,8 +13880,7 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
13764
13880
|
return resource;
|
|
13765
13881
|
}
|
|
13766
13882
|
_getDestResourceObj(resource) {
|
|
13767
|
-
|
|
13768
|
-
const available = Object.keys(this.client.resources);
|
|
13883
|
+
const available = Object.keys(this.client.resources || {});
|
|
13769
13884
|
const norm = normalizeResourceName$1(resource);
|
|
13770
13885
|
const found = available.find((r) => normalizeResourceName$1(r) === norm);
|
|
13771
13886
|
if (!found) {
|
|
@@ -13787,8 +13902,14 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
13787
13902
|
data: record.data,
|
|
13788
13903
|
beforeData: record.beforeData
|
|
13789
13904
|
}));
|
|
13790
|
-
if (ok)
|
|
13791
|
-
|
|
13905
|
+
if (ok) {
|
|
13906
|
+
results.push(result);
|
|
13907
|
+
} else {
|
|
13908
|
+
if (this.config.verbose) {
|
|
13909
|
+
console.warn(`[S3dbReplicator] Batch replication failed for record ${record.id}: ${err.message}`);
|
|
13910
|
+
}
|
|
13911
|
+
errors.push({ id: record.id, error: err.message });
|
|
13912
|
+
}
|
|
13792
13913
|
}
|
|
13793
13914
|
this.emit("batch_replicated", {
|
|
13794
13915
|
replicator: this.name,
|
|
@@ -13806,18 +13927,20 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
13806
13927
|
}
|
|
13807
13928
|
async testConnection() {
|
|
13808
13929
|
const [ok, err] = await try_fn_default(async () => {
|
|
13809
|
-
if (!this.targetDatabase)
|
|
13810
|
-
|
|
13930
|
+
if (!this.targetDatabase) throw new Error("No target database configured");
|
|
13931
|
+
if (typeof this.targetDatabase.connect === "function") {
|
|
13932
|
+
await this.targetDatabase.connect();
|
|
13811
13933
|
}
|
|
13812
|
-
await this.targetDatabase.listResources();
|
|
13813
13934
|
return true;
|
|
13814
13935
|
});
|
|
13815
|
-
if (ok)
|
|
13816
|
-
|
|
13817
|
-
|
|
13818
|
-
|
|
13819
|
-
|
|
13820
|
-
|
|
13936
|
+
if (!ok) {
|
|
13937
|
+
if (this.config.verbose) {
|
|
13938
|
+
console.warn(`[S3dbReplicator] Connection test failed: ${err.message}`);
|
|
13939
|
+
}
|
|
13940
|
+
this.emit("connection_error", { replicator: this.name, error: err.message });
|
|
13941
|
+
return false;
|
|
13942
|
+
}
|
|
13943
|
+
return true;
|
|
13821
13944
|
}
|
|
13822
13945
|
async getStatus() {
|
|
13823
13946
|
const baseStatus = await super.getStatus();
|
|
@@ -13849,7 +13972,7 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
13849
13972
|
} else {
|
|
13850
13973
|
return true;
|
|
13851
13974
|
}
|
|
13852
|
-
} else if (typeof item === "string"
|
|
13975
|
+
} else if (typeof item === "string") {
|
|
13853
13976
|
return true;
|
|
13854
13977
|
}
|
|
13855
13978
|
}
|
|
@@ -13976,6 +14099,9 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
13976
14099
|
if (!this.sqsClient) {
|
|
13977
14100
|
const [ok, err, sdk] = await try_fn_default(() => import('@aws-sdk/client-sqs'));
|
|
13978
14101
|
if (!ok) {
|
|
14102
|
+
if (this.config.verbose) {
|
|
14103
|
+
console.warn(`[SqsReplicator] Failed to import SQS SDK: ${err.message}`);
|
|
14104
|
+
}
|
|
13979
14105
|
this.emit("initialization_error", {
|
|
13980
14106
|
replicator: this.name,
|
|
13981
14107
|
error: err.message
|
|
@@ -14027,6 +14153,9 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
14027
14153
|
return { success: true, results };
|
|
14028
14154
|
});
|
|
14029
14155
|
if (ok) return result;
|
|
14156
|
+
if (this.config.verbose) {
|
|
14157
|
+
console.warn(`[SqsReplicator] Replication failed for ${resource}: ${err.message}`);
|
|
14158
|
+
}
|
|
14030
14159
|
this.emit("replicator_error", {
|
|
14031
14160
|
replicator: this.name,
|
|
14032
14161
|
resource,
|
|
@@ -14099,6 +14228,9 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
14099
14228
|
});
|
|
14100
14229
|
if (ok) return result;
|
|
14101
14230
|
const errorMessage = err?.message || err || "Unknown error";
|
|
14231
|
+
if (this.config.verbose) {
|
|
14232
|
+
console.warn(`[SqsReplicator] Batch replication failed for ${resource}: ${errorMessage}`);
|
|
14233
|
+
}
|
|
14102
14234
|
this.emit("batch_replicator_error", {
|
|
14103
14235
|
replicator: this.name,
|
|
14104
14236
|
resource,
|
|
@@ -14120,6 +14252,9 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
14120
14252
|
return true;
|
|
14121
14253
|
});
|
|
14122
14254
|
if (ok) return true;
|
|
14255
|
+
if (this.config.verbose) {
|
|
14256
|
+
console.warn(`[SqsReplicator] Connection test failed: ${err.message}`);
|
|
14257
|
+
}
|
|
14123
14258
|
this.emit("connection_error", {
|
|
14124
14259
|
replicator: this.name,
|
|
14125
14260
|
error: err.message
|
|
@@ -14216,25 +14351,37 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
14216
14351
|
return;
|
|
14217
14352
|
}
|
|
14218
14353
|
resource.on("insert", async (data) => {
|
|
14219
|
-
|
|
14354
|
+
const [ok, error] = await try_fn_default(async () => {
|
|
14220
14355
|
const completeData = { ...data, createdAt: (/* @__PURE__ */ new Date()).toISOString() };
|
|
14221
14356
|
await plugin.processReplicatorEvent("insert", resource.name, completeData.id, completeData);
|
|
14222
|
-
}
|
|
14357
|
+
});
|
|
14358
|
+
if (!ok) {
|
|
14359
|
+
if (this.config.verbose) {
|
|
14360
|
+
console.warn(`[ReplicatorPlugin] Insert event failed for resource ${resource.name}: ${error.message}`);
|
|
14361
|
+
}
|
|
14223
14362
|
this.emit("error", { operation: "insert", error: error.message, resource: resource.name });
|
|
14224
14363
|
}
|
|
14225
14364
|
});
|
|
14226
14365
|
resource.on("update", async (data, beforeData) => {
|
|
14227
|
-
|
|
14366
|
+
const [ok, error] = await try_fn_default(async () => {
|
|
14228
14367
|
const completeData = { ...data, updatedAt: (/* @__PURE__ */ new Date()).toISOString() };
|
|
14229
14368
|
await plugin.processReplicatorEvent("update", resource.name, completeData.id, completeData, beforeData);
|
|
14230
|
-
}
|
|
14369
|
+
});
|
|
14370
|
+
if (!ok) {
|
|
14371
|
+
if (this.config.verbose) {
|
|
14372
|
+
console.warn(`[ReplicatorPlugin] Update event failed for resource ${resource.name}: ${error.message}`);
|
|
14373
|
+
}
|
|
14231
14374
|
this.emit("error", { operation: "update", error: error.message, resource: resource.name });
|
|
14232
14375
|
}
|
|
14233
14376
|
});
|
|
14234
14377
|
resource.on("delete", async (data) => {
|
|
14235
|
-
|
|
14378
|
+
const [ok, error] = await try_fn_default(async () => {
|
|
14236
14379
|
await plugin.processReplicatorEvent("delete", resource.name, data.id, data);
|
|
14237
|
-
}
|
|
14380
|
+
});
|
|
14381
|
+
if (!ok) {
|
|
14382
|
+
if (this.config.verbose) {
|
|
14383
|
+
console.warn(`[ReplicatorPlugin] Delete event failed for resource ${resource.name}: ${error.message}`);
|
|
14384
|
+
}
|
|
14238
14385
|
this.emit("error", { operation: "delete", error: error.message, resource: resource.name });
|
|
14239
14386
|
}
|
|
14240
14387
|
});
|
|
@@ -14250,13 +14397,17 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
14250
14397
|
}
|
|
14251
14398
|
async setup(database) {
|
|
14252
14399
|
this.database = database;
|
|
14253
|
-
|
|
14400
|
+
const [initOk, initError] = await try_fn_default(async () => {
|
|
14254
14401
|
await this.initializeReplicators(database);
|
|
14255
|
-
}
|
|
14256
|
-
|
|
14257
|
-
|
|
14402
|
+
});
|
|
14403
|
+
if (!initOk) {
|
|
14404
|
+
if (this.config.verbose) {
|
|
14405
|
+
console.warn(`[ReplicatorPlugin] Replicator initialization failed: ${initError.message}`);
|
|
14406
|
+
}
|
|
14407
|
+
this.emit("error", { operation: "setup", error: initError.message });
|
|
14408
|
+
throw initError;
|
|
14258
14409
|
}
|
|
14259
|
-
|
|
14410
|
+
const [logOk, logError] = await try_fn_default(async () => {
|
|
14260
14411
|
if (this.config.replicatorLogResource) {
|
|
14261
14412
|
const logRes = await database.createResource({
|
|
14262
14413
|
name: this.config.replicatorLogResource,
|
|
@@ -14273,7 +14424,15 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
14273
14424
|
}
|
|
14274
14425
|
});
|
|
14275
14426
|
}
|
|
14276
|
-
}
|
|
14427
|
+
});
|
|
14428
|
+
if (!logOk) {
|
|
14429
|
+
if (this.config.verbose) {
|
|
14430
|
+
console.warn(`[ReplicatorPlugin] Failed to create log resource ${this.config.replicatorLogResource}: ${logError.message}`);
|
|
14431
|
+
}
|
|
14432
|
+
this.emit("replicator_log_resource_creation_error", {
|
|
14433
|
+
resourceName: this.config.replicatorLogResource,
|
|
14434
|
+
error: logError.message
|
|
14435
|
+
});
|
|
14277
14436
|
}
|
|
14278
14437
|
await this.uploadMetadataFile(database);
|
|
14279
14438
|
const originalCreateResource = database.createResource.bind(database);
|
|
@@ -14316,21 +14475,28 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
14316
14475
|
async retryWithBackoff(operation, maxRetries = 3) {
|
|
14317
14476
|
let lastError;
|
|
14318
14477
|
for (let attempt = 1; attempt <= maxRetries; attempt++) {
|
|
14319
|
-
|
|
14320
|
-
|
|
14321
|
-
|
|
14478
|
+
const [ok, error] = await try_fn_default(operation);
|
|
14479
|
+
if (ok) {
|
|
14480
|
+
return ok;
|
|
14481
|
+
} else {
|
|
14322
14482
|
lastError = error;
|
|
14483
|
+
if (this.config.verbose) {
|
|
14484
|
+
console.warn(`[ReplicatorPlugin] Retry attempt ${attempt}/${maxRetries} failed: ${error.message}`);
|
|
14485
|
+
}
|
|
14323
14486
|
if (attempt === maxRetries) {
|
|
14324
14487
|
throw error;
|
|
14325
14488
|
}
|
|
14326
14489
|
const delay = Math.pow(2, attempt - 1) * 1e3;
|
|
14490
|
+
if (this.config.verbose) {
|
|
14491
|
+
console.warn(`[ReplicatorPlugin] Waiting ${delay}ms before retry...`);
|
|
14492
|
+
}
|
|
14327
14493
|
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
14328
14494
|
}
|
|
14329
14495
|
}
|
|
14330
14496
|
throw lastError;
|
|
14331
14497
|
}
|
|
14332
14498
|
async logError(replicator, resourceName, operation, recordId, data, error) {
|
|
14333
|
-
|
|
14499
|
+
const [ok, logError] = await try_fn_default(async () => {
|
|
14334
14500
|
const logResourceName = this.config.replicatorLogResource;
|
|
14335
14501
|
if (this.database && this.database.resources && this.database.resources[logResourceName]) {
|
|
14336
14502
|
const logResource = this.database.resources[logResourceName];
|
|
@@ -14345,7 +14511,19 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
14345
14511
|
status: "error"
|
|
14346
14512
|
});
|
|
14347
14513
|
}
|
|
14348
|
-
}
|
|
14514
|
+
});
|
|
14515
|
+
if (!ok) {
|
|
14516
|
+
if (this.config.verbose) {
|
|
14517
|
+
console.warn(`[ReplicatorPlugin] Failed to log error for ${resourceName}: ${logError.message}`);
|
|
14518
|
+
}
|
|
14519
|
+
this.emit("replicator_log_error", {
|
|
14520
|
+
replicator: replicator.name || replicator.id,
|
|
14521
|
+
resourceName,
|
|
14522
|
+
operation,
|
|
14523
|
+
recordId,
|
|
14524
|
+
originalError: error.message,
|
|
14525
|
+
logError: logError.message
|
|
14526
|
+
});
|
|
14349
14527
|
}
|
|
14350
14528
|
}
|
|
14351
14529
|
async processReplicatorEvent(operation, resourceName, recordId, data, beforeData = null) {
|
|
@@ -14358,8 +14536,8 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
14358
14536
|
return;
|
|
14359
14537
|
}
|
|
14360
14538
|
const promises = applicableReplicators.map(async (replicator) => {
|
|
14361
|
-
|
|
14362
|
-
const
|
|
14539
|
+
const [ok, error, result] = await try_fn_default(async () => {
|
|
14540
|
+
const result2 = await this.retryWithBackoff(
|
|
14363
14541
|
() => replicator.replicate(resourceName, operation, data, recordId, beforeData),
|
|
14364
14542
|
this.config.maxRetries
|
|
14365
14543
|
);
|
|
@@ -14368,11 +14546,17 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
14368
14546
|
resourceName,
|
|
14369
14547
|
operation,
|
|
14370
14548
|
recordId,
|
|
14371
|
-
result,
|
|
14549
|
+
result: result2,
|
|
14372
14550
|
success: true
|
|
14373
14551
|
});
|
|
14552
|
+
return result2;
|
|
14553
|
+
});
|
|
14554
|
+
if (ok) {
|
|
14374
14555
|
return result;
|
|
14375
|
-
}
|
|
14556
|
+
} else {
|
|
14557
|
+
if (this.config.verbose) {
|
|
14558
|
+
console.warn(`[ReplicatorPlugin] Replication failed for ${replicator.name || replicator.id} on ${resourceName}: ${error.message}`);
|
|
14559
|
+
}
|
|
14376
14560
|
this.emit("replicator_error", {
|
|
14377
14561
|
replicator: replicator.name || replicator.id,
|
|
14378
14562
|
resourceName,
|
|
@@ -14397,11 +14581,14 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
14397
14581
|
return;
|
|
14398
14582
|
}
|
|
14399
14583
|
const promises = applicableReplicators.map(async (replicator) => {
|
|
14400
|
-
|
|
14584
|
+
const [wrapperOk, wrapperError] = await try_fn_default(async () => {
|
|
14401
14585
|
const [ok, err, result] = await try_fn_default(
|
|
14402
14586
|
() => replicator.replicate(item.resourceName, item.operation, item.data, item.recordId, item.beforeData)
|
|
14403
14587
|
);
|
|
14404
14588
|
if (!ok) {
|
|
14589
|
+
if (this.config.verbose) {
|
|
14590
|
+
console.warn(`[ReplicatorPlugin] Replicator item processing failed for ${replicator.name || replicator.id} on ${item.resourceName}: ${err.message}`);
|
|
14591
|
+
}
|
|
14405
14592
|
this.emit("replicator_error", {
|
|
14406
14593
|
replicator: replicator.name || replicator.id,
|
|
14407
14594
|
resourceName: item.resourceName,
|
|
@@ -14423,18 +14610,24 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
14423
14610
|
success: true
|
|
14424
14611
|
});
|
|
14425
14612
|
return { success: true, result };
|
|
14426
|
-
}
|
|
14613
|
+
});
|
|
14614
|
+
if (wrapperOk) {
|
|
14615
|
+
return wrapperOk;
|
|
14616
|
+
} else {
|
|
14617
|
+
if (this.config.verbose) {
|
|
14618
|
+
console.warn(`[ReplicatorPlugin] Wrapper processing failed for ${replicator.name || replicator.id} on ${item.resourceName}: ${wrapperError.message}`);
|
|
14619
|
+
}
|
|
14427
14620
|
this.emit("replicator_error", {
|
|
14428
14621
|
replicator: replicator.name || replicator.id,
|
|
14429
14622
|
resourceName: item.resourceName,
|
|
14430
14623
|
operation: item.operation,
|
|
14431
14624
|
recordId: item.recordId,
|
|
14432
|
-
error:
|
|
14625
|
+
error: wrapperError.message
|
|
14433
14626
|
});
|
|
14434
14627
|
if (this.config.logErrors && this.database) {
|
|
14435
|
-
await this.logError(replicator, item.resourceName, item.operation, item.recordId, item.data,
|
|
14628
|
+
await this.logError(replicator, item.resourceName, item.operation, item.recordId, item.data, wrapperError);
|
|
14436
14629
|
}
|
|
14437
|
-
return { success: false, error:
|
|
14630
|
+
return { success: false, error: wrapperError.message };
|
|
14438
14631
|
}
|
|
14439
14632
|
});
|
|
14440
14633
|
return Promise.allSettled(promises);
|
|
@@ -14456,9 +14649,13 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
14456
14649
|
timestamp: typeof item.timestamp === "number" ? item.timestamp : Date.now(),
|
|
14457
14650
|
createdAt: item.createdAt || (/* @__PURE__ */ new Date()).toISOString().slice(0, 10)
|
|
14458
14651
|
};
|
|
14459
|
-
|
|
14652
|
+
const [ok, err] = await try_fn_default(async () => {
|
|
14460
14653
|
await logRes.insert(logItem);
|
|
14461
|
-
}
|
|
14654
|
+
});
|
|
14655
|
+
if (!ok) {
|
|
14656
|
+
if (this.config.verbose) {
|
|
14657
|
+
console.warn(`[ReplicatorPlugin] Failed to log replicator item: ${err.message}`);
|
|
14658
|
+
}
|
|
14462
14659
|
this.emit("replicator.log.failed", { error: err, item });
|
|
14463
14660
|
}
|
|
14464
14661
|
}
|
|
@@ -14564,14 +14761,23 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
14564
14761
|
this.emit("replicator.sync.completed", { replicatorId, stats: this.stats });
|
|
14565
14762
|
}
|
|
14566
14763
|
async cleanup() {
|
|
14567
|
-
|
|
14764
|
+
const [ok, error] = await try_fn_default(async () => {
|
|
14568
14765
|
if (this.replicators && this.replicators.length > 0) {
|
|
14569
14766
|
const cleanupPromises = this.replicators.map(async (replicator) => {
|
|
14570
|
-
|
|
14767
|
+
const [replicatorOk, replicatorError] = await try_fn_default(async () => {
|
|
14571
14768
|
if (replicator && typeof replicator.cleanup === "function") {
|
|
14572
14769
|
await replicator.cleanup();
|
|
14573
14770
|
}
|
|
14574
|
-
}
|
|
14771
|
+
});
|
|
14772
|
+
if (!replicatorOk) {
|
|
14773
|
+
if (this.config.verbose) {
|
|
14774
|
+
console.warn(`[ReplicatorPlugin] Failed to cleanup replicator ${replicator.name || replicator.id}: ${replicatorError.message}`);
|
|
14775
|
+
}
|
|
14776
|
+
this.emit("replicator_cleanup_error", {
|
|
14777
|
+
replicator: replicator.name || replicator.id || "unknown",
|
|
14778
|
+
driver: replicator.driver || "unknown",
|
|
14779
|
+
error: replicatorError.message
|
|
14780
|
+
});
|
|
14575
14781
|
}
|
|
14576
14782
|
});
|
|
14577
14783
|
await Promise.allSettled(cleanupPromises);
|
|
@@ -14580,7 +14786,14 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
14580
14786
|
this.database = null;
|
|
14581
14787
|
this.eventListenersInstalled.clear();
|
|
14582
14788
|
this.removeAllListeners();
|
|
14583
|
-
}
|
|
14789
|
+
});
|
|
14790
|
+
if (!ok) {
|
|
14791
|
+
if (this.config.verbose) {
|
|
14792
|
+
console.warn(`[ReplicatorPlugin] Failed to cleanup plugin: ${error.message}`);
|
|
14793
|
+
}
|
|
14794
|
+
this.emit("replicator_plugin_cleanup_error", {
|
|
14795
|
+
error: error.message
|
|
14796
|
+
});
|
|
14584
14797
|
}
|
|
14585
14798
|
}
|
|
14586
14799
|
}
|