s3db.js 7.3.5 → 7.3.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/PLUGINS.md +1285 -157
- package/dist/s3db.cjs.js +296 -83
- package/dist/s3db.cjs.min.js +1 -1
- package/dist/s3db.es.js +296 -83
- package/dist/s3db.es.min.js +1 -1
- package/dist/s3db.iife.js +296 -83
- package/dist/s3db.iife.min.js +1 -1
- package/mcp/server.js +1410 -0
- package/package.json +7 -1
- package/src/plugins/cache/filesystem-cache.class.js +9 -0
- package/src/plugins/replicator.plugin.js +130 -46
- package/src/plugins/replicators/bigquery-replicator.class.js +31 -5
- package/src/plugins/replicators/postgres-replicator.class.js +17 -2
- package/src/plugins/replicators/s3db-replicator.class.js +172 -68
- package/src/plugins/replicators/sqs-replicator.class.js +13 -1
package/dist/s3db.es.js
CHANGED
|
@@ -7079,6 +7079,12 @@ class FilesystemCache extends Cache {
|
|
|
7079
7079
|
}
|
|
7080
7080
|
async _clear(prefix) {
|
|
7081
7081
|
try {
|
|
7082
|
+
if (!await this._fileExists(this.directory)) {
|
|
7083
|
+
if (this.enableStats) {
|
|
7084
|
+
this.stats.clears++;
|
|
7085
|
+
}
|
|
7086
|
+
return true;
|
|
7087
|
+
}
|
|
7082
7088
|
const files = await readdir(this.directory);
|
|
7083
7089
|
const cacheFiles = files.filter((file) => {
|
|
7084
7090
|
if (!file.startsWith(this.prefix)) return false;
|
|
@@ -8950,6 +8956,9 @@ class BigqueryReplicator extends base_replicator_class_default {
|
|
|
8950
8956
|
await super.initialize(database);
|
|
8951
8957
|
const [ok, err, sdk] = await try_fn_default(() => import('@google-cloud/bigquery'));
|
|
8952
8958
|
if (!ok) {
|
|
8959
|
+
if (this.config.verbose) {
|
|
8960
|
+
console.warn(`[BigqueryReplicator] Failed to import BigQuery SDK: ${err.message}`);
|
|
8961
|
+
}
|
|
8953
8962
|
this.emit("initialization_error", { replicator: this.name, error: err.message });
|
|
8954
8963
|
throw err;
|
|
8955
8964
|
}
|
|
@@ -9019,19 +9028,28 @@ class BigqueryReplicator extends base_replicator_class_default {
|
|
|
9019
9028
|
const maxRetries = 2;
|
|
9020
9029
|
let lastError = null;
|
|
9021
9030
|
for (let attempt = 1; attempt <= maxRetries; attempt++) {
|
|
9022
|
-
|
|
9031
|
+
const [ok2, error] = await try_fn_default(async () => {
|
|
9023
9032
|
const [updateJob] = await this.bigqueryClient.createQueryJob({
|
|
9024
9033
|
query,
|
|
9025
9034
|
params,
|
|
9026
9035
|
location: this.location
|
|
9027
9036
|
});
|
|
9028
9037
|
await updateJob.getQueryResults();
|
|
9029
|
-
|
|
9038
|
+
return [updateJob];
|
|
9039
|
+
});
|
|
9040
|
+
if (ok2) {
|
|
9041
|
+
job = ok2;
|
|
9030
9042
|
break;
|
|
9031
|
-
}
|
|
9043
|
+
} else {
|
|
9032
9044
|
lastError = error;
|
|
9045
|
+
if (this.config.verbose) {
|
|
9046
|
+
console.warn(`[BigqueryReplicator] Update attempt ${attempt} failed: ${error.message}`);
|
|
9047
|
+
}
|
|
9033
9048
|
if (error?.message?.includes("streaming buffer") && attempt < maxRetries) {
|
|
9034
9049
|
const delaySeconds = 30;
|
|
9050
|
+
if (this.config.verbose) {
|
|
9051
|
+
console.warn(`[BigqueryReplicator] Retrying in ${delaySeconds} seconds due to streaming buffer issue`);
|
|
9052
|
+
}
|
|
9035
9053
|
await new Promise((resolve) => setTimeout(resolve, delaySeconds * 1e3));
|
|
9036
9054
|
continue;
|
|
9037
9055
|
}
|
|
@@ -9098,6 +9116,9 @@ class BigqueryReplicator extends base_replicator_class_default {
|
|
|
9098
9116
|
};
|
|
9099
9117
|
});
|
|
9100
9118
|
if (ok) return result;
|
|
9119
|
+
if (this.config.verbose) {
|
|
9120
|
+
console.warn(`[BigqueryReplicator] Replication failed for ${resourceName}: ${err.message}`);
|
|
9121
|
+
}
|
|
9101
9122
|
this.emit("replicator_error", {
|
|
9102
9123
|
replicator: this.name,
|
|
9103
9124
|
resourceName,
|
|
@@ -9118,8 +9139,14 @@ class BigqueryReplicator extends base_replicator_class_default {
|
|
|
9118
9139
|
record.id,
|
|
9119
9140
|
record.beforeData
|
|
9120
9141
|
));
|
|
9121
|
-
if (ok)
|
|
9122
|
-
|
|
9142
|
+
if (ok) {
|
|
9143
|
+
results.push(res);
|
|
9144
|
+
} else {
|
|
9145
|
+
if (this.config.verbose) {
|
|
9146
|
+
console.warn(`[BigqueryReplicator] Batch replication failed for record ${record.id}: ${err.message}`);
|
|
9147
|
+
}
|
|
9148
|
+
errors.push({ id: record.id, error: err.message });
|
|
9149
|
+
}
|
|
9123
9150
|
}
|
|
9124
9151
|
return {
|
|
9125
9152
|
success: errors.length === 0,
|
|
@@ -9135,6 +9162,9 @@ class BigqueryReplicator extends base_replicator_class_default {
|
|
|
9135
9162
|
return true;
|
|
9136
9163
|
});
|
|
9137
9164
|
if (ok) return true;
|
|
9165
|
+
if (this.config.verbose) {
|
|
9166
|
+
console.warn(`[BigqueryReplicator] Connection test failed: ${err.message}`);
|
|
9167
|
+
}
|
|
9138
9168
|
this.emit("connection_error", { replicator: this.name, error: err.message });
|
|
9139
9169
|
return false;
|
|
9140
9170
|
}
|
|
@@ -9222,6 +9252,9 @@ class PostgresReplicator extends base_replicator_class_default {
|
|
|
9222
9252
|
await super.initialize(database);
|
|
9223
9253
|
const [ok, err, sdk] = await try_fn_default(() => import('pg'));
|
|
9224
9254
|
if (!ok) {
|
|
9255
|
+
if (this.config.verbose) {
|
|
9256
|
+
console.warn(`[PostgresReplicator] Failed to import pg SDK: ${err.message}`);
|
|
9257
|
+
}
|
|
9225
9258
|
this.emit("initialization_error", {
|
|
9226
9259
|
replicator: this.name,
|
|
9227
9260
|
error: err.message
|
|
@@ -9363,6 +9396,9 @@ class PostgresReplicator extends base_replicator_class_default {
|
|
|
9363
9396
|
};
|
|
9364
9397
|
});
|
|
9365
9398
|
if (ok) return result;
|
|
9399
|
+
if (this.config.verbose) {
|
|
9400
|
+
console.warn(`[PostgresReplicator] Replication failed for ${resourceName}: ${err.message}`);
|
|
9401
|
+
}
|
|
9366
9402
|
this.emit("replicator_error", {
|
|
9367
9403
|
replicator: this.name,
|
|
9368
9404
|
resourceName,
|
|
@@ -9383,8 +9419,14 @@ class PostgresReplicator extends base_replicator_class_default {
|
|
|
9383
9419
|
record.id,
|
|
9384
9420
|
record.beforeData
|
|
9385
9421
|
));
|
|
9386
|
-
if (ok)
|
|
9387
|
-
|
|
9422
|
+
if (ok) {
|
|
9423
|
+
results.push(res);
|
|
9424
|
+
} else {
|
|
9425
|
+
if (this.config.verbose) {
|
|
9426
|
+
console.warn(`[PostgresReplicator] Batch replication failed for record ${record.id}: ${err.message}`);
|
|
9427
|
+
}
|
|
9428
|
+
errors.push({ id: record.id, error: err.message });
|
|
9429
|
+
}
|
|
9388
9430
|
}
|
|
9389
9431
|
return {
|
|
9390
9432
|
success: errors.length === 0,
|
|
@@ -9399,6 +9441,9 @@ class PostgresReplicator extends base_replicator_class_default {
|
|
|
9399
9441
|
return true;
|
|
9400
9442
|
});
|
|
9401
9443
|
if (ok) return true;
|
|
9444
|
+
if (this.config.verbose) {
|
|
9445
|
+
console.warn(`[PostgresReplicator] Connection test failed: ${err.message}`);
|
|
9446
|
+
}
|
|
9402
9447
|
this.emit("connection_error", { replicator: this.name, error: err.message });
|
|
9403
9448
|
return false;
|
|
9404
9449
|
}
|
|
@@ -13133,7 +13178,7 @@ class Database extends EventEmitter {
|
|
|
13133
13178
|
super();
|
|
13134
13179
|
this.version = "1";
|
|
13135
13180
|
this.s3dbVersion = (() => {
|
|
13136
|
-
const [ok, err, version] = try_fn_default(() => true ? "7.3.
|
|
13181
|
+
const [ok, err, version] = try_fn_default(() => true ? "7.3.6" : "latest");
|
|
13137
13182
|
return ok ? version : "latest";
|
|
13138
13183
|
})();
|
|
13139
13184
|
this.resources = {};
|
|
@@ -13637,9 +13682,8 @@ class S3dbReplicator extends base_replicator_class_default {
|
|
|
13637
13682
|
const map = {};
|
|
13638
13683
|
for (const res of resources) {
|
|
13639
13684
|
if (typeof res === "string") map[normalizeResourceName$1(res)] = res;
|
|
13640
|
-
else if (Array.isArray(res) && typeof res[0] === "string") map[normalizeResourceName$1(res[0])] = res;
|
|
13641
13685
|
else if (typeof res === "object" && res.resource) {
|
|
13642
|
-
map[normalizeResourceName$1(res.resource)] =
|
|
13686
|
+
map[normalizeResourceName$1(res.resource)] = res;
|
|
13643
13687
|
}
|
|
13644
13688
|
}
|
|
13645
13689
|
return map;
|
|
@@ -13652,15 +13696,14 @@ class S3dbReplicator extends base_replicator_class_default {
|
|
|
13652
13696
|
else if (Array.isArray(dest)) {
|
|
13653
13697
|
map[normSrc] = dest.map((item) => {
|
|
13654
13698
|
if (typeof item === "string") return item;
|
|
13655
|
-
if (typeof item === "function") return item;
|
|
13656
13699
|
if (typeof item === "object" && item.resource) {
|
|
13657
|
-
return
|
|
13700
|
+
return item;
|
|
13658
13701
|
}
|
|
13659
13702
|
return item;
|
|
13660
13703
|
});
|
|
13661
13704
|
} else if (typeof dest === "function") map[normSrc] = dest;
|
|
13662
13705
|
else if (typeof dest === "object" && dest.resource) {
|
|
13663
|
-
map[normSrc] =
|
|
13706
|
+
map[normSrc] = dest;
|
|
13664
13707
|
}
|
|
13665
13708
|
}
|
|
13666
13709
|
return map;
|
|
@@ -13668,10 +13711,6 @@ class S3dbReplicator extends base_replicator_class_default {
|
|
|
13668
13711
|
if (typeof resources === "function") {
|
|
13669
13712
|
return resources;
|
|
13670
13713
|
}
|
|
13671
|
-
if (typeof resources === "string") {
|
|
13672
|
-
const map = { [normalizeResourceName$1(resources)]: resources };
|
|
13673
|
-
return map;
|
|
13674
|
-
}
|
|
13675
13714
|
return {};
|
|
13676
13715
|
}
|
|
13677
13716
|
validateConfig() {
|
|
@@ -13685,8 +13724,8 @@ class S3dbReplicator extends base_replicator_class_default {
|
|
|
13685
13724
|
return { isValid: errors.length === 0, errors };
|
|
13686
13725
|
}
|
|
13687
13726
|
async initialize(database) {
|
|
13688
|
-
|
|
13689
|
-
|
|
13727
|
+
await super.initialize(database);
|
|
13728
|
+
const [ok, err] = await try_fn_default(async () => {
|
|
13690
13729
|
if (this.client) {
|
|
13691
13730
|
this.targetDatabase = this.client;
|
|
13692
13731
|
} else if (this.connectionString) {
|
|
@@ -13705,7 +13744,11 @@ class S3dbReplicator extends base_replicator_class_default {
|
|
|
13705
13744
|
replicator: this.name,
|
|
13706
13745
|
target: this.connectionString || "client-provided"
|
|
13707
13746
|
});
|
|
13708
|
-
}
|
|
13747
|
+
});
|
|
13748
|
+
if (!ok) {
|
|
13749
|
+
if (this.config.verbose) {
|
|
13750
|
+
console.warn(`[S3dbReplicator] Initialization failed: ${err.message}`);
|
|
13751
|
+
}
|
|
13709
13752
|
throw err;
|
|
13710
13753
|
}
|
|
13711
13754
|
}
|
|
@@ -13724,18 +13767,77 @@ class S3dbReplicator extends base_replicator_class_default {
|
|
|
13724
13767
|
id = recordId;
|
|
13725
13768
|
}
|
|
13726
13769
|
const normResource = normalizeResourceName$1(resource);
|
|
13727
|
-
const
|
|
13728
|
-
|
|
13729
|
-
|
|
13770
|
+
const entry = this.resourcesMap[normResource];
|
|
13771
|
+
if (!entry) {
|
|
13772
|
+
throw new Error(`[S3dbReplicator] Resource not configured: ${resource}`);
|
|
13773
|
+
}
|
|
13774
|
+
if (Array.isArray(entry)) {
|
|
13775
|
+
const results = [];
|
|
13776
|
+
for (const destConfig of entry) {
|
|
13777
|
+
const [ok, error, result] = await try_fn_default(async () => {
|
|
13778
|
+
return await this._replicateToSingleDestination(destConfig, normResource, op, payload, id);
|
|
13779
|
+
});
|
|
13780
|
+
if (!ok) {
|
|
13781
|
+
if (this.config && this.config.verbose) {
|
|
13782
|
+
console.warn(`[S3dbReplicator] Failed to replicate to destination ${JSON.stringify(destConfig)}: ${error.message}`);
|
|
13783
|
+
}
|
|
13784
|
+
throw error;
|
|
13785
|
+
}
|
|
13786
|
+
results.push(result);
|
|
13787
|
+
}
|
|
13788
|
+
return results;
|
|
13789
|
+
} else {
|
|
13790
|
+
const [ok, error, result] = await try_fn_default(async () => {
|
|
13791
|
+
return await this._replicateToSingleDestination(entry, normResource, op, payload, id);
|
|
13792
|
+
});
|
|
13793
|
+
if (!ok) {
|
|
13794
|
+
if (this.config && this.config.verbose) {
|
|
13795
|
+
console.warn(`[S3dbReplicator] Failed to replicate to destination ${JSON.stringify(entry)}: ${error.message}`);
|
|
13796
|
+
}
|
|
13797
|
+
throw error;
|
|
13798
|
+
}
|
|
13799
|
+
return result;
|
|
13800
|
+
}
|
|
13801
|
+
}
|
|
13802
|
+
async _replicateToSingleDestination(destConfig, sourceResource, operation, data, recordId) {
|
|
13803
|
+
let destResourceName;
|
|
13804
|
+
if (typeof destConfig === "string") {
|
|
13805
|
+
destResourceName = destConfig;
|
|
13806
|
+
} else if (typeof destConfig === "object" && destConfig.resource) {
|
|
13807
|
+
destResourceName = destConfig.resource;
|
|
13808
|
+
} else {
|
|
13809
|
+
destResourceName = sourceResource;
|
|
13810
|
+
}
|
|
13811
|
+
if (typeof destConfig === "object" && destConfig.actions && Array.isArray(destConfig.actions)) {
|
|
13812
|
+
if (!destConfig.actions.includes(operation)) {
|
|
13813
|
+
return { skipped: true, reason: "action_not_supported", action: operation, destination: destResourceName };
|
|
13814
|
+
}
|
|
13815
|
+
}
|
|
13816
|
+
const destResourceObj = this._getDestResourceObj(destResourceName);
|
|
13817
|
+
let transformedData;
|
|
13818
|
+
if (typeof destConfig === "object" && destConfig.transform && typeof destConfig.transform === "function") {
|
|
13819
|
+
transformedData = destConfig.transform(data);
|
|
13820
|
+
if (transformedData && data && data.id && !transformedData.id) {
|
|
13821
|
+
transformedData.id = data.id;
|
|
13822
|
+
}
|
|
13823
|
+
} else if (typeof destConfig === "object" && destConfig.transformer && typeof destConfig.transformer === "function") {
|
|
13824
|
+
transformedData = destConfig.transformer(data);
|
|
13825
|
+
if (transformedData && data && data.id && !transformedData.id) {
|
|
13826
|
+
transformedData.id = data.id;
|
|
13827
|
+
}
|
|
13828
|
+
} else {
|
|
13829
|
+
transformedData = data;
|
|
13830
|
+
}
|
|
13831
|
+
if (!transformedData && data) transformedData = data;
|
|
13730
13832
|
let result;
|
|
13731
|
-
if (
|
|
13833
|
+
if (operation === "insert") {
|
|
13732
13834
|
result = await destResourceObj.insert(transformedData);
|
|
13733
|
-
} else if (
|
|
13734
|
-
result = await destResourceObj.update(
|
|
13735
|
-
} else if (
|
|
13736
|
-
result = await destResourceObj.delete(
|
|
13835
|
+
} else if (operation === "update") {
|
|
13836
|
+
result = await destResourceObj.update(recordId, transformedData);
|
|
13837
|
+
} else if (operation === "delete") {
|
|
13838
|
+
result = await destResourceObj.delete(recordId);
|
|
13737
13839
|
} else {
|
|
13738
|
-
throw new Error(`Invalid operation: ${
|
|
13840
|
+
throw new Error(`Invalid operation: ${operation}. Supported operations are: insert, update, delete`);
|
|
13739
13841
|
}
|
|
13740
13842
|
return result;
|
|
13741
13843
|
}
|
|
@@ -13744,13 +13846,25 @@ class S3dbReplicator extends base_replicator_class_default {
|
|
|
13744
13846
|
const entry = this.resourcesMap[normResource];
|
|
13745
13847
|
let result;
|
|
13746
13848
|
if (!entry) return data;
|
|
13747
|
-
if (Array.isArray(entry)
|
|
13748
|
-
|
|
13849
|
+
if (Array.isArray(entry)) {
|
|
13850
|
+
for (const item of entry) {
|
|
13851
|
+
if (typeof item === "object" && item.transform && typeof item.transform === "function") {
|
|
13852
|
+
result = item.transform(data);
|
|
13853
|
+
break;
|
|
13854
|
+
} else if (typeof item === "object" && item.transformer && typeof item.transformer === "function") {
|
|
13855
|
+
result = item.transformer(data);
|
|
13856
|
+
break;
|
|
13857
|
+
}
|
|
13858
|
+
}
|
|
13859
|
+
if (!result) result = data;
|
|
13860
|
+
} else if (typeof entry === "object") {
|
|
13861
|
+
if (typeof entry.transform === "function") {
|
|
13862
|
+
result = entry.transform(data);
|
|
13863
|
+
} else if (typeof entry.transformer === "function") {
|
|
13864
|
+
result = entry.transformer(data);
|
|
13865
|
+
}
|
|
13749
13866
|
} else if (typeof entry === "function") {
|
|
13750
13867
|
result = entry(data);
|
|
13751
|
-
} else if (typeof entry === "object") {
|
|
13752
|
-
if (typeof entry.transform === "function") result = entry.transform(data);
|
|
13753
|
-
else if (typeof entry.transformer === "function") result = entry.transformer(data);
|
|
13754
13868
|
} else {
|
|
13755
13869
|
result = data;
|
|
13756
13870
|
}
|
|
@@ -13763,9 +13877,11 @@ class S3dbReplicator extends base_replicator_class_default {
|
|
|
13763
13877
|
const entry = this.resourcesMap[normResource];
|
|
13764
13878
|
if (!entry) return resource;
|
|
13765
13879
|
if (Array.isArray(entry)) {
|
|
13766
|
-
|
|
13767
|
-
|
|
13768
|
-
|
|
13880
|
+
for (const item of entry) {
|
|
13881
|
+
if (typeof item === "string") return item;
|
|
13882
|
+
if (typeof item === "object" && item.resource) return item.resource;
|
|
13883
|
+
}
|
|
13884
|
+
return resource;
|
|
13769
13885
|
}
|
|
13770
13886
|
if (typeof entry === "string") return entry;
|
|
13771
13887
|
if (typeof entry === "function") return resource;
|
|
@@ -13773,8 +13889,7 @@ class S3dbReplicator extends base_replicator_class_default {
|
|
|
13773
13889
|
return resource;
|
|
13774
13890
|
}
|
|
13775
13891
|
_getDestResourceObj(resource) {
|
|
13776
|
-
|
|
13777
|
-
const available = Object.keys(this.client.resources);
|
|
13892
|
+
const available = Object.keys(this.client.resources || {});
|
|
13778
13893
|
const norm = normalizeResourceName$1(resource);
|
|
13779
13894
|
const found = available.find((r) => normalizeResourceName$1(r) === norm);
|
|
13780
13895
|
if (!found) {
|
|
@@ -13796,8 +13911,14 @@ class S3dbReplicator extends base_replicator_class_default {
|
|
|
13796
13911
|
data: record.data,
|
|
13797
13912
|
beforeData: record.beforeData
|
|
13798
13913
|
}));
|
|
13799
|
-
if (ok)
|
|
13800
|
-
|
|
13914
|
+
if (ok) {
|
|
13915
|
+
results.push(result);
|
|
13916
|
+
} else {
|
|
13917
|
+
if (this.config.verbose) {
|
|
13918
|
+
console.warn(`[S3dbReplicator] Batch replication failed for record ${record.id}: ${err.message}`);
|
|
13919
|
+
}
|
|
13920
|
+
errors.push({ id: record.id, error: err.message });
|
|
13921
|
+
}
|
|
13801
13922
|
}
|
|
13802
13923
|
this.emit("batch_replicated", {
|
|
13803
13924
|
replicator: this.name,
|
|
@@ -13815,18 +13936,20 @@ class S3dbReplicator extends base_replicator_class_default {
|
|
|
13815
13936
|
}
|
|
13816
13937
|
async testConnection() {
|
|
13817
13938
|
const [ok, err] = await try_fn_default(async () => {
|
|
13818
|
-
if (!this.targetDatabase)
|
|
13819
|
-
|
|
13939
|
+
if (!this.targetDatabase) throw new Error("No target database configured");
|
|
13940
|
+
if (typeof this.targetDatabase.connect === "function") {
|
|
13941
|
+
await this.targetDatabase.connect();
|
|
13820
13942
|
}
|
|
13821
|
-
await this.targetDatabase.listResources();
|
|
13822
13943
|
return true;
|
|
13823
13944
|
});
|
|
13824
|
-
if (ok)
|
|
13825
|
-
|
|
13826
|
-
|
|
13827
|
-
|
|
13828
|
-
|
|
13829
|
-
|
|
13945
|
+
if (!ok) {
|
|
13946
|
+
if (this.config.verbose) {
|
|
13947
|
+
console.warn(`[S3dbReplicator] Connection test failed: ${err.message}`);
|
|
13948
|
+
}
|
|
13949
|
+
this.emit("connection_error", { replicator: this.name, error: err.message });
|
|
13950
|
+
return false;
|
|
13951
|
+
}
|
|
13952
|
+
return true;
|
|
13830
13953
|
}
|
|
13831
13954
|
async getStatus() {
|
|
13832
13955
|
const baseStatus = await super.getStatus();
|
|
@@ -13858,7 +13981,7 @@ class S3dbReplicator extends base_replicator_class_default {
|
|
|
13858
13981
|
} else {
|
|
13859
13982
|
return true;
|
|
13860
13983
|
}
|
|
13861
|
-
} else if (typeof item === "string"
|
|
13984
|
+
} else if (typeof item === "string") {
|
|
13862
13985
|
return true;
|
|
13863
13986
|
}
|
|
13864
13987
|
}
|
|
@@ -13985,6 +14108,9 @@ class SqsReplicator extends base_replicator_class_default {
|
|
|
13985
14108
|
if (!this.sqsClient) {
|
|
13986
14109
|
const [ok, err, sdk] = await try_fn_default(() => import('@aws-sdk/client-sqs'));
|
|
13987
14110
|
if (!ok) {
|
|
14111
|
+
if (this.config.verbose) {
|
|
14112
|
+
console.warn(`[SqsReplicator] Failed to import SQS SDK: ${err.message}`);
|
|
14113
|
+
}
|
|
13988
14114
|
this.emit("initialization_error", {
|
|
13989
14115
|
replicator: this.name,
|
|
13990
14116
|
error: err.message
|
|
@@ -14036,6 +14162,9 @@ class SqsReplicator extends base_replicator_class_default {
|
|
|
14036
14162
|
return { success: true, results };
|
|
14037
14163
|
});
|
|
14038
14164
|
if (ok) return result;
|
|
14165
|
+
if (this.config.verbose) {
|
|
14166
|
+
console.warn(`[SqsReplicator] Replication failed for ${resource}: ${err.message}`);
|
|
14167
|
+
}
|
|
14039
14168
|
this.emit("replicator_error", {
|
|
14040
14169
|
replicator: this.name,
|
|
14041
14170
|
resource,
|
|
@@ -14108,6 +14237,9 @@ class SqsReplicator extends base_replicator_class_default {
|
|
|
14108
14237
|
});
|
|
14109
14238
|
if (ok) return result;
|
|
14110
14239
|
const errorMessage = err?.message || err || "Unknown error";
|
|
14240
|
+
if (this.config.verbose) {
|
|
14241
|
+
console.warn(`[SqsReplicator] Batch replication failed for ${resource}: ${errorMessage}`);
|
|
14242
|
+
}
|
|
14111
14243
|
this.emit("batch_replicator_error", {
|
|
14112
14244
|
replicator: this.name,
|
|
14113
14245
|
resource,
|
|
@@ -14129,6 +14261,9 @@ class SqsReplicator extends base_replicator_class_default {
|
|
|
14129
14261
|
return true;
|
|
14130
14262
|
});
|
|
14131
14263
|
if (ok) return true;
|
|
14264
|
+
if (this.config.verbose) {
|
|
14265
|
+
console.warn(`[SqsReplicator] Connection test failed: ${err.message}`);
|
|
14266
|
+
}
|
|
14132
14267
|
this.emit("connection_error", {
|
|
14133
14268
|
replicator: this.name,
|
|
14134
14269
|
error: err.message
|
|
@@ -14225,25 +14360,37 @@ class ReplicatorPlugin extends plugin_class_default {
|
|
|
14225
14360
|
return;
|
|
14226
14361
|
}
|
|
14227
14362
|
resource.on("insert", async (data) => {
|
|
14228
|
-
|
|
14363
|
+
const [ok, error] = await try_fn_default(async () => {
|
|
14229
14364
|
const completeData = { ...data, createdAt: (/* @__PURE__ */ new Date()).toISOString() };
|
|
14230
14365
|
await plugin.processReplicatorEvent("insert", resource.name, completeData.id, completeData);
|
|
14231
|
-
}
|
|
14366
|
+
});
|
|
14367
|
+
if (!ok) {
|
|
14368
|
+
if (this.config.verbose) {
|
|
14369
|
+
console.warn(`[ReplicatorPlugin] Insert event failed for resource ${resource.name}: ${error.message}`);
|
|
14370
|
+
}
|
|
14232
14371
|
this.emit("error", { operation: "insert", error: error.message, resource: resource.name });
|
|
14233
14372
|
}
|
|
14234
14373
|
});
|
|
14235
14374
|
resource.on("update", async (data, beforeData) => {
|
|
14236
|
-
|
|
14375
|
+
const [ok, error] = await try_fn_default(async () => {
|
|
14237
14376
|
const completeData = { ...data, updatedAt: (/* @__PURE__ */ new Date()).toISOString() };
|
|
14238
14377
|
await plugin.processReplicatorEvent("update", resource.name, completeData.id, completeData, beforeData);
|
|
14239
|
-
}
|
|
14378
|
+
});
|
|
14379
|
+
if (!ok) {
|
|
14380
|
+
if (this.config.verbose) {
|
|
14381
|
+
console.warn(`[ReplicatorPlugin] Update event failed for resource ${resource.name}: ${error.message}`);
|
|
14382
|
+
}
|
|
14240
14383
|
this.emit("error", { operation: "update", error: error.message, resource: resource.name });
|
|
14241
14384
|
}
|
|
14242
14385
|
});
|
|
14243
14386
|
resource.on("delete", async (data) => {
|
|
14244
|
-
|
|
14387
|
+
const [ok, error] = await try_fn_default(async () => {
|
|
14245
14388
|
await plugin.processReplicatorEvent("delete", resource.name, data.id, data);
|
|
14246
|
-
}
|
|
14389
|
+
});
|
|
14390
|
+
if (!ok) {
|
|
14391
|
+
if (this.config.verbose) {
|
|
14392
|
+
console.warn(`[ReplicatorPlugin] Delete event failed for resource ${resource.name}: ${error.message}`);
|
|
14393
|
+
}
|
|
14247
14394
|
this.emit("error", { operation: "delete", error: error.message, resource: resource.name });
|
|
14248
14395
|
}
|
|
14249
14396
|
});
|
|
@@ -14259,13 +14406,17 @@ class ReplicatorPlugin extends plugin_class_default {
|
|
|
14259
14406
|
}
|
|
14260
14407
|
async setup(database) {
|
|
14261
14408
|
this.database = database;
|
|
14262
|
-
|
|
14409
|
+
const [initOk, initError] = await try_fn_default(async () => {
|
|
14263
14410
|
await this.initializeReplicators(database);
|
|
14264
|
-
}
|
|
14265
|
-
|
|
14266
|
-
|
|
14411
|
+
});
|
|
14412
|
+
if (!initOk) {
|
|
14413
|
+
if (this.config.verbose) {
|
|
14414
|
+
console.warn(`[ReplicatorPlugin] Replicator initialization failed: ${initError.message}`);
|
|
14415
|
+
}
|
|
14416
|
+
this.emit("error", { operation: "setup", error: initError.message });
|
|
14417
|
+
throw initError;
|
|
14267
14418
|
}
|
|
14268
|
-
|
|
14419
|
+
const [logOk, logError] = await try_fn_default(async () => {
|
|
14269
14420
|
if (this.config.replicatorLogResource) {
|
|
14270
14421
|
const logRes = await database.createResource({
|
|
14271
14422
|
name: this.config.replicatorLogResource,
|
|
@@ -14282,7 +14433,15 @@ class ReplicatorPlugin extends plugin_class_default {
|
|
|
14282
14433
|
}
|
|
14283
14434
|
});
|
|
14284
14435
|
}
|
|
14285
|
-
}
|
|
14436
|
+
});
|
|
14437
|
+
if (!logOk) {
|
|
14438
|
+
if (this.config.verbose) {
|
|
14439
|
+
console.warn(`[ReplicatorPlugin] Failed to create log resource ${this.config.replicatorLogResource}: ${logError.message}`);
|
|
14440
|
+
}
|
|
14441
|
+
this.emit("replicator_log_resource_creation_error", {
|
|
14442
|
+
resourceName: this.config.replicatorLogResource,
|
|
14443
|
+
error: logError.message
|
|
14444
|
+
});
|
|
14286
14445
|
}
|
|
14287
14446
|
await this.uploadMetadataFile(database);
|
|
14288
14447
|
const originalCreateResource = database.createResource.bind(database);
|
|
@@ -14325,21 +14484,28 @@ class ReplicatorPlugin extends plugin_class_default {
|
|
|
14325
14484
|
async retryWithBackoff(operation, maxRetries = 3) {
|
|
14326
14485
|
let lastError;
|
|
14327
14486
|
for (let attempt = 1; attempt <= maxRetries; attempt++) {
|
|
14328
|
-
|
|
14329
|
-
|
|
14330
|
-
|
|
14487
|
+
const [ok, error] = await try_fn_default(operation);
|
|
14488
|
+
if (ok) {
|
|
14489
|
+
return ok;
|
|
14490
|
+
} else {
|
|
14331
14491
|
lastError = error;
|
|
14492
|
+
if (this.config.verbose) {
|
|
14493
|
+
console.warn(`[ReplicatorPlugin] Retry attempt ${attempt}/${maxRetries} failed: ${error.message}`);
|
|
14494
|
+
}
|
|
14332
14495
|
if (attempt === maxRetries) {
|
|
14333
14496
|
throw error;
|
|
14334
14497
|
}
|
|
14335
14498
|
const delay = Math.pow(2, attempt - 1) * 1e3;
|
|
14499
|
+
if (this.config.verbose) {
|
|
14500
|
+
console.warn(`[ReplicatorPlugin] Waiting ${delay}ms before retry...`);
|
|
14501
|
+
}
|
|
14336
14502
|
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
14337
14503
|
}
|
|
14338
14504
|
}
|
|
14339
14505
|
throw lastError;
|
|
14340
14506
|
}
|
|
14341
14507
|
async logError(replicator, resourceName, operation, recordId, data, error) {
|
|
14342
|
-
|
|
14508
|
+
const [ok, logError] = await try_fn_default(async () => {
|
|
14343
14509
|
const logResourceName = this.config.replicatorLogResource;
|
|
14344
14510
|
if (this.database && this.database.resources && this.database.resources[logResourceName]) {
|
|
14345
14511
|
const logResource = this.database.resources[logResourceName];
|
|
@@ -14354,7 +14520,19 @@ class ReplicatorPlugin extends plugin_class_default {
|
|
|
14354
14520
|
status: "error"
|
|
14355
14521
|
});
|
|
14356
14522
|
}
|
|
14357
|
-
}
|
|
14523
|
+
});
|
|
14524
|
+
if (!ok) {
|
|
14525
|
+
if (this.config.verbose) {
|
|
14526
|
+
console.warn(`[ReplicatorPlugin] Failed to log error for ${resourceName}: ${logError.message}`);
|
|
14527
|
+
}
|
|
14528
|
+
this.emit("replicator_log_error", {
|
|
14529
|
+
replicator: replicator.name || replicator.id,
|
|
14530
|
+
resourceName,
|
|
14531
|
+
operation,
|
|
14532
|
+
recordId,
|
|
14533
|
+
originalError: error.message,
|
|
14534
|
+
logError: logError.message
|
|
14535
|
+
});
|
|
14358
14536
|
}
|
|
14359
14537
|
}
|
|
14360
14538
|
async processReplicatorEvent(operation, resourceName, recordId, data, beforeData = null) {
|
|
@@ -14367,8 +14545,8 @@ class ReplicatorPlugin extends plugin_class_default {
|
|
|
14367
14545
|
return;
|
|
14368
14546
|
}
|
|
14369
14547
|
const promises = applicableReplicators.map(async (replicator) => {
|
|
14370
|
-
|
|
14371
|
-
const
|
|
14548
|
+
const [ok, error, result] = await try_fn_default(async () => {
|
|
14549
|
+
const result2 = await this.retryWithBackoff(
|
|
14372
14550
|
() => replicator.replicate(resourceName, operation, data, recordId, beforeData),
|
|
14373
14551
|
this.config.maxRetries
|
|
14374
14552
|
);
|
|
@@ -14377,11 +14555,17 @@ class ReplicatorPlugin extends plugin_class_default {
|
|
|
14377
14555
|
resourceName,
|
|
14378
14556
|
operation,
|
|
14379
14557
|
recordId,
|
|
14380
|
-
result,
|
|
14558
|
+
result: result2,
|
|
14381
14559
|
success: true
|
|
14382
14560
|
});
|
|
14561
|
+
return result2;
|
|
14562
|
+
});
|
|
14563
|
+
if (ok) {
|
|
14383
14564
|
return result;
|
|
14384
|
-
}
|
|
14565
|
+
} else {
|
|
14566
|
+
if (this.config.verbose) {
|
|
14567
|
+
console.warn(`[ReplicatorPlugin] Replication failed for ${replicator.name || replicator.id} on ${resourceName}: ${error.message}`);
|
|
14568
|
+
}
|
|
14385
14569
|
this.emit("replicator_error", {
|
|
14386
14570
|
replicator: replicator.name || replicator.id,
|
|
14387
14571
|
resourceName,
|
|
@@ -14406,11 +14590,14 @@ class ReplicatorPlugin extends plugin_class_default {
|
|
|
14406
14590
|
return;
|
|
14407
14591
|
}
|
|
14408
14592
|
const promises = applicableReplicators.map(async (replicator) => {
|
|
14409
|
-
|
|
14593
|
+
const [wrapperOk, wrapperError] = await try_fn_default(async () => {
|
|
14410
14594
|
const [ok, err, result] = await try_fn_default(
|
|
14411
14595
|
() => replicator.replicate(item.resourceName, item.operation, item.data, item.recordId, item.beforeData)
|
|
14412
14596
|
);
|
|
14413
14597
|
if (!ok) {
|
|
14598
|
+
if (this.config.verbose) {
|
|
14599
|
+
console.warn(`[ReplicatorPlugin] Replicator item processing failed for ${replicator.name || replicator.id} on ${item.resourceName}: ${err.message}`);
|
|
14600
|
+
}
|
|
14414
14601
|
this.emit("replicator_error", {
|
|
14415
14602
|
replicator: replicator.name || replicator.id,
|
|
14416
14603
|
resourceName: item.resourceName,
|
|
@@ -14432,18 +14619,24 @@ class ReplicatorPlugin extends plugin_class_default {
|
|
|
14432
14619
|
success: true
|
|
14433
14620
|
});
|
|
14434
14621
|
return { success: true, result };
|
|
14435
|
-
}
|
|
14622
|
+
});
|
|
14623
|
+
if (wrapperOk) {
|
|
14624
|
+
return wrapperOk;
|
|
14625
|
+
} else {
|
|
14626
|
+
if (this.config.verbose) {
|
|
14627
|
+
console.warn(`[ReplicatorPlugin] Wrapper processing failed for ${replicator.name || replicator.id} on ${item.resourceName}: ${wrapperError.message}`);
|
|
14628
|
+
}
|
|
14436
14629
|
this.emit("replicator_error", {
|
|
14437
14630
|
replicator: replicator.name || replicator.id,
|
|
14438
14631
|
resourceName: item.resourceName,
|
|
14439
14632
|
operation: item.operation,
|
|
14440
14633
|
recordId: item.recordId,
|
|
14441
|
-
error:
|
|
14634
|
+
error: wrapperError.message
|
|
14442
14635
|
});
|
|
14443
14636
|
if (this.config.logErrors && this.database) {
|
|
14444
|
-
await this.logError(replicator, item.resourceName, item.operation, item.recordId, item.data,
|
|
14637
|
+
await this.logError(replicator, item.resourceName, item.operation, item.recordId, item.data, wrapperError);
|
|
14445
14638
|
}
|
|
14446
|
-
return { success: false, error:
|
|
14639
|
+
return { success: false, error: wrapperError.message };
|
|
14447
14640
|
}
|
|
14448
14641
|
});
|
|
14449
14642
|
return Promise.allSettled(promises);
|
|
@@ -14465,9 +14658,13 @@ class ReplicatorPlugin extends plugin_class_default {
|
|
|
14465
14658
|
timestamp: typeof item.timestamp === "number" ? item.timestamp : Date.now(),
|
|
14466
14659
|
createdAt: item.createdAt || (/* @__PURE__ */ new Date()).toISOString().slice(0, 10)
|
|
14467
14660
|
};
|
|
14468
|
-
|
|
14661
|
+
const [ok, err] = await try_fn_default(async () => {
|
|
14469
14662
|
await logRes.insert(logItem);
|
|
14470
|
-
}
|
|
14663
|
+
});
|
|
14664
|
+
if (!ok) {
|
|
14665
|
+
if (this.config.verbose) {
|
|
14666
|
+
console.warn(`[ReplicatorPlugin] Failed to log replicator item: ${err.message}`);
|
|
14667
|
+
}
|
|
14471
14668
|
this.emit("replicator.log.failed", { error: err, item });
|
|
14472
14669
|
}
|
|
14473
14670
|
}
|
|
@@ -14573,14 +14770,23 @@ class ReplicatorPlugin extends plugin_class_default {
|
|
|
14573
14770
|
this.emit("replicator.sync.completed", { replicatorId, stats: this.stats });
|
|
14574
14771
|
}
|
|
14575
14772
|
async cleanup() {
|
|
14576
|
-
|
|
14773
|
+
const [ok, error] = await try_fn_default(async () => {
|
|
14577
14774
|
if (this.replicators && this.replicators.length > 0) {
|
|
14578
14775
|
const cleanupPromises = this.replicators.map(async (replicator) => {
|
|
14579
|
-
|
|
14776
|
+
const [replicatorOk, replicatorError] = await try_fn_default(async () => {
|
|
14580
14777
|
if (replicator && typeof replicator.cleanup === "function") {
|
|
14581
14778
|
await replicator.cleanup();
|
|
14582
14779
|
}
|
|
14583
|
-
}
|
|
14780
|
+
});
|
|
14781
|
+
if (!replicatorOk) {
|
|
14782
|
+
if (this.config.verbose) {
|
|
14783
|
+
console.warn(`[ReplicatorPlugin] Failed to cleanup replicator ${replicator.name || replicator.id}: ${replicatorError.message}`);
|
|
14784
|
+
}
|
|
14785
|
+
this.emit("replicator_cleanup_error", {
|
|
14786
|
+
replicator: replicator.name || replicator.id || "unknown",
|
|
14787
|
+
driver: replicator.driver || "unknown",
|
|
14788
|
+
error: replicatorError.message
|
|
14789
|
+
});
|
|
14584
14790
|
}
|
|
14585
14791
|
});
|
|
14586
14792
|
await Promise.allSettled(cleanupPromises);
|
|
@@ -14589,7 +14795,14 @@ class ReplicatorPlugin extends plugin_class_default {
|
|
|
14589
14795
|
this.database = null;
|
|
14590
14796
|
this.eventListenersInstalled.clear();
|
|
14591
14797
|
this.removeAllListeners();
|
|
14592
|
-
}
|
|
14798
|
+
});
|
|
14799
|
+
if (!ok) {
|
|
14800
|
+
if (this.config.verbose) {
|
|
14801
|
+
console.warn(`[ReplicatorPlugin] Failed to cleanup plugin: ${error.message}`);
|
|
14802
|
+
}
|
|
14803
|
+
this.emit("replicator_plugin_cleanup_error", {
|
|
14804
|
+
error: error.message
|
|
14805
|
+
});
|
|
14593
14806
|
}
|
|
14594
14807
|
}
|
|
14595
14808
|
}
|