s3db.js 7.3.5 → 7.3.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/PLUGINS.md +1285 -157
- package/dist/s3db.cjs.js +314 -83
- package/dist/s3db.cjs.min.js +1 -1
- package/dist/s3db.es.js +314 -83
- package/dist/s3db.es.min.js +1 -1
- package/dist/s3db.iife.js +314 -83
- package/dist/s3db.iife.min.js +1 -1
- package/mcp/server.js +1410 -0
- package/package.json +7 -1
- package/src/plugins/cache/filesystem-cache.class.js +9 -0
- package/src/plugins/replicator.plugin.js +130 -46
- package/src/plugins/replicators/bigquery-replicator.class.js +42 -5
- package/src/plugins/replicators/postgres-replicator.class.js +28 -2
- package/src/plugins/replicators/s3db-replicator.class.js +177 -68
- package/src/plugins/replicators/sqs-replicator.class.js +18 -1
package/dist/s3db.es.js
CHANGED
|
@@ -7079,6 +7079,12 @@ class FilesystemCache extends Cache {
|
|
|
7079
7079
|
}
|
|
7080
7080
|
async _clear(prefix) {
|
|
7081
7081
|
try {
|
|
7082
|
+
if (!await this._fileExists(this.directory)) {
|
|
7083
|
+
if (this.enableStats) {
|
|
7084
|
+
this.stats.clears++;
|
|
7085
|
+
}
|
|
7086
|
+
return true;
|
|
7087
|
+
}
|
|
7082
7088
|
const files = await readdir(this.directory);
|
|
7083
7089
|
const cacheFiles = files.filter((file) => {
|
|
7084
7090
|
if (!file.startsWith(this.prefix)) return false;
|
|
@@ -8950,6 +8956,9 @@ class BigqueryReplicator extends base_replicator_class_default {
|
|
|
8950
8956
|
await super.initialize(database);
|
|
8951
8957
|
const [ok, err, sdk] = await try_fn_default(() => import('@google-cloud/bigquery'));
|
|
8952
8958
|
if (!ok) {
|
|
8959
|
+
if (this.config.verbose) {
|
|
8960
|
+
console.warn(`[BigqueryReplicator] Failed to import BigQuery SDK: ${err.message}`);
|
|
8961
|
+
}
|
|
8953
8962
|
this.emit("initialization_error", { replicator: this.name, error: err.message });
|
|
8954
8963
|
throw err;
|
|
8955
8964
|
}
|
|
@@ -9019,19 +9028,28 @@ class BigqueryReplicator extends base_replicator_class_default {
|
|
|
9019
9028
|
const maxRetries = 2;
|
|
9020
9029
|
let lastError = null;
|
|
9021
9030
|
for (let attempt = 1; attempt <= maxRetries; attempt++) {
|
|
9022
|
-
|
|
9031
|
+
const [ok2, error] = await try_fn_default(async () => {
|
|
9023
9032
|
const [updateJob] = await this.bigqueryClient.createQueryJob({
|
|
9024
9033
|
query,
|
|
9025
9034
|
params,
|
|
9026
9035
|
location: this.location
|
|
9027
9036
|
});
|
|
9028
9037
|
await updateJob.getQueryResults();
|
|
9029
|
-
|
|
9038
|
+
return [updateJob];
|
|
9039
|
+
});
|
|
9040
|
+
if (ok2) {
|
|
9041
|
+
job = ok2;
|
|
9030
9042
|
break;
|
|
9031
|
-
}
|
|
9043
|
+
} else {
|
|
9032
9044
|
lastError = error;
|
|
9045
|
+
if (this.config.verbose) {
|
|
9046
|
+
console.warn(`[BigqueryReplicator] Update attempt ${attempt} failed: ${error.message}`);
|
|
9047
|
+
}
|
|
9033
9048
|
if (error?.message?.includes("streaming buffer") && attempt < maxRetries) {
|
|
9034
9049
|
const delaySeconds = 30;
|
|
9050
|
+
if (this.config.verbose) {
|
|
9051
|
+
console.warn(`[BigqueryReplicator] Retrying in ${delaySeconds} seconds due to streaming buffer issue`);
|
|
9052
|
+
}
|
|
9035
9053
|
await new Promise((resolve) => setTimeout(resolve, delaySeconds * 1e3));
|
|
9036
9054
|
continue;
|
|
9037
9055
|
}
|
|
@@ -9080,6 +9098,9 @@ class BigqueryReplicator extends base_replicator_class_default {
|
|
|
9080
9098
|
}
|
|
9081
9099
|
}
|
|
9082
9100
|
const success = errors.length === 0;
|
|
9101
|
+
if (errors.length > 0) {
|
|
9102
|
+
console.warn(`[BigqueryReplicator] Replication completed with errors for ${resourceName}:`, errors);
|
|
9103
|
+
}
|
|
9083
9104
|
this.emit("replicated", {
|
|
9084
9105
|
replicator: this.name,
|
|
9085
9106
|
resourceName,
|
|
@@ -9098,6 +9119,9 @@ class BigqueryReplicator extends base_replicator_class_default {
|
|
|
9098
9119
|
};
|
|
9099
9120
|
});
|
|
9100
9121
|
if (ok) return result;
|
|
9122
|
+
if (this.config.verbose) {
|
|
9123
|
+
console.warn(`[BigqueryReplicator] Replication failed for ${resourceName}: ${err.message}`);
|
|
9124
|
+
}
|
|
9101
9125
|
this.emit("replicator_error", {
|
|
9102
9126
|
replicator: this.name,
|
|
9103
9127
|
resourceName,
|
|
@@ -9118,8 +9142,17 @@ class BigqueryReplicator extends base_replicator_class_default {
|
|
|
9118
9142
|
record.id,
|
|
9119
9143
|
record.beforeData
|
|
9120
9144
|
));
|
|
9121
|
-
if (ok)
|
|
9122
|
-
|
|
9145
|
+
if (ok) {
|
|
9146
|
+
results.push(res);
|
|
9147
|
+
} else {
|
|
9148
|
+
if (this.config.verbose) {
|
|
9149
|
+
console.warn(`[BigqueryReplicator] Batch replication failed for record ${record.id}: ${err.message}`);
|
|
9150
|
+
}
|
|
9151
|
+
errors.push({ id: record.id, error: err.message });
|
|
9152
|
+
}
|
|
9153
|
+
}
|
|
9154
|
+
if (errors.length > 0) {
|
|
9155
|
+
console.warn(`[BigqueryReplicator] Batch replication completed with ${errors.length} error(s) for ${resourceName}:`, errors);
|
|
9123
9156
|
}
|
|
9124
9157
|
return {
|
|
9125
9158
|
success: errors.length === 0,
|
|
@@ -9135,6 +9168,9 @@ class BigqueryReplicator extends base_replicator_class_default {
|
|
|
9135
9168
|
return true;
|
|
9136
9169
|
});
|
|
9137
9170
|
if (ok) return true;
|
|
9171
|
+
if (this.config.verbose) {
|
|
9172
|
+
console.warn(`[BigqueryReplicator] Connection test failed: ${err.message}`);
|
|
9173
|
+
}
|
|
9138
9174
|
this.emit("connection_error", { replicator: this.name, error: err.message });
|
|
9139
9175
|
return false;
|
|
9140
9176
|
}
|
|
@@ -9222,6 +9258,9 @@ class PostgresReplicator extends base_replicator_class_default {
|
|
|
9222
9258
|
await super.initialize(database);
|
|
9223
9259
|
const [ok, err, sdk] = await try_fn_default(() => import('pg'));
|
|
9224
9260
|
if (!ok) {
|
|
9261
|
+
if (this.config.verbose) {
|
|
9262
|
+
console.warn(`[PostgresReplicator] Failed to import pg SDK: ${err.message}`);
|
|
9263
|
+
}
|
|
9225
9264
|
this.emit("initialization_error", {
|
|
9226
9265
|
replicator: this.name,
|
|
9227
9266
|
error: err.message
|
|
@@ -9345,6 +9384,9 @@ class PostgresReplicator extends base_replicator_class_default {
|
|
|
9345
9384
|
}
|
|
9346
9385
|
}
|
|
9347
9386
|
const success = errors.length === 0;
|
|
9387
|
+
if (errors.length > 0) {
|
|
9388
|
+
console.warn(`[PostgresReplicator] Replication completed with errors for ${resourceName}:`, errors);
|
|
9389
|
+
}
|
|
9348
9390
|
this.emit("replicated", {
|
|
9349
9391
|
replicator: this.name,
|
|
9350
9392
|
resourceName,
|
|
@@ -9363,6 +9405,9 @@ class PostgresReplicator extends base_replicator_class_default {
|
|
|
9363
9405
|
};
|
|
9364
9406
|
});
|
|
9365
9407
|
if (ok) return result;
|
|
9408
|
+
if (this.config.verbose) {
|
|
9409
|
+
console.warn(`[PostgresReplicator] Replication failed for ${resourceName}: ${err.message}`);
|
|
9410
|
+
}
|
|
9366
9411
|
this.emit("replicator_error", {
|
|
9367
9412
|
replicator: this.name,
|
|
9368
9413
|
resourceName,
|
|
@@ -9383,8 +9428,17 @@ class PostgresReplicator extends base_replicator_class_default {
|
|
|
9383
9428
|
record.id,
|
|
9384
9429
|
record.beforeData
|
|
9385
9430
|
));
|
|
9386
|
-
if (ok)
|
|
9387
|
-
|
|
9431
|
+
if (ok) {
|
|
9432
|
+
results.push(res);
|
|
9433
|
+
} else {
|
|
9434
|
+
if (this.config.verbose) {
|
|
9435
|
+
console.warn(`[PostgresReplicator] Batch replication failed for record ${record.id}: ${err.message}`);
|
|
9436
|
+
}
|
|
9437
|
+
errors.push({ id: record.id, error: err.message });
|
|
9438
|
+
}
|
|
9439
|
+
}
|
|
9440
|
+
if (errors.length > 0) {
|
|
9441
|
+
console.warn(`[PostgresReplicator] Batch replication completed with ${errors.length} error(s) for ${resourceName}:`, errors);
|
|
9388
9442
|
}
|
|
9389
9443
|
return {
|
|
9390
9444
|
success: errors.length === 0,
|
|
@@ -9399,6 +9453,9 @@ class PostgresReplicator extends base_replicator_class_default {
|
|
|
9399
9453
|
return true;
|
|
9400
9454
|
});
|
|
9401
9455
|
if (ok) return true;
|
|
9456
|
+
if (this.config.verbose) {
|
|
9457
|
+
console.warn(`[PostgresReplicator] Connection test failed: ${err.message}`);
|
|
9458
|
+
}
|
|
9402
9459
|
this.emit("connection_error", { replicator: this.name, error: err.message });
|
|
9403
9460
|
return false;
|
|
9404
9461
|
}
|
|
@@ -13133,7 +13190,7 @@ class Database extends EventEmitter {
|
|
|
13133
13190
|
super();
|
|
13134
13191
|
this.version = "1";
|
|
13135
13192
|
this.s3dbVersion = (() => {
|
|
13136
|
-
const [ok, err, version] = try_fn_default(() => true ? "7.3.
|
|
13193
|
+
const [ok, err, version] = try_fn_default(() => true ? "7.3.7" : "latest");
|
|
13137
13194
|
return ok ? version : "latest";
|
|
13138
13195
|
})();
|
|
13139
13196
|
this.resources = {};
|
|
@@ -13637,9 +13694,8 @@ class S3dbReplicator extends base_replicator_class_default {
|
|
|
13637
13694
|
const map = {};
|
|
13638
13695
|
for (const res of resources) {
|
|
13639
13696
|
if (typeof res === "string") map[normalizeResourceName$1(res)] = res;
|
|
13640
|
-
else if (Array.isArray(res) && typeof res[0] === "string") map[normalizeResourceName$1(res[0])] = res;
|
|
13641
13697
|
else if (typeof res === "object" && res.resource) {
|
|
13642
|
-
map[normalizeResourceName$1(res.resource)] =
|
|
13698
|
+
map[normalizeResourceName$1(res.resource)] = res;
|
|
13643
13699
|
}
|
|
13644
13700
|
}
|
|
13645
13701
|
return map;
|
|
@@ -13652,15 +13708,14 @@ class S3dbReplicator extends base_replicator_class_default {
|
|
|
13652
13708
|
else if (Array.isArray(dest)) {
|
|
13653
13709
|
map[normSrc] = dest.map((item) => {
|
|
13654
13710
|
if (typeof item === "string") return item;
|
|
13655
|
-
if (typeof item === "function") return item;
|
|
13656
13711
|
if (typeof item === "object" && item.resource) {
|
|
13657
|
-
return
|
|
13712
|
+
return item;
|
|
13658
13713
|
}
|
|
13659
13714
|
return item;
|
|
13660
13715
|
});
|
|
13661
13716
|
} else if (typeof dest === "function") map[normSrc] = dest;
|
|
13662
13717
|
else if (typeof dest === "object" && dest.resource) {
|
|
13663
|
-
map[normSrc] =
|
|
13718
|
+
map[normSrc] = dest;
|
|
13664
13719
|
}
|
|
13665
13720
|
}
|
|
13666
13721
|
return map;
|
|
@@ -13668,10 +13723,6 @@ class S3dbReplicator extends base_replicator_class_default {
|
|
|
13668
13723
|
if (typeof resources === "function") {
|
|
13669
13724
|
return resources;
|
|
13670
13725
|
}
|
|
13671
|
-
if (typeof resources === "string") {
|
|
13672
|
-
const map = { [normalizeResourceName$1(resources)]: resources };
|
|
13673
|
-
return map;
|
|
13674
|
-
}
|
|
13675
13726
|
return {};
|
|
13676
13727
|
}
|
|
13677
13728
|
validateConfig() {
|
|
@@ -13685,8 +13736,8 @@ class S3dbReplicator extends base_replicator_class_default {
|
|
|
13685
13736
|
return { isValid: errors.length === 0, errors };
|
|
13686
13737
|
}
|
|
13687
13738
|
async initialize(database) {
|
|
13688
|
-
|
|
13689
|
-
|
|
13739
|
+
await super.initialize(database);
|
|
13740
|
+
const [ok, err] = await try_fn_default(async () => {
|
|
13690
13741
|
if (this.client) {
|
|
13691
13742
|
this.targetDatabase = this.client;
|
|
13692
13743
|
} else if (this.connectionString) {
|
|
@@ -13705,7 +13756,11 @@ class S3dbReplicator extends base_replicator_class_default {
|
|
|
13705
13756
|
replicator: this.name,
|
|
13706
13757
|
target: this.connectionString || "client-provided"
|
|
13707
13758
|
});
|
|
13708
|
-
}
|
|
13759
|
+
});
|
|
13760
|
+
if (!ok) {
|
|
13761
|
+
if (this.config.verbose) {
|
|
13762
|
+
console.warn(`[S3dbReplicator] Initialization failed: ${err.message}`);
|
|
13763
|
+
}
|
|
13709
13764
|
throw err;
|
|
13710
13765
|
}
|
|
13711
13766
|
}
|
|
@@ -13724,18 +13779,77 @@ class S3dbReplicator extends base_replicator_class_default {
|
|
|
13724
13779
|
id = recordId;
|
|
13725
13780
|
}
|
|
13726
13781
|
const normResource = normalizeResourceName$1(resource);
|
|
13727
|
-
const
|
|
13728
|
-
|
|
13729
|
-
|
|
13782
|
+
const entry = this.resourcesMap[normResource];
|
|
13783
|
+
if (!entry) {
|
|
13784
|
+
throw new Error(`[S3dbReplicator] Resource not configured: ${resource}`);
|
|
13785
|
+
}
|
|
13786
|
+
if (Array.isArray(entry)) {
|
|
13787
|
+
const results = [];
|
|
13788
|
+
for (const destConfig of entry) {
|
|
13789
|
+
const [ok, error, result] = await try_fn_default(async () => {
|
|
13790
|
+
return await this._replicateToSingleDestination(destConfig, normResource, op, payload, id);
|
|
13791
|
+
});
|
|
13792
|
+
if (!ok) {
|
|
13793
|
+
if (this.config && this.config.verbose) {
|
|
13794
|
+
console.warn(`[S3dbReplicator] Failed to replicate to destination ${JSON.stringify(destConfig)}: ${error.message}`);
|
|
13795
|
+
}
|
|
13796
|
+
throw error;
|
|
13797
|
+
}
|
|
13798
|
+
results.push(result);
|
|
13799
|
+
}
|
|
13800
|
+
return results;
|
|
13801
|
+
} else {
|
|
13802
|
+
const [ok, error, result] = await try_fn_default(async () => {
|
|
13803
|
+
return await this._replicateToSingleDestination(entry, normResource, op, payload, id);
|
|
13804
|
+
});
|
|
13805
|
+
if (!ok) {
|
|
13806
|
+
if (this.config && this.config.verbose) {
|
|
13807
|
+
console.warn(`[S3dbReplicator] Failed to replicate to destination ${JSON.stringify(entry)}: ${error.message}`);
|
|
13808
|
+
}
|
|
13809
|
+
throw error;
|
|
13810
|
+
}
|
|
13811
|
+
return result;
|
|
13812
|
+
}
|
|
13813
|
+
}
|
|
13814
|
+
async _replicateToSingleDestination(destConfig, sourceResource, operation, data, recordId) {
|
|
13815
|
+
let destResourceName;
|
|
13816
|
+
if (typeof destConfig === "string") {
|
|
13817
|
+
destResourceName = destConfig;
|
|
13818
|
+
} else if (typeof destConfig === "object" && destConfig.resource) {
|
|
13819
|
+
destResourceName = destConfig.resource;
|
|
13820
|
+
} else {
|
|
13821
|
+
destResourceName = sourceResource;
|
|
13822
|
+
}
|
|
13823
|
+
if (typeof destConfig === "object" && destConfig.actions && Array.isArray(destConfig.actions)) {
|
|
13824
|
+
if (!destConfig.actions.includes(operation)) {
|
|
13825
|
+
return { skipped: true, reason: "action_not_supported", action: operation, destination: destResourceName };
|
|
13826
|
+
}
|
|
13827
|
+
}
|
|
13828
|
+
const destResourceObj = this._getDestResourceObj(destResourceName);
|
|
13829
|
+
let transformedData;
|
|
13830
|
+
if (typeof destConfig === "object" && destConfig.transform && typeof destConfig.transform === "function") {
|
|
13831
|
+
transformedData = destConfig.transform(data);
|
|
13832
|
+
if (transformedData && data && data.id && !transformedData.id) {
|
|
13833
|
+
transformedData.id = data.id;
|
|
13834
|
+
}
|
|
13835
|
+
} else if (typeof destConfig === "object" && destConfig.transformer && typeof destConfig.transformer === "function") {
|
|
13836
|
+
transformedData = destConfig.transformer(data);
|
|
13837
|
+
if (transformedData && data && data.id && !transformedData.id) {
|
|
13838
|
+
transformedData.id = data.id;
|
|
13839
|
+
}
|
|
13840
|
+
} else {
|
|
13841
|
+
transformedData = data;
|
|
13842
|
+
}
|
|
13843
|
+
if (!transformedData && data) transformedData = data;
|
|
13730
13844
|
let result;
|
|
13731
|
-
if (
|
|
13845
|
+
if (operation === "insert") {
|
|
13732
13846
|
result = await destResourceObj.insert(transformedData);
|
|
13733
|
-
} else if (
|
|
13734
|
-
result = await destResourceObj.update(
|
|
13735
|
-
} else if (
|
|
13736
|
-
result = await destResourceObj.delete(
|
|
13847
|
+
} else if (operation === "update") {
|
|
13848
|
+
result = await destResourceObj.update(recordId, transformedData);
|
|
13849
|
+
} else if (operation === "delete") {
|
|
13850
|
+
result = await destResourceObj.delete(recordId);
|
|
13737
13851
|
} else {
|
|
13738
|
-
throw new Error(`Invalid operation: ${
|
|
13852
|
+
throw new Error(`Invalid operation: ${operation}. Supported operations are: insert, update, delete`);
|
|
13739
13853
|
}
|
|
13740
13854
|
return result;
|
|
13741
13855
|
}
|
|
@@ -13744,13 +13858,25 @@ class S3dbReplicator extends base_replicator_class_default {
|
|
|
13744
13858
|
const entry = this.resourcesMap[normResource];
|
|
13745
13859
|
let result;
|
|
13746
13860
|
if (!entry) return data;
|
|
13747
|
-
if (Array.isArray(entry)
|
|
13748
|
-
|
|
13861
|
+
if (Array.isArray(entry)) {
|
|
13862
|
+
for (const item of entry) {
|
|
13863
|
+
if (typeof item === "object" && item.transform && typeof item.transform === "function") {
|
|
13864
|
+
result = item.transform(data);
|
|
13865
|
+
break;
|
|
13866
|
+
} else if (typeof item === "object" && item.transformer && typeof item.transformer === "function") {
|
|
13867
|
+
result = item.transformer(data);
|
|
13868
|
+
break;
|
|
13869
|
+
}
|
|
13870
|
+
}
|
|
13871
|
+
if (!result) result = data;
|
|
13872
|
+
} else if (typeof entry === "object") {
|
|
13873
|
+
if (typeof entry.transform === "function") {
|
|
13874
|
+
result = entry.transform(data);
|
|
13875
|
+
} else if (typeof entry.transformer === "function") {
|
|
13876
|
+
result = entry.transformer(data);
|
|
13877
|
+
}
|
|
13749
13878
|
} else if (typeof entry === "function") {
|
|
13750
13879
|
result = entry(data);
|
|
13751
|
-
} else if (typeof entry === "object") {
|
|
13752
|
-
if (typeof entry.transform === "function") result = entry.transform(data);
|
|
13753
|
-
else if (typeof entry.transformer === "function") result = entry.transformer(data);
|
|
13754
13880
|
} else {
|
|
13755
13881
|
result = data;
|
|
13756
13882
|
}
|
|
@@ -13763,9 +13889,11 @@ class S3dbReplicator extends base_replicator_class_default {
|
|
|
13763
13889
|
const entry = this.resourcesMap[normResource];
|
|
13764
13890
|
if (!entry) return resource;
|
|
13765
13891
|
if (Array.isArray(entry)) {
|
|
13766
|
-
|
|
13767
|
-
|
|
13768
|
-
|
|
13892
|
+
for (const item of entry) {
|
|
13893
|
+
if (typeof item === "string") return item;
|
|
13894
|
+
if (typeof item === "object" && item.resource) return item.resource;
|
|
13895
|
+
}
|
|
13896
|
+
return resource;
|
|
13769
13897
|
}
|
|
13770
13898
|
if (typeof entry === "string") return entry;
|
|
13771
13899
|
if (typeof entry === "function") return resource;
|
|
@@ -13773,8 +13901,7 @@ class S3dbReplicator extends base_replicator_class_default {
|
|
|
13773
13901
|
return resource;
|
|
13774
13902
|
}
|
|
13775
13903
|
_getDestResourceObj(resource) {
|
|
13776
|
-
|
|
13777
|
-
const available = Object.keys(this.client.resources);
|
|
13904
|
+
const available = Object.keys(this.client.resources || {});
|
|
13778
13905
|
const norm = normalizeResourceName$1(resource);
|
|
13779
13906
|
const found = available.find((r) => normalizeResourceName$1(r) === norm);
|
|
13780
13907
|
if (!found) {
|
|
@@ -13796,8 +13923,17 @@ class S3dbReplicator extends base_replicator_class_default {
|
|
|
13796
13923
|
data: record.data,
|
|
13797
13924
|
beforeData: record.beforeData
|
|
13798
13925
|
}));
|
|
13799
|
-
if (ok)
|
|
13800
|
-
|
|
13926
|
+
if (ok) {
|
|
13927
|
+
results.push(result);
|
|
13928
|
+
} else {
|
|
13929
|
+
if (this.config.verbose) {
|
|
13930
|
+
console.warn(`[S3dbReplicator] Batch replication failed for record ${record.id}: ${err.message}`);
|
|
13931
|
+
}
|
|
13932
|
+
errors.push({ id: record.id, error: err.message });
|
|
13933
|
+
}
|
|
13934
|
+
}
|
|
13935
|
+
if (errors.length > 0) {
|
|
13936
|
+
console.warn(`[S3dbReplicator] Batch replication completed with ${errors.length} error(s) for ${resourceName}:`, errors);
|
|
13801
13937
|
}
|
|
13802
13938
|
this.emit("batch_replicated", {
|
|
13803
13939
|
replicator: this.name,
|
|
@@ -13815,18 +13951,20 @@ class S3dbReplicator extends base_replicator_class_default {
|
|
|
13815
13951
|
}
|
|
13816
13952
|
async testConnection() {
|
|
13817
13953
|
const [ok, err] = await try_fn_default(async () => {
|
|
13818
|
-
if (!this.targetDatabase)
|
|
13819
|
-
|
|
13954
|
+
if (!this.targetDatabase) throw new Error("No target database configured");
|
|
13955
|
+
if (typeof this.targetDatabase.connect === "function") {
|
|
13956
|
+
await this.targetDatabase.connect();
|
|
13820
13957
|
}
|
|
13821
|
-
await this.targetDatabase.listResources();
|
|
13822
13958
|
return true;
|
|
13823
13959
|
});
|
|
13824
|
-
if (ok)
|
|
13825
|
-
|
|
13826
|
-
|
|
13827
|
-
|
|
13828
|
-
|
|
13829
|
-
|
|
13960
|
+
if (!ok) {
|
|
13961
|
+
if (this.config.verbose) {
|
|
13962
|
+
console.warn(`[S3dbReplicator] Connection test failed: ${err.message}`);
|
|
13963
|
+
}
|
|
13964
|
+
this.emit("connection_error", { replicator: this.name, error: err.message });
|
|
13965
|
+
return false;
|
|
13966
|
+
}
|
|
13967
|
+
return true;
|
|
13830
13968
|
}
|
|
13831
13969
|
async getStatus() {
|
|
13832
13970
|
const baseStatus = await super.getStatus();
|
|
@@ -13858,7 +13996,7 @@ class S3dbReplicator extends base_replicator_class_default {
|
|
|
13858
13996
|
} else {
|
|
13859
13997
|
return true;
|
|
13860
13998
|
}
|
|
13861
|
-
} else if (typeof item === "string"
|
|
13999
|
+
} else if (typeof item === "string") {
|
|
13862
14000
|
return true;
|
|
13863
14001
|
}
|
|
13864
14002
|
}
|
|
@@ -13985,6 +14123,9 @@ class SqsReplicator extends base_replicator_class_default {
|
|
|
13985
14123
|
if (!this.sqsClient) {
|
|
13986
14124
|
const [ok, err, sdk] = await try_fn_default(() => import('@aws-sdk/client-sqs'));
|
|
13987
14125
|
if (!ok) {
|
|
14126
|
+
if (this.config.verbose) {
|
|
14127
|
+
console.warn(`[SqsReplicator] Failed to import SQS SDK: ${err.message}`);
|
|
14128
|
+
}
|
|
13988
14129
|
this.emit("initialization_error", {
|
|
13989
14130
|
replicator: this.name,
|
|
13990
14131
|
error: err.message
|
|
@@ -14036,6 +14177,9 @@ class SqsReplicator extends base_replicator_class_default {
|
|
|
14036
14177
|
return { success: true, results };
|
|
14037
14178
|
});
|
|
14038
14179
|
if (ok) return result;
|
|
14180
|
+
if (this.config.verbose) {
|
|
14181
|
+
console.warn(`[SqsReplicator] Replication failed for ${resource}: ${err.message}`);
|
|
14182
|
+
}
|
|
14039
14183
|
this.emit("replicator_error", {
|
|
14040
14184
|
replicator: this.name,
|
|
14041
14185
|
resource,
|
|
@@ -14088,6 +14232,9 @@ class SqsReplicator extends base_replicator_class_default {
|
|
|
14088
14232
|
}
|
|
14089
14233
|
}
|
|
14090
14234
|
}
|
|
14235
|
+
if (errors.length > 0) {
|
|
14236
|
+
console.warn(`[SqsReplicator] Batch replication completed with ${errors.length} error(s) for ${resource}:`, errors);
|
|
14237
|
+
}
|
|
14091
14238
|
this.emit("batch_replicated", {
|
|
14092
14239
|
replicator: this.name,
|
|
14093
14240
|
resource,
|
|
@@ -14108,6 +14255,9 @@ class SqsReplicator extends base_replicator_class_default {
|
|
|
14108
14255
|
});
|
|
14109
14256
|
if (ok) return result;
|
|
14110
14257
|
const errorMessage = err?.message || err || "Unknown error";
|
|
14258
|
+
if (this.config.verbose) {
|
|
14259
|
+
console.warn(`[SqsReplicator] Batch replication failed for ${resource}: ${errorMessage}`);
|
|
14260
|
+
}
|
|
14111
14261
|
this.emit("batch_replicator_error", {
|
|
14112
14262
|
replicator: this.name,
|
|
14113
14263
|
resource,
|
|
@@ -14129,6 +14279,9 @@ class SqsReplicator extends base_replicator_class_default {
|
|
|
14129
14279
|
return true;
|
|
14130
14280
|
});
|
|
14131
14281
|
if (ok) return true;
|
|
14282
|
+
if (this.config.verbose) {
|
|
14283
|
+
console.warn(`[SqsReplicator] Connection test failed: ${err.message}`);
|
|
14284
|
+
}
|
|
14132
14285
|
this.emit("connection_error", {
|
|
14133
14286
|
replicator: this.name,
|
|
14134
14287
|
error: err.message
|
|
@@ -14225,25 +14378,37 @@ class ReplicatorPlugin extends plugin_class_default {
|
|
|
14225
14378
|
return;
|
|
14226
14379
|
}
|
|
14227
14380
|
resource.on("insert", async (data) => {
|
|
14228
|
-
|
|
14381
|
+
const [ok, error] = await try_fn_default(async () => {
|
|
14229
14382
|
const completeData = { ...data, createdAt: (/* @__PURE__ */ new Date()).toISOString() };
|
|
14230
14383
|
await plugin.processReplicatorEvent("insert", resource.name, completeData.id, completeData);
|
|
14231
|
-
}
|
|
14384
|
+
});
|
|
14385
|
+
if (!ok) {
|
|
14386
|
+
if (this.config.verbose) {
|
|
14387
|
+
console.warn(`[ReplicatorPlugin] Insert event failed for resource ${resource.name}: ${error.message}`);
|
|
14388
|
+
}
|
|
14232
14389
|
this.emit("error", { operation: "insert", error: error.message, resource: resource.name });
|
|
14233
14390
|
}
|
|
14234
14391
|
});
|
|
14235
14392
|
resource.on("update", async (data, beforeData) => {
|
|
14236
|
-
|
|
14393
|
+
const [ok, error] = await try_fn_default(async () => {
|
|
14237
14394
|
const completeData = { ...data, updatedAt: (/* @__PURE__ */ new Date()).toISOString() };
|
|
14238
14395
|
await plugin.processReplicatorEvent("update", resource.name, completeData.id, completeData, beforeData);
|
|
14239
|
-
}
|
|
14396
|
+
});
|
|
14397
|
+
if (!ok) {
|
|
14398
|
+
if (this.config.verbose) {
|
|
14399
|
+
console.warn(`[ReplicatorPlugin] Update event failed for resource ${resource.name}: ${error.message}`);
|
|
14400
|
+
}
|
|
14240
14401
|
this.emit("error", { operation: "update", error: error.message, resource: resource.name });
|
|
14241
14402
|
}
|
|
14242
14403
|
});
|
|
14243
14404
|
resource.on("delete", async (data) => {
|
|
14244
|
-
|
|
14405
|
+
const [ok, error] = await try_fn_default(async () => {
|
|
14245
14406
|
await plugin.processReplicatorEvent("delete", resource.name, data.id, data);
|
|
14246
|
-
}
|
|
14407
|
+
});
|
|
14408
|
+
if (!ok) {
|
|
14409
|
+
if (this.config.verbose) {
|
|
14410
|
+
console.warn(`[ReplicatorPlugin] Delete event failed for resource ${resource.name}: ${error.message}`);
|
|
14411
|
+
}
|
|
14247
14412
|
this.emit("error", { operation: "delete", error: error.message, resource: resource.name });
|
|
14248
14413
|
}
|
|
14249
14414
|
});
|
|
@@ -14259,13 +14424,17 @@ class ReplicatorPlugin extends plugin_class_default {
|
|
|
14259
14424
|
}
|
|
14260
14425
|
async setup(database) {
|
|
14261
14426
|
this.database = database;
|
|
14262
|
-
|
|
14427
|
+
const [initOk, initError] = await try_fn_default(async () => {
|
|
14263
14428
|
await this.initializeReplicators(database);
|
|
14264
|
-
}
|
|
14265
|
-
|
|
14266
|
-
|
|
14429
|
+
});
|
|
14430
|
+
if (!initOk) {
|
|
14431
|
+
if (this.config.verbose) {
|
|
14432
|
+
console.warn(`[ReplicatorPlugin] Replicator initialization failed: ${initError.message}`);
|
|
14433
|
+
}
|
|
14434
|
+
this.emit("error", { operation: "setup", error: initError.message });
|
|
14435
|
+
throw initError;
|
|
14267
14436
|
}
|
|
14268
|
-
|
|
14437
|
+
const [logOk, logError] = await try_fn_default(async () => {
|
|
14269
14438
|
if (this.config.replicatorLogResource) {
|
|
14270
14439
|
const logRes = await database.createResource({
|
|
14271
14440
|
name: this.config.replicatorLogResource,
|
|
@@ -14282,7 +14451,15 @@ class ReplicatorPlugin extends plugin_class_default {
|
|
|
14282
14451
|
}
|
|
14283
14452
|
});
|
|
14284
14453
|
}
|
|
14285
|
-
}
|
|
14454
|
+
});
|
|
14455
|
+
if (!logOk) {
|
|
14456
|
+
if (this.config.verbose) {
|
|
14457
|
+
console.warn(`[ReplicatorPlugin] Failed to create log resource ${this.config.replicatorLogResource}: ${logError.message}`);
|
|
14458
|
+
}
|
|
14459
|
+
this.emit("replicator_log_resource_creation_error", {
|
|
14460
|
+
resourceName: this.config.replicatorLogResource,
|
|
14461
|
+
error: logError.message
|
|
14462
|
+
});
|
|
14286
14463
|
}
|
|
14287
14464
|
await this.uploadMetadataFile(database);
|
|
14288
14465
|
const originalCreateResource = database.createResource.bind(database);
|
|
@@ -14325,21 +14502,28 @@ class ReplicatorPlugin extends plugin_class_default {
|
|
|
14325
14502
|
async retryWithBackoff(operation, maxRetries = 3) {
|
|
14326
14503
|
let lastError;
|
|
14327
14504
|
for (let attempt = 1; attempt <= maxRetries; attempt++) {
|
|
14328
|
-
|
|
14329
|
-
|
|
14330
|
-
|
|
14505
|
+
const [ok, error] = await try_fn_default(operation);
|
|
14506
|
+
if (ok) {
|
|
14507
|
+
return ok;
|
|
14508
|
+
} else {
|
|
14331
14509
|
lastError = error;
|
|
14510
|
+
if (this.config.verbose) {
|
|
14511
|
+
console.warn(`[ReplicatorPlugin] Retry attempt ${attempt}/${maxRetries} failed: ${error.message}`);
|
|
14512
|
+
}
|
|
14332
14513
|
if (attempt === maxRetries) {
|
|
14333
14514
|
throw error;
|
|
14334
14515
|
}
|
|
14335
14516
|
const delay = Math.pow(2, attempt - 1) * 1e3;
|
|
14517
|
+
if (this.config.verbose) {
|
|
14518
|
+
console.warn(`[ReplicatorPlugin] Waiting ${delay}ms before retry...`);
|
|
14519
|
+
}
|
|
14336
14520
|
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
14337
14521
|
}
|
|
14338
14522
|
}
|
|
14339
14523
|
throw lastError;
|
|
14340
14524
|
}
|
|
14341
14525
|
async logError(replicator, resourceName, operation, recordId, data, error) {
|
|
14342
|
-
|
|
14526
|
+
const [ok, logError] = await try_fn_default(async () => {
|
|
14343
14527
|
const logResourceName = this.config.replicatorLogResource;
|
|
14344
14528
|
if (this.database && this.database.resources && this.database.resources[logResourceName]) {
|
|
14345
14529
|
const logResource = this.database.resources[logResourceName];
|
|
@@ -14354,7 +14538,19 @@ class ReplicatorPlugin extends plugin_class_default {
|
|
|
14354
14538
|
status: "error"
|
|
14355
14539
|
});
|
|
14356
14540
|
}
|
|
14357
|
-
}
|
|
14541
|
+
});
|
|
14542
|
+
if (!ok) {
|
|
14543
|
+
if (this.config.verbose) {
|
|
14544
|
+
console.warn(`[ReplicatorPlugin] Failed to log error for ${resourceName}: ${logError.message}`);
|
|
14545
|
+
}
|
|
14546
|
+
this.emit("replicator_log_error", {
|
|
14547
|
+
replicator: replicator.name || replicator.id,
|
|
14548
|
+
resourceName,
|
|
14549
|
+
operation,
|
|
14550
|
+
recordId,
|
|
14551
|
+
originalError: error.message,
|
|
14552
|
+
logError: logError.message
|
|
14553
|
+
});
|
|
14358
14554
|
}
|
|
14359
14555
|
}
|
|
14360
14556
|
async processReplicatorEvent(operation, resourceName, recordId, data, beforeData = null) {
|
|
@@ -14367,8 +14563,8 @@ class ReplicatorPlugin extends plugin_class_default {
|
|
|
14367
14563
|
return;
|
|
14368
14564
|
}
|
|
14369
14565
|
const promises = applicableReplicators.map(async (replicator) => {
|
|
14370
|
-
|
|
14371
|
-
const
|
|
14566
|
+
const [ok, error, result] = await try_fn_default(async () => {
|
|
14567
|
+
const result2 = await this.retryWithBackoff(
|
|
14372
14568
|
() => replicator.replicate(resourceName, operation, data, recordId, beforeData),
|
|
14373
14569
|
this.config.maxRetries
|
|
14374
14570
|
);
|
|
@@ -14377,11 +14573,17 @@ class ReplicatorPlugin extends plugin_class_default {
|
|
|
14377
14573
|
resourceName,
|
|
14378
14574
|
operation,
|
|
14379
14575
|
recordId,
|
|
14380
|
-
result,
|
|
14576
|
+
result: result2,
|
|
14381
14577
|
success: true
|
|
14382
14578
|
});
|
|
14579
|
+
return result2;
|
|
14580
|
+
});
|
|
14581
|
+
if (ok) {
|
|
14383
14582
|
return result;
|
|
14384
|
-
}
|
|
14583
|
+
} else {
|
|
14584
|
+
if (this.config.verbose) {
|
|
14585
|
+
console.warn(`[ReplicatorPlugin] Replication failed for ${replicator.name || replicator.id} on ${resourceName}: ${error.message}`);
|
|
14586
|
+
}
|
|
14385
14587
|
this.emit("replicator_error", {
|
|
14386
14588
|
replicator: replicator.name || replicator.id,
|
|
14387
14589
|
resourceName,
|
|
@@ -14406,11 +14608,14 @@ class ReplicatorPlugin extends plugin_class_default {
|
|
|
14406
14608
|
return;
|
|
14407
14609
|
}
|
|
14408
14610
|
const promises = applicableReplicators.map(async (replicator) => {
|
|
14409
|
-
|
|
14611
|
+
const [wrapperOk, wrapperError] = await try_fn_default(async () => {
|
|
14410
14612
|
const [ok, err, result] = await try_fn_default(
|
|
14411
14613
|
() => replicator.replicate(item.resourceName, item.operation, item.data, item.recordId, item.beforeData)
|
|
14412
14614
|
);
|
|
14413
14615
|
if (!ok) {
|
|
14616
|
+
if (this.config.verbose) {
|
|
14617
|
+
console.warn(`[ReplicatorPlugin] Replicator item processing failed for ${replicator.name || replicator.id} on ${item.resourceName}: ${err.message}`);
|
|
14618
|
+
}
|
|
14414
14619
|
this.emit("replicator_error", {
|
|
14415
14620
|
replicator: replicator.name || replicator.id,
|
|
14416
14621
|
resourceName: item.resourceName,
|
|
@@ -14432,18 +14637,24 @@ class ReplicatorPlugin extends plugin_class_default {
|
|
|
14432
14637
|
success: true
|
|
14433
14638
|
});
|
|
14434
14639
|
return { success: true, result };
|
|
14435
|
-
}
|
|
14640
|
+
});
|
|
14641
|
+
if (wrapperOk) {
|
|
14642
|
+
return wrapperOk;
|
|
14643
|
+
} else {
|
|
14644
|
+
if (this.config.verbose) {
|
|
14645
|
+
console.warn(`[ReplicatorPlugin] Wrapper processing failed for ${replicator.name || replicator.id} on ${item.resourceName}: ${wrapperError.message}`);
|
|
14646
|
+
}
|
|
14436
14647
|
this.emit("replicator_error", {
|
|
14437
14648
|
replicator: replicator.name || replicator.id,
|
|
14438
14649
|
resourceName: item.resourceName,
|
|
14439
14650
|
operation: item.operation,
|
|
14440
14651
|
recordId: item.recordId,
|
|
14441
|
-
error:
|
|
14652
|
+
error: wrapperError.message
|
|
14442
14653
|
});
|
|
14443
14654
|
if (this.config.logErrors && this.database) {
|
|
14444
|
-
await this.logError(replicator, item.resourceName, item.operation, item.recordId, item.data,
|
|
14655
|
+
await this.logError(replicator, item.resourceName, item.operation, item.recordId, item.data, wrapperError);
|
|
14445
14656
|
}
|
|
14446
|
-
return { success: false, error:
|
|
14657
|
+
return { success: false, error: wrapperError.message };
|
|
14447
14658
|
}
|
|
14448
14659
|
});
|
|
14449
14660
|
return Promise.allSettled(promises);
|
|
@@ -14465,9 +14676,13 @@ class ReplicatorPlugin extends plugin_class_default {
|
|
|
14465
14676
|
timestamp: typeof item.timestamp === "number" ? item.timestamp : Date.now(),
|
|
14466
14677
|
createdAt: item.createdAt || (/* @__PURE__ */ new Date()).toISOString().slice(0, 10)
|
|
14467
14678
|
};
|
|
14468
|
-
|
|
14679
|
+
const [ok, err] = await try_fn_default(async () => {
|
|
14469
14680
|
await logRes.insert(logItem);
|
|
14470
|
-
}
|
|
14681
|
+
});
|
|
14682
|
+
if (!ok) {
|
|
14683
|
+
if (this.config.verbose) {
|
|
14684
|
+
console.warn(`[ReplicatorPlugin] Failed to log replicator item: ${err.message}`);
|
|
14685
|
+
}
|
|
14471
14686
|
this.emit("replicator.log.failed", { error: err, item });
|
|
14472
14687
|
}
|
|
14473
14688
|
}
|
|
@@ -14573,14 +14788,23 @@ class ReplicatorPlugin extends plugin_class_default {
|
|
|
14573
14788
|
this.emit("replicator.sync.completed", { replicatorId, stats: this.stats });
|
|
14574
14789
|
}
|
|
14575
14790
|
async cleanup() {
|
|
14576
|
-
|
|
14791
|
+
const [ok, error] = await try_fn_default(async () => {
|
|
14577
14792
|
if (this.replicators && this.replicators.length > 0) {
|
|
14578
14793
|
const cleanupPromises = this.replicators.map(async (replicator) => {
|
|
14579
|
-
|
|
14794
|
+
const [replicatorOk, replicatorError] = await try_fn_default(async () => {
|
|
14580
14795
|
if (replicator && typeof replicator.cleanup === "function") {
|
|
14581
14796
|
await replicator.cleanup();
|
|
14582
14797
|
}
|
|
14583
|
-
}
|
|
14798
|
+
});
|
|
14799
|
+
if (!replicatorOk) {
|
|
14800
|
+
if (this.config.verbose) {
|
|
14801
|
+
console.warn(`[ReplicatorPlugin] Failed to cleanup replicator ${replicator.name || replicator.id}: ${replicatorError.message}`);
|
|
14802
|
+
}
|
|
14803
|
+
this.emit("replicator_cleanup_error", {
|
|
14804
|
+
replicator: replicator.name || replicator.id || "unknown",
|
|
14805
|
+
driver: replicator.driver || "unknown",
|
|
14806
|
+
error: replicatorError.message
|
|
14807
|
+
});
|
|
14584
14808
|
}
|
|
14585
14809
|
});
|
|
14586
14810
|
await Promise.allSettled(cleanupPromises);
|
|
@@ -14589,7 +14813,14 @@ class ReplicatorPlugin extends plugin_class_default {
|
|
|
14589
14813
|
this.database = null;
|
|
14590
14814
|
this.eventListenersInstalled.clear();
|
|
14591
14815
|
this.removeAllListeners();
|
|
14592
|
-
}
|
|
14816
|
+
});
|
|
14817
|
+
if (!ok) {
|
|
14818
|
+
if (this.config.verbose) {
|
|
14819
|
+
console.warn(`[ReplicatorPlugin] Failed to cleanup plugin: ${error.message}`);
|
|
14820
|
+
}
|
|
14821
|
+
this.emit("replicator_plugin_cleanup_error", {
|
|
14822
|
+
error: error.message
|
|
14823
|
+
});
|
|
14593
14824
|
}
|
|
14594
14825
|
}
|
|
14595
14826
|
}
|