s3db.js 7.3.5 → 7.3.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/PLUGINS.md +1285 -157
- package/dist/s3db.cjs.js +314 -83
- package/dist/s3db.cjs.min.js +1 -1
- package/dist/s3db.es.js +314 -83
- package/dist/s3db.es.min.js +1 -1
- package/dist/s3db.iife.js +314 -83
- package/dist/s3db.iife.min.js +1 -1
- package/mcp/server.js +1410 -0
- package/package.json +7 -1
- package/src/plugins/cache/filesystem-cache.class.js +9 -0
- package/src/plugins/replicator.plugin.js +130 -46
- package/src/plugins/replicators/bigquery-replicator.class.js +42 -5
- package/src/plugins/replicators/postgres-replicator.class.js +28 -2
- package/src/plugins/replicators/s3db-replicator.class.js +177 -68
- package/src/plugins/replicators/sqs-replicator.class.js +18 -1
package/dist/s3db.cjs.js
CHANGED
|
@@ -7083,6 +7083,12 @@ class FilesystemCache extends Cache {
|
|
|
7083
7083
|
}
|
|
7084
7084
|
async _clear(prefix) {
|
|
7085
7085
|
try {
|
|
7086
|
+
if (!await this._fileExists(this.directory)) {
|
|
7087
|
+
if (this.enableStats) {
|
|
7088
|
+
this.stats.clears++;
|
|
7089
|
+
}
|
|
7090
|
+
return true;
|
|
7091
|
+
}
|
|
7086
7092
|
const files = await promises.readdir(this.directory);
|
|
7087
7093
|
const cacheFiles = files.filter((file) => {
|
|
7088
7094
|
if (!file.startsWith(this.prefix)) return false;
|
|
@@ -8954,6 +8960,9 @@ class BigqueryReplicator extends base_replicator_class_default {
|
|
|
8954
8960
|
await super.initialize(database);
|
|
8955
8961
|
const [ok, err, sdk] = await try_fn_default(() => import('@google-cloud/bigquery'));
|
|
8956
8962
|
if (!ok) {
|
|
8963
|
+
if (this.config.verbose) {
|
|
8964
|
+
console.warn(`[BigqueryReplicator] Failed to import BigQuery SDK: ${err.message}`);
|
|
8965
|
+
}
|
|
8957
8966
|
this.emit("initialization_error", { replicator: this.name, error: err.message });
|
|
8958
8967
|
throw err;
|
|
8959
8968
|
}
|
|
@@ -9023,19 +9032,28 @@ class BigqueryReplicator extends base_replicator_class_default {
|
|
|
9023
9032
|
const maxRetries = 2;
|
|
9024
9033
|
let lastError = null;
|
|
9025
9034
|
for (let attempt = 1; attempt <= maxRetries; attempt++) {
|
|
9026
|
-
|
|
9035
|
+
const [ok2, error] = await try_fn_default(async () => {
|
|
9027
9036
|
const [updateJob] = await this.bigqueryClient.createQueryJob({
|
|
9028
9037
|
query,
|
|
9029
9038
|
params,
|
|
9030
9039
|
location: this.location
|
|
9031
9040
|
});
|
|
9032
9041
|
await updateJob.getQueryResults();
|
|
9033
|
-
|
|
9042
|
+
return [updateJob];
|
|
9043
|
+
});
|
|
9044
|
+
if (ok2) {
|
|
9045
|
+
job = ok2;
|
|
9034
9046
|
break;
|
|
9035
|
-
}
|
|
9047
|
+
} else {
|
|
9036
9048
|
lastError = error;
|
|
9049
|
+
if (this.config.verbose) {
|
|
9050
|
+
console.warn(`[BigqueryReplicator] Update attempt ${attempt} failed: ${error.message}`);
|
|
9051
|
+
}
|
|
9037
9052
|
if (error?.message?.includes("streaming buffer") && attempt < maxRetries) {
|
|
9038
9053
|
const delaySeconds = 30;
|
|
9054
|
+
if (this.config.verbose) {
|
|
9055
|
+
console.warn(`[BigqueryReplicator] Retrying in ${delaySeconds} seconds due to streaming buffer issue`);
|
|
9056
|
+
}
|
|
9039
9057
|
await new Promise((resolve) => setTimeout(resolve, delaySeconds * 1e3));
|
|
9040
9058
|
continue;
|
|
9041
9059
|
}
|
|
@@ -9084,6 +9102,9 @@ class BigqueryReplicator extends base_replicator_class_default {
|
|
|
9084
9102
|
}
|
|
9085
9103
|
}
|
|
9086
9104
|
const success = errors.length === 0;
|
|
9105
|
+
if (errors.length > 0) {
|
|
9106
|
+
console.warn(`[BigqueryReplicator] Replication completed with errors for ${resourceName}:`, errors);
|
|
9107
|
+
}
|
|
9087
9108
|
this.emit("replicated", {
|
|
9088
9109
|
replicator: this.name,
|
|
9089
9110
|
resourceName,
|
|
@@ -9102,6 +9123,9 @@ class BigqueryReplicator extends base_replicator_class_default {
|
|
|
9102
9123
|
};
|
|
9103
9124
|
});
|
|
9104
9125
|
if (ok) return result;
|
|
9126
|
+
if (this.config.verbose) {
|
|
9127
|
+
console.warn(`[BigqueryReplicator] Replication failed for ${resourceName}: ${err.message}`);
|
|
9128
|
+
}
|
|
9105
9129
|
this.emit("replicator_error", {
|
|
9106
9130
|
replicator: this.name,
|
|
9107
9131
|
resourceName,
|
|
@@ -9122,8 +9146,17 @@ class BigqueryReplicator extends base_replicator_class_default {
|
|
|
9122
9146
|
record.id,
|
|
9123
9147
|
record.beforeData
|
|
9124
9148
|
));
|
|
9125
|
-
if (ok)
|
|
9126
|
-
|
|
9149
|
+
if (ok) {
|
|
9150
|
+
results.push(res);
|
|
9151
|
+
} else {
|
|
9152
|
+
if (this.config.verbose) {
|
|
9153
|
+
console.warn(`[BigqueryReplicator] Batch replication failed for record ${record.id}: ${err.message}`);
|
|
9154
|
+
}
|
|
9155
|
+
errors.push({ id: record.id, error: err.message });
|
|
9156
|
+
}
|
|
9157
|
+
}
|
|
9158
|
+
if (errors.length > 0) {
|
|
9159
|
+
console.warn(`[BigqueryReplicator] Batch replication completed with ${errors.length} error(s) for ${resourceName}:`, errors);
|
|
9127
9160
|
}
|
|
9128
9161
|
return {
|
|
9129
9162
|
success: errors.length === 0,
|
|
@@ -9139,6 +9172,9 @@ class BigqueryReplicator extends base_replicator_class_default {
|
|
|
9139
9172
|
return true;
|
|
9140
9173
|
});
|
|
9141
9174
|
if (ok) return true;
|
|
9175
|
+
if (this.config.verbose) {
|
|
9176
|
+
console.warn(`[BigqueryReplicator] Connection test failed: ${err.message}`);
|
|
9177
|
+
}
|
|
9142
9178
|
this.emit("connection_error", { replicator: this.name, error: err.message });
|
|
9143
9179
|
return false;
|
|
9144
9180
|
}
|
|
@@ -9226,6 +9262,9 @@ class PostgresReplicator extends base_replicator_class_default {
|
|
|
9226
9262
|
await super.initialize(database);
|
|
9227
9263
|
const [ok, err, sdk] = await try_fn_default(() => import('pg'));
|
|
9228
9264
|
if (!ok) {
|
|
9265
|
+
if (this.config.verbose) {
|
|
9266
|
+
console.warn(`[PostgresReplicator] Failed to import pg SDK: ${err.message}`);
|
|
9267
|
+
}
|
|
9229
9268
|
this.emit("initialization_error", {
|
|
9230
9269
|
replicator: this.name,
|
|
9231
9270
|
error: err.message
|
|
@@ -9349,6 +9388,9 @@ class PostgresReplicator extends base_replicator_class_default {
|
|
|
9349
9388
|
}
|
|
9350
9389
|
}
|
|
9351
9390
|
const success = errors.length === 0;
|
|
9391
|
+
if (errors.length > 0) {
|
|
9392
|
+
console.warn(`[PostgresReplicator] Replication completed with errors for ${resourceName}:`, errors);
|
|
9393
|
+
}
|
|
9352
9394
|
this.emit("replicated", {
|
|
9353
9395
|
replicator: this.name,
|
|
9354
9396
|
resourceName,
|
|
@@ -9367,6 +9409,9 @@ class PostgresReplicator extends base_replicator_class_default {
|
|
|
9367
9409
|
};
|
|
9368
9410
|
});
|
|
9369
9411
|
if (ok) return result;
|
|
9412
|
+
if (this.config.verbose) {
|
|
9413
|
+
console.warn(`[PostgresReplicator] Replication failed for ${resourceName}: ${err.message}`);
|
|
9414
|
+
}
|
|
9370
9415
|
this.emit("replicator_error", {
|
|
9371
9416
|
replicator: this.name,
|
|
9372
9417
|
resourceName,
|
|
@@ -9387,8 +9432,17 @@ class PostgresReplicator extends base_replicator_class_default {
|
|
|
9387
9432
|
record.id,
|
|
9388
9433
|
record.beforeData
|
|
9389
9434
|
));
|
|
9390
|
-
if (ok)
|
|
9391
|
-
|
|
9435
|
+
if (ok) {
|
|
9436
|
+
results.push(res);
|
|
9437
|
+
} else {
|
|
9438
|
+
if (this.config.verbose) {
|
|
9439
|
+
console.warn(`[PostgresReplicator] Batch replication failed for record ${record.id}: ${err.message}`);
|
|
9440
|
+
}
|
|
9441
|
+
errors.push({ id: record.id, error: err.message });
|
|
9442
|
+
}
|
|
9443
|
+
}
|
|
9444
|
+
if (errors.length > 0) {
|
|
9445
|
+
console.warn(`[PostgresReplicator] Batch replication completed with ${errors.length} error(s) for ${resourceName}:`, errors);
|
|
9392
9446
|
}
|
|
9393
9447
|
return {
|
|
9394
9448
|
success: errors.length === 0,
|
|
@@ -9403,6 +9457,9 @@ class PostgresReplicator extends base_replicator_class_default {
|
|
|
9403
9457
|
return true;
|
|
9404
9458
|
});
|
|
9405
9459
|
if (ok) return true;
|
|
9460
|
+
if (this.config.verbose) {
|
|
9461
|
+
console.warn(`[PostgresReplicator] Connection test failed: ${err.message}`);
|
|
9462
|
+
}
|
|
9406
9463
|
this.emit("connection_error", { replicator: this.name, error: err.message });
|
|
9407
9464
|
return false;
|
|
9408
9465
|
}
|
|
@@ -13137,7 +13194,7 @@ class Database extends EventEmitter {
|
|
|
13137
13194
|
super();
|
|
13138
13195
|
this.version = "1";
|
|
13139
13196
|
this.s3dbVersion = (() => {
|
|
13140
|
-
const [ok, err, version] = try_fn_default(() => true ? "7.3.
|
|
13197
|
+
const [ok, err, version] = try_fn_default(() => true ? "7.3.7" : "latest");
|
|
13141
13198
|
return ok ? version : "latest";
|
|
13142
13199
|
})();
|
|
13143
13200
|
this.resources = {};
|
|
@@ -13641,9 +13698,8 @@ class S3dbReplicator extends base_replicator_class_default {
|
|
|
13641
13698
|
const map = {};
|
|
13642
13699
|
for (const res of resources) {
|
|
13643
13700
|
if (typeof res === "string") map[normalizeResourceName$1(res)] = res;
|
|
13644
|
-
else if (Array.isArray(res) && typeof res[0] === "string") map[normalizeResourceName$1(res[0])] = res;
|
|
13645
13701
|
else if (typeof res === "object" && res.resource) {
|
|
13646
|
-
map[normalizeResourceName$1(res.resource)] =
|
|
13702
|
+
map[normalizeResourceName$1(res.resource)] = res;
|
|
13647
13703
|
}
|
|
13648
13704
|
}
|
|
13649
13705
|
return map;
|
|
@@ -13656,15 +13712,14 @@ class S3dbReplicator extends base_replicator_class_default {
|
|
|
13656
13712
|
else if (Array.isArray(dest)) {
|
|
13657
13713
|
map[normSrc] = dest.map((item) => {
|
|
13658
13714
|
if (typeof item === "string") return item;
|
|
13659
|
-
if (typeof item === "function") return item;
|
|
13660
13715
|
if (typeof item === "object" && item.resource) {
|
|
13661
|
-
return
|
|
13716
|
+
return item;
|
|
13662
13717
|
}
|
|
13663
13718
|
return item;
|
|
13664
13719
|
});
|
|
13665
13720
|
} else if (typeof dest === "function") map[normSrc] = dest;
|
|
13666
13721
|
else if (typeof dest === "object" && dest.resource) {
|
|
13667
|
-
map[normSrc] =
|
|
13722
|
+
map[normSrc] = dest;
|
|
13668
13723
|
}
|
|
13669
13724
|
}
|
|
13670
13725
|
return map;
|
|
@@ -13672,10 +13727,6 @@ class S3dbReplicator extends base_replicator_class_default {
|
|
|
13672
13727
|
if (typeof resources === "function") {
|
|
13673
13728
|
return resources;
|
|
13674
13729
|
}
|
|
13675
|
-
if (typeof resources === "string") {
|
|
13676
|
-
const map = { [normalizeResourceName$1(resources)]: resources };
|
|
13677
|
-
return map;
|
|
13678
|
-
}
|
|
13679
13730
|
return {};
|
|
13680
13731
|
}
|
|
13681
13732
|
validateConfig() {
|
|
@@ -13689,8 +13740,8 @@ class S3dbReplicator extends base_replicator_class_default {
|
|
|
13689
13740
|
return { isValid: errors.length === 0, errors };
|
|
13690
13741
|
}
|
|
13691
13742
|
async initialize(database) {
|
|
13692
|
-
|
|
13693
|
-
|
|
13743
|
+
await super.initialize(database);
|
|
13744
|
+
const [ok, err] = await try_fn_default(async () => {
|
|
13694
13745
|
if (this.client) {
|
|
13695
13746
|
this.targetDatabase = this.client;
|
|
13696
13747
|
} else if (this.connectionString) {
|
|
@@ -13709,7 +13760,11 @@ class S3dbReplicator extends base_replicator_class_default {
|
|
|
13709
13760
|
replicator: this.name,
|
|
13710
13761
|
target: this.connectionString || "client-provided"
|
|
13711
13762
|
});
|
|
13712
|
-
}
|
|
13763
|
+
});
|
|
13764
|
+
if (!ok) {
|
|
13765
|
+
if (this.config.verbose) {
|
|
13766
|
+
console.warn(`[S3dbReplicator] Initialization failed: ${err.message}`);
|
|
13767
|
+
}
|
|
13713
13768
|
throw err;
|
|
13714
13769
|
}
|
|
13715
13770
|
}
|
|
@@ -13728,18 +13783,77 @@ class S3dbReplicator extends base_replicator_class_default {
|
|
|
13728
13783
|
id = recordId;
|
|
13729
13784
|
}
|
|
13730
13785
|
const normResource = normalizeResourceName$1(resource);
|
|
13731
|
-
const
|
|
13732
|
-
|
|
13733
|
-
|
|
13786
|
+
const entry = this.resourcesMap[normResource];
|
|
13787
|
+
if (!entry) {
|
|
13788
|
+
throw new Error(`[S3dbReplicator] Resource not configured: ${resource}`);
|
|
13789
|
+
}
|
|
13790
|
+
if (Array.isArray(entry)) {
|
|
13791
|
+
const results = [];
|
|
13792
|
+
for (const destConfig of entry) {
|
|
13793
|
+
const [ok, error, result] = await try_fn_default(async () => {
|
|
13794
|
+
return await this._replicateToSingleDestination(destConfig, normResource, op, payload, id);
|
|
13795
|
+
});
|
|
13796
|
+
if (!ok) {
|
|
13797
|
+
if (this.config && this.config.verbose) {
|
|
13798
|
+
console.warn(`[S3dbReplicator] Failed to replicate to destination ${JSON.stringify(destConfig)}: ${error.message}`);
|
|
13799
|
+
}
|
|
13800
|
+
throw error;
|
|
13801
|
+
}
|
|
13802
|
+
results.push(result);
|
|
13803
|
+
}
|
|
13804
|
+
return results;
|
|
13805
|
+
} else {
|
|
13806
|
+
const [ok, error, result] = await try_fn_default(async () => {
|
|
13807
|
+
return await this._replicateToSingleDestination(entry, normResource, op, payload, id);
|
|
13808
|
+
});
|
|
13809
|
+
if (!ok) {
|
|
13810
|
+
if (this.config && this.config.verbose) {
|
|
13811
|
+
console.warn(`[S3dbReplicator] Failed to replicate to destination ${JSON.stringify(entry)}: ${error.message}`);
|
|
13812
|
+
}
|
|
13813
|
+
throw error;
|
|
13814
|
+
}
|
|
13815
|
+
return result;
|
|
13816
|
+
}
|
|
13817
|
+
}
|
|
13818
|
+
async _replicateToSingleDestination(destConfig, sourceResource, operation, data, recordId) {
|
|
13819
|
+
let destResourceName;
|
|
13820
|
+
if (typeof destConfig === "string") {
|
|
13821
|
+
destResourceName = destConfig;
|
|
13822
|
+
} else if (typeof destConfig === "object" && destConfig.resource) {
|
|
13823
|
+
destResourceName = destConfig.resource;
|
|
13824
|
+
} else {
|
|
13825
|
+
destResourceName = sourceResource;
|
|
13826
|
+
}
|
|
13827
|
+
if (typeof destConfig === "object" && destConfig.actions && Array.isArray(destConfig.actions)) {
|
|
13828
|
+
if (!destConfig.actions.includes(operation)) {
|
|
13829
|
+
return { skipped: true, reason: "action_not_supported", action: operation, destination: destResourceName };
|
|
13830
|
+
}
|
|
13831
|
+
}
|
|
13832
|
+
const destResourceObj = this._getDestResourceObj(destResourceName);
|
|
13833
|
+
let transformedData;
|
|
13834
|
+
if (typeof destConfig === "object" && destConfig.transform && typeof destConfig.transform === "function") {
|
|
13835
|
+
transformedData = destConfig.transform(data);
|
|
13836
|
+
if (transformedData && data && data.id && !transformedData.id) {
|
|
13837
|
+
transformedData.id = data.id;
|
|
13838
|
+
}
|
|
13839
|
+
} else if (typeof destConfig === "object" && destConfig.transformer && typeof destConfig.transformer === "function") {
|
|
13840
|
+
transformedData = destConfig.transformer(data);
|
|
13841
|
+
if (transformedData && data && data.id && !transformedData.id) {
|
|
13842
|
+
transformedData.id = data.id;
|
|
13843
|
+
}
|
|
13844
|
+
} else {
|
|
13845
|
+
transformedData = data;
|
|
13846
|
+
}
|
|
13847
|
+
if (!transformedData && data) transformedData = data;
|
|
13734
13848
|
let result;
|
|
13735
|
-
if (
|
|
13849
|
+
if (operation === "insert") {
|
|
13736
13850
|
result = await destResourceObj.insert(transformedData);
|
|
13737
|
-
} else if (
|
|
13738
|
-
result = await destResourceObj.update(
|
|
13739
|
-
} else if (
|
|
13740
|
-
result = await destResourceObj.delete(
|
|
13851
|
+
} else if (operation === "update") {
|
|
13852
|
+
result = await destResourceObj.update(recordId, transformedData);
|
|
13853
|
+
} else if (operation === "delete") {
|
|
13854
|
+
result = await destResourceObj.delete(recordId);
|
|
13741
13855
|
} else {
|
|
13742
|
-
throw new Error(`Invalid operation: ${
|
|
13856
|
+
throw new Error(`Invalid operation: ${operation}. Supported operations are: insert, update, delete`);
|
|
13743
13857
|
}
|
|
13744
13858
|
return result;
|
|
13745
13859
|
}
|
|
@@ -13748,13 +13862,25 @@ class S3dbReplicator extends base_replicator_class_default {
|
|
|
13748
13862
|
const entry = this.resourcesMap[normResource];
|
|
13749
13863
|
let result;
|
|
13750
13864
|
if (!entry) return data;
|
|
13751
|
-
if (Array.isArray(entry)
|
|
13752
|
-
|
|
13865
|
+
if (Array.isArray(entry)) {
|
|
13866
|
+
for (const item of entry) {
|
|
13867
|
+
if (typeof item === "object" && item.transform && typeof item.transform === "function") {
|
|
13868
|
+
result = item.transform(data);
|
|
13869
|
+
break;
|
|
13870
|
+
} else if (typeof item === "object" && item.transformer && typeof item.transformer === "function") {
|
|
13871
|
+
result = item.transformer(data);
|
|
13872
|
+
break;
|
|
13873
|
+
}
|
|
13874
|
+
}
|
|
13875
|
+
if (!result) result = data;
|
|
13876
|
+
} else if (typeof entry === "object") {
|
|
13877
|
+
if (typeof entry.transform === "function") {
|
|
13878
|
+
result = entry.transform(data);
|
|
13879
|
+
} else if (typeof entry.transformer === "function") {
|
|
13880
|
+
result = entry.transformer(data);
|
|
13881
|
+
}
|
|
13753
13882
|
} else if (typeof entry === "function") {
|
|
13754
13883
|
result = entry(data);
|
|
13755
|
-
} else if (typeof entry === "object") {
|
|
13756
|
-
if (typeof entry.transform === "function") result = entry.transform(data);
|
|
13757
|
-
else if (typeof entry.transformer === "function") result = entry.transformer(data);
|
|
13758
13884
|
} else {
|
|
13759
13885
|
result = data;
|
|
13760
13886
|
}
|
|
@@ -13767,9 +13893,11 @@ class S3dbReplicator extends base_replicator_class_default {
|
|
|
13767
13893
|
const entry = this.resourcesMap[normResource];
|
|
13768
13894
|
if (!entry) return resource;
|
|
13769
13895
|
if (Array.isArray(entry)) {
|
|
13770
|
-
|
|
13771
|
-
|
|
13772
|
-
|
|
13896
|
+
for (const item of entry) {
|
|
13897
|
+
if (typeof item === "string") return item;
|
|
13898
|
+
if (typeof item === "object" && item.resource) return item.resource;
|
|
13899
|
+
}
|
|
13900
|
+
return resource;
|
|
13773
13901
|
}
|
|
13774
13902
|
if (typeof entry === "string") return entry;
|
|
13775
13903
|
if (typeof entry === "function") return resource;
|
|
@@ -13777,8 +13905,7 @@ class S3dbReplicator extends base_replicator_class_default {
|
|
|
13777
13905
|
return resource;
|
|
13778
13906
|
}
|
|
13779
13907
|
_getDestResourceObj(resource) {
|
|
13780
|
-
|
|
13781
|
-
const available = Object.keys(this.client.resources);
|
|
13908
|
+
const available = Object.keys(this.client.resources || {});
|
|
13782
13909
|
const norm = normalizeResourceName$1(resource);
|
|
13783
13910
|
const found = available.find((r) => normalizeResourceName$1(r) === norm);
|
|
13784
13911
|
if (!found) {
|
|
@@ -13800,8 +13927,17 @@ class S3dbReplicator extends base_replicator_class_default {
|
|
|
13800
13927
|
data: record.data,
|
|
13801
13928
|
beforeData: record.beforeData
|
|
13802
13929
|
}));
|
|
13803
|
-
if (ok)
|
|
13804
|
-
|
|
13930
|
+
if (ok) {
|
|
13931
|
+
results.push(result);
|
|
13932
|
+
} else {
|
|
13933
|
+
if (this.config.verbose) {
|
|
13934
|
+
console.warn(`[S3dbReplicator] Batch replication failed for record ${record.id}: ${err.message}`);
|
|
13935
|
+
}
|
|
13936
|
+
errors.push({ id: record.id, error: err.message });
|
|
13937
|
+
}
|
|
13938
|
+
}
|
|
13939
|
+
if (errors.length > 0) {
|
|
13940
|
+
console.warn(`[S3dbReplicator] Batch replication completed with ${errors.length} error(s) for ${resourceName}:`, errors);
|
|
13805
13941
|
}
|
|
13806
13942
|
this.emit("batch_replicated", {
|
|
13807
13943
|
replicator: this.name,
|
|
@@ -13819,18 +13955,20 @@ class S3dbReplicator extends base_replicator_class_default {
|
|
|
13819
13955
|
}
|
|
13820
13956
|
async testConnection() {
|
|
13821
13957
|
const [ok, err] = await try_fn_default(async () => {
|
|
13822
|
-
if (!this.targetDatabase)
|
|
13823
|
-
|
|
13958
|
+
if (!this.targetDatabase) throw new Error("No target database configured");
|
|
13959
|
+
if (typeof this.targetDatabase.connect === "function") {
|
|
13960
|
+
await this.targetDatabase.connect();
|
|
13824
13961
|
}
|
|
13825
|
-
await this.targetDatabase.listResources();
|
|
13826
13962
|
return true;
|
|
13827
13963
|
});
|
|
13828
|
-
if (ok)
|
|
13829
|
-
|
|
13830
|
-
|
|
13831
|
-
|
|
13832
|
-
|
|
13833
|
-
|
|
13964
|
+
if (!ok) {
|
|
13965
|
+
if (this.config.verbose) {
|
|
13966
|
+
console.warn(`[S3dbReplicator] Connection test failed: ${err.message}`);
|
|
13967
|
+
}
|
|
13968
|
+
this.emit("connection_error", { replicator: this.name, error: err.message });
|
|
13969
|
+
return false;
|
|
13970
|
+
}
|
|
13971
|
+
return true;
|
|
13834
13972
|
}
|
|
13835
13973
|
async getStatus() {
|
|
13836
13974
|
const baseStatus = await super.getStatus();
|
|
@@ -13862,7 +14000,7 @@ class S3dbReplicator extends base_replicator_class_default {
|
|
|
13862
14000
|
} else {
|
|
13863
14001
|
return true;
|
|
13864
14002
|
}
|
|
13865
|
-
} else if (typeof item === "string"
|
|
14003
|
+
} else if (typeof item === "string") {
|
|
13866
14004
|
return true;
|
|
13867
14005
|
}
|
|
13868
14006
|
}
|
|
@@ -13989,6 +14127,9 @@ class SqsReplicator extends base_replicator_class_default {
|
|
|
13989
14127
|
if (!this.sqsClient) {
|
|
13990
14128
|
const [ok, err, sdk] = await try_fn_default(() => import('@aws-sdk/client-sqs'));
|
|
13991
14129
|
if (!ok) {
|
|
14130
|
+
if (this.config.verbose) {
|
|
14131
|
+
console.warn(`[SqsReplicator] Failed to import SQS SDK: ${err.message}`);
|
|
14132
|
+
}
|
|
13992
14133
|
this.emit("initialization_error", {
|
|
13993
14134
|
replicator: this.name,
|
|
13994
14135
|
error: err.message
|
|
@@ -14040,6 +14181,9 @@ class SqsReplicator extends base_replicator_class_default {
|
|
|
14040
14181
|
return { success: true, results };
|
|
14041
14182
|
});
|
|
14042
14183
|
if (ok) return result;
|
|
14184
|
+
if (this.config.verbose) {
|
|
14185
|
+
console.warn(`[SqsReplicator] Replication failed for ${resource}: ${err.message}`);
|
|
14186
|
+
}
|
|
14043
14187
|
this.emit("replicator_error", {
|
|
14044
14188
|
replicator: this.name,
|
|
14045
14189
|
resource,
|
|
@@ -14092,6 +14236,9 @@ class SqsReplicator extends base_replicator_class_default {
|
|
|
14092
14236
|
}
|
|
14093
14237
|
}
|
|
14094
14238
|
}
|
|
14239
|
+
if (errors.length > 0) {
|
|
14240
|
+
console.warn(`[SqsReplicator] Batch replication completed with ${errors.length} error(s) for ${resource}:`, errors);
|
|
14241
|
+
}
|
|
14095
14242
|
this.emit("batch_replicated", {
|
|
14096
14243
|
replicator: this.name,
|
|
14097
14244
|
resource,
|
|
@@ -14112,6 +14259,9 @@ class SqsReplicator extends base_replicator_class_default {
|
|
|
14112
14259
|
});
|
|
14113
14260
|
if (ok) return result;
|
|
14114
14261
|
const errorMessage = err?.message || err || "Unknown error";
|
|
14262
|
+
if (this.config.verbose) {
|
|
14263
|
+
console.warn(`[SqsReplicator] Batch replication failed for ${resource}: ${errorMessage}`);
|
|
14264
|
+
}
|
|
14115
14265
|
this.emit("batch_replicator_error", {
|
|
14116
14266
|
replicator: this.name,
|
|
14117
14267
|
resource,
|
|
@@ -14133,6 +14283,9 @@ class SqsReplicator extends base_replicator_class_default {
|
|
|
14133
14283
|
return true;
|
|
14134
14284
|
});
|
|
14135
14285
|
if (ok) return true;
|
|
14286
|
+
if (this.config.verbose) {
|
|
14287
|
+
console.warn(`[SqsReplicator] Connection test failed: ${err.message}`);
|
|
14288
|
+
}
|
|
14136
14289
|
this.emit("connection_error", {
|
|
14137
14290
|
replicator: this.name,
|
|
14138
14291
|
error: err.message
|
|
@@ -14229,25 +14382,37 @@ class ReplicatorPlugin extends plugin_class_default {
|
|
|
14229
14382
|
return;
|
|
14230
14383
|
}
|
|
14231
14384
|
resource.on("insert", async (data) => {
|
|
14232
|
-
|
|
14385
|
+
const [ok, error] = await try_fn_default(async () => {
|
|
14233
14386
|
const completeData = { ...data, createdAt: (/* @__PURE__ */ new Date()).toISOString() };
|
|
14234
14387
|
await plugin.processReplicatorEvent("insert", resource.name, completeData.id, completeData);
|
|
14235
|
-
}
|
|
14388
|
+
});
|
|
14389
|
+
if (!ok) {
|
|
14390
|
+
if (this.config.verbose) {
|
|
14391
|
+
console.warn(`[ReplicatorPlugin] Insert event failed for resource ${resource.name}: ${error.message}`);
|
|
14392
|
+
}
|
|
14236
14393
|
this.emit("error", { operation: "insert", error: error.message, resource: resource.name });
|
|
14237
14394
|
}
|
|
14238
14395
|
});
|
|
14239
14396
|
resource.on("update", async (data, beforeData) => {
|
|
14240
|
-
|
|
14397
|
+
const [ok, error] = await try_fn_default(async () => {
|
|
14241
14398
|
const completeData = { ...data, updatedAt: (/* @__PURE__ */ new Date()).toISOString() };
|
|
14242
14399
|
await plugin.processReplicatorEvent("update", resource.name, completeData.id, completeData, beforeData);
|
|
14243
|
-
}
|
|
14400
|
+
});
|
|
14401
|
+
if (!ok) {
|
|
14402
|
+
if (this.config.verbose) {
|
|
14403
|
+
console.warn(`[ReplicatorPlugin] Update event failed for resource ${resource.name}: ${error.message}`);
|
|
14404
|
+
}
|
|
14244
14405
|
this.emit("error", { operation: "update", error: error.message, resource: resource.name });
|
|
14245
14406
|
}
|
|
14246
14407
|
});
|
|
14247
14408
|
resource.on("delete", async (data) => {
|
|
14248
|
-
|
|
14409
|
+
const [ok, error] = await try_fn_default(async () => {
|
|
14249
14410
|
await plugin.processReplicatorEvent("delete", resource.name, data.id, data);
|
|
14250
|
-
}
|
|
14411
|
+
});
|
|
14412
|
+
if (!ok) {
|
|
14413
|
+
if (this.config.verbose) {
|
|
14414
|
+
console.warn(`[ReplicatorPlugin] Delete event failed for resource ${resource.name}: ${error.message}`);
|
|
14415
|
+
}
|
|
14251
14416
|
this.emit("error", { operation: "delete", error: error.message, resource: resource.name });
|
|
14252
14417
|
}
|
|
14253
14418
|
});
|
|
@@ -14263,13 +14428,17 @@ class ReplicatorPlugin extends plugin_class_default {
|
|
|
14263
14428
|
}
|
|
14264
14429
|
async setup(database) {
|
|
14265
14430
|
this.database = database;
|
|
14266
|
-
|
|
14431
|
+
const [initOk, initError] = await try_fn_default(async () => {
|
|
14267
14432
|
await this.initializeReplicators(database);
|
|
14268
|
-
}
|
|
14269
|
-
|
|
14270
|
-
|
|
14433
|
+
});
|
|
14434
|
+
if (!initOk) {
|
|
14435
|
+
if (this.config.verbose) {
|
|
14436
|
+
console.warn(`[ReplicatorPlugin] Replicator initialization failed: ${initError.message}`);
|
|
14437
|
+
}
|
|
14438
|
+
this.emit("error", { operation: "setup", error: initError.message });
|
|
14439
|
+
throw initError;
|
|
14271
14440
|
}
|
|
14272
|
-
|
|
14441
|
+
const [logOk, logError] = await try_fn_default(async () => {
|
|
14273
14442
|
if (this.config.replicatorLogResource) {
|
|
14274
14443
|
const logRes = await database.createResource({
|
|
14275
14444
|
name: this.config.replicatorLogResource,
|
|
@@ -14286,7 +14455,15 @@ class ReplicatorPlugin extends plugin_class_default {
|
|
|
14286
14455
|
}
|
|
14287
14456
|
});
|
|
14288
14457
|
}
|
|
14289
|
-
}
|
|
14458
|
+
});
|
|
14459
|
+
if (!logOk) {
|
|
14460
|
+
if (this.config.verbose) {
|
|
14461
|
+
console.warn(`[ReplicatorPlugin] Failed to create log resource ${this.config.replicatorLogResource}: ${logError.message}`);
|
|
14462
|
+
}
|
|
14463
|
+
this.emit("replicator_log_resource_creation_error", {
|
|
14464
|
+
resourceName: this.config.replicatorLogResource,
|
|
14465
|
+
error: logError.message
|
|
14466
|
+
});
|
|
14290
14467
|
}
|
|
14291
14468
|
await this.uploadMetadataFile(database);
|
|
14292
14469
|
const originalCreateResource = database.createResource.bind(database);
|
|
@@ -14329,21 +14506,28 @@ class ReplicatorPlugin extends plugin_class_default {
|
|
|
14329
14506
|
async retryWithBackoff(operation, maxRetries = 3) {
|
|
14330
14507
|
let lastError;
|
|
14331
14508
|
for (let attempt = 1; attempt <= maxRetries; attempt++) {
|
|
14332
|
-
|
|
14333
|
-
|
|
14334
|
-
|
|
14509
|
+
const [ok, error] = await try_fn_default(operation);
|
|
14510
|
+
if (ok) {
|
|
14511
|
+
return ok;
|
|
14512
|
+
} else {
|
|
14335
14513
|
lastError = error;
|
|
14514
|
+
if (this.config.verbose) {
|
|
14515
|
+
console.warn(`[ReplicatorPlugin] Retry attempt ${attempt}/${maxRetries} failed: ${error.message}`);
|
|
14516
|
+
}
|
|
14336
14517
|
if (attempt === maxRetries) {
|
|
14337
14518
|
throw error;
|
|
14338
14519
|
}
|
|
14339
14520
|
const delay = Math.pow(2, attempt - 1) * 1e3;
|
|
14521
|
+
if (this.config.verbose) {
|
|
14522
|
+
console.warn(`[ReplicatorPlugin] Waiting ${delay}ms before retry...`);
|
|
14523
|
+
}
|
|
14340
14524
|
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
14341
14525
|
}
|
|
14342
14526
|
}
|
|
14343
14527
|
throw lastError;
|
|
14344
14528
|
}
|
|
14345
14529
|
async logError(replicator, resourceName, operation, recordId, data, error) {
|
|
14346
|
-
|
|
14530
|
+
const [ok, logError] = await try_fn_default(async () => {
|
|
14347
14531
|
const logResourceName = this.config.replicatorLogResource;
|
|
14348
14532
|
if (this.database && this.database.resources && this.database.resources[logResourceName]) {
|
|
14349
14533
|
const logResource = this.database.resources[logResourceName];
|
|
@@ -14358,7 +14542,19 @@ class ReplicatorPlugin extends plugin_class_default {
|
|
|
14358
14542
|
status: "error"
|
|
14359
14543
|
});
|
|
14360
14544
|
}
|
|
14361
|
-
}
|
|
14545
|
+
});
|
|
14546
|
+
if (!ok) {
|
|
14547
|
+
if (this.config.verbose) {
|
|
14548
|
+
console.warn(`[ReplicatorPlugin] Failed to log error for ${resourceName}: ${logError.message}`);
|
|
14549
|
+
}
|
|
14550
|
+
this.emit("replicator_log_error", {
|
|
14551
|
+
replicator: replicator.name || replicator.id,
|
|
14552
|
+
resourceName,
|
|
14553
|
+
operation,
|
|
14554
|
+
recordId,
|
|
14555
|
+
originalError: error.message,
|
|
14556
|
+
logError: logError.message
|
|
14557
|
+
});
|
|
14362
14558
|
}
|
|
14363
14559
|
}
|
|
14364
14560
|
async processReplicatorEvent(operation, resourceName, recordId, data, beforeData = null) {
|
|
@@ -14371,8 +14567,8 @@ class ReplicatorPlugin extends plugin_class_default {
|
|
|
14371
14567
|
return;
|
|
14372
14568
|
}
|
|
14373
14569
|
const promises = applicableReplicators.map(async (replicator) => {
|
|
14374
|
-
|
|
14375
|
-
const
|
|
14570
|
+
const [ok, error, result] = await try_fn_default(async () => {
|
|
14571
|
+
const result2 = await this.retryWithBackoff(
|
|
14376
14572
|
() => replicator.replicate(resourceName, operation, data, recordId, beforeData),
|
|
14377
14573
|
this.config.maxRetries
|
|
14378
14574
|
);
|
|
@@ -14381,11 +14577,17 @@ class ReplicatorPlugin extends plugin_class_default {
|
|
|
14381
14577
|
resourceName,
|
|
14382
14578
|
operation,
|
|
14383
14579
|
recordId,
|
|
14384
|
-
result,
|
|
14580
|
+
result: result2,
|
|
14385
14581
|
success: true
|
|
14386
14582
|
});
|
|
14583
|
+
return result2;
|
|
14584
|
+
});
|
|
14585
|
+
if (ok) {
|
|
14387
14586
|
return result;
|
|
14388
|
-
}
|
|
14587
|
+
} else {
|
|
14588
|
+
if (this.config.verbose) {
|
|
14589
|
+
console.warn(`[ReplicatorPlugin] Replication failed for ${replicator.name || replicator.id} on ${resourceName}: ${error.message}`);
|
|
14590
|
+
}
|
|
14389
14591
|
this.emit("replicator_error", {
|
|
14390
14592
|
replicator: replicator.name || replicator.id,
|
|
14391
14593
|
resourceName,
|
|
@@ -14410,11 +14612,14 @@ class ReplicatorPlugin extends plugin_class_default {
|
|
|
14410
14612
|
return;
|
|
14411
14613
|
}
|
|
14412
14614
|
const promises = applicableReplicators.map(async (replicator) => {
|
|
14413
|
-
|
|
14615
|
+
const [wrapperOk, wrapperError] = await try_fn_default(async () => {
|
|
14414
14616
|
const [ok, err, result] = await try_fn_default(
|
|
14415
14617
|
() => replicator.replicate(item.resourceName, item.operation, item.data, item.recordId, item.beforeData)
|
|
14416
14618
|
);
|
|
14417
14619
|
if (!ok) {
|
|
14620
|
+
if (this.config.verbose) {
|
|
14621
|
+
console.warn(`[ReplicatorPlugin] Replicator item processing failed for ${replicator.name || replicator.id} on ${item.resourceName}: ${err.message}`);
|
|
14622
|
+
}
|
|
14418
14623
|
this.emit("replicator_error", {
|
|
14419
14624
|
replicator: replicator.name || replicator.id,
|
|
14420
14625
|
resourceName: item.resourceName,
|
|
@@ -14436,18 +14641,24 @@ class ReplicatorPlugin extends plugin_class_default {
|
|
|
14436
14641
|
success: true
|
|
14437
14642
|
});
|
|
14438
14643
|
return { success: true, result };
|
|
14439
|
-
}
|
|
14644
|
+
});
|
|
14645
|
+
if (wrapperOk) {
|
|
14646
|
+
return wrapperOk;
|
|
14647
|
+
} else {
|
|
14648
|
+
if (this.config.verbose) {
|
|
14649
|
+
console.warn(`[ReplicatorPlugin] Wrapper processing failed for ${replicator.name || replicator.id} on ${item.resourceName}: ${wrapperError.message}`);
|
|
14650
|
+
}
|
|
14440
14651
|
this.emit("replicator_error", {
|
|
14441
14652
|
replicator: replicator.name || replicator.id,
|
|
14442
14653
|
resourceName: item.resourceName,
|
|
14443
14654
|
operation: item.operation,
|
|
14444
14655
|
recordId: item.recordId,
|
|
14445
|
-
error:
|
|
14656
|
+
error: wrapperError.message
|
|
14446
14657
|
});
|
|
14447
14658
|
if (this.config.logErrors && this.database) {
|
|
14448
|
-
await this.logError(replicator, item.resourceName, item.operation, item.recordId, item.data,
|
|
14659
|
+
await this.logError(replicator, item.resourceName, item.operation, item.recordId, item.data, wrapperError);
|
|
14449
14660
|
}
|
|
14450
|
-
return { success: false, error:
|
|
14661
|
+
return { success: false, error: wrapperError.message };
|
|
14451
14662
|
}
|
|
14452
14663
|
});
|
|
14453
14664
|
return Promise.allSettled(promises);
|
|
@@ -14469,9 +14680,13 @@ class ReplicatorPlugin extends plugin_class_default {
|
|
|
14469
14680
|
timestamp: typeof item.timestamp === "number" ? item.timestamp : Date.now(),
|
|
14470
14681
|
createdAt: item.createdAt || (/* @__PURE__ */ new Date()).toISOString().slice(0, 10)
|
|
14471
14682
|
};
|
|
14472
|
-
|
|
14683
|
+
const [ok, err] = await try_fn_default(async () => {
|
|
14473
14684
|
await logRes.insert(logItem);
|
|
14474
|
-
}
|
|
14685
|
+
});
|
|
14686
|
+
if (!ok) {
|
|
14687
|
+
if (this.config.verbose) {
|
|
14688
|
+
console.warn(`[ReplicatorPlugin] Failed to log replicator item: ${err.message}`);
|
|
14689
|
+
}
|
|
14475
14690
|
this.emit("replicator.log.failed", { error: err, item });
|
|
14476
14691
|
}
|
|
14477
14692
|
}
|
|
@@ -14577,14 +14792,23 @@ class ReplicatorPlugin extends plugin_class_default {
|
|
|
14577
14792
|
this.emit("replicator.sync.completed", { replicatorId, stats: this.stats });
|
|
14578
14793
|
}
|
|
14579
14794
|
async cleanup() {
|
|
14580
|
-
|
|
14795
|
+
const [ok, error] = await try_fn_default(async () => {
|
|
14581
14796
|
if (this.replicators && this.replicators.length > 0) {
|
|
14582
14797
|
const cleanupPromises = this.replicators.map(async (replicator) => {
|
|
14583
|
-
|
|
14798
|
+
const [replicatorOk, replicatorError] = await try_fn_default(async () => {
|
|
14584
14799
|
if (replicator && typeof replicator.cleanup === "function") {
|
|
14585
14800
|
await replicator.cleanup();
|
|
14586
14801
|
}
|
|
14587
|
-
}
|
|
14802
|
+
});
|
|
14803
|
+
if (!replicatorOk) {
|
|
14804
|
+
if (this.config.verbose) {
|
|
14805
|
+
console.warn(`[ReplicatorPlugin] Failed to cleanup replicator ${replicator.name || replicator.id}: ${replicatorError.message}`);
|
|
14806
|
+
}
|
|
14807
|
+
this.emit("replicator_cleanup_error", {
|
|
14808
|
+
replicator: replicator.name || replicator.id || "unknown",
|
|
14809
|
+
driver: replicator.driver || "unknown",
|
|
14810
|
+
error: replicatorError.message
|
|
14811
|
+
});
|
|
14588
14812
|
}
|
|
14589
14813
|
});
|
|
14590
14814
|
await Promise.allSettled(cleanupPromises);
|
|
@@ -14593,7 +14817,14 @@ class ReplicatorPlugin extends plugin_class_default {
|
|
|
14593
14817
|
this.database = null;
|
|
14594
14818
|
this.eventListenersInstalled.clear();
|
|
14595
14819
|
this.removeAllListeners();
|
|
14596
|
-
}
|
|
14820
|
+
});
|
|
14821
|
+
if (!ok) {
|
|
14822
|
+
if (this.config.verbose) {
|
|
14823
|
+
console.warn(`[ReplicatorPlugin] Failed to cleanup plugin: ${error.message}`);
|
|
14824
|
+
}
|
|
14825
|
+
this.emit("replicator_plugin_cleanup_error", {
|
|
14826
|
+
error: error.message
|
|
14827
|
+
});
|
|
14597
14828
|
}
|
|
14598
14829
|
}
|
|
14599
14830
|
}
|