s3db.js 7.3.5 → 7.3.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/PLUGINS.md +1285 -157
- package/dist/s3db.cjs.js +296 -83
- package/dist/s3db.cjs.min.js +1 -1
- package/dist/s3db.es.js +296 -83
- package/dist/s3db.es.min.js +1 -1
- package/dist/s3db.iife.js +296 -83
- package/dist/s3db.iife.min.js +1 -1
- package/mcp/server.js +1410 -0
- package/package.json +7 -1
- package/src/plugins/cache/filesystem-cache.class.js +9 -0
- package/src/plugins/replicator.plugin.js +130 -46
- package/src/plugins/replicators/bigquery-replicator.class.js +31 -5
- package/src/plugins/replicators/postgres-replicator.class.js +17 -2
- package/src/plugins/replicators/s3db-replicator.class.js +172 -68
- package/src/plugins/replicators/sqs-replicator.class.js +13 -1
package/dist/s3db.cjs.js
CHANGED
|
@@ -7083,6 +7083,12 @@ class FilesystemCache extends Cache {
|
|
|
7083
7083
|
}
|
|
7084
7084
|
async _clear(prefix) {
|
|
7085
7085
|
try {
|
|
7086
|
+
if (!await this._fileExists(this.directory)) {
|
|
7087
|
+
if (this.enableStats) {
|
|
7088
|
+
this.stats.clears++;
|
|
7089
|
+
}
|
|
7090
|
+
return true;
|
|
7091
|
+
}
|
|
7086
7092
|
const files = await promises.readdir(this.directory);
|
|
7087
7093
|
const cacheFiles = files.filter((file) => {
|
|
7088
7094
|
if (!file.startsWith(this.prefix)) return false;
|
|
@@ -8954,6 +8960,9 @@ class BigqueryReplicator extends base_replicator_class_default {
|
|
|
8954
8960
|
await super.initialize(database);
|
|
8955
8961
|
const [ok, err, sdk] = await try_fn_default(() => import('@google-cloud/bigquery'));
|
|
8956
8962
|
if (!ok) {
|
|
8963
|
+
if (this.config.verbose) {
|
|
8964
|
+
console.warn(`[BigqueryReplicator] Failed to import BigQuery SDK: ${err.message}`);
|
|
8965
|
+
}
|
|
8957
8966
|
this.emit("initialization_error", { replicator: this.name, error: err.message });
|
|
8958
8967
|
throw err;
|
|
8959
8968
|
}
|
|
@@ -9023,19 +9032,28 @@ class BigqueryReplicator extends base_replicator_class_default {
|
|
|
9023
9032
|
const maxRetries = 2;
|
|
9024
9033
|
let lastError = null;
|
|
9025
9034
|
for (let attempt = 1; attempt <= maxRetries; attempt++) {
|
|
9026
|
-
|
|
9035
|
+
const [ok2, error] = await try_fn_default(async () => {
|
|
9027
9036
|
const [updateJob] = await this.bigqueryClient.createQueryJob({
|
|
9028
9037
|
query,
|
|
9029
9038
|
params,
|
|
9030
9039
|
location: this.location
|
|
9031
9040
|
});
|
|
9032
9041
|
await updateJob.getQueryResults();
|
|
9033
|
-
|
|
9042
|
+
return [updateJob];
|
|
9043
|
+
});
|
|
9044
|
+
if (ok2) {
|
|
9045
|
+
job = ok2;
|
|
9034
9046
|
break;
|
|
9035
|
-
}
|
|
9047
|
+
} else {
|
|
9036
9048
|
lastError = error;
|
|
9049
|
+
if (this.config.verbose) {
|
|
9050
|
+
console.warn(`[BigqueryReplicator] Update attempt ${attempt} failed: ${error.message}`);
|
|
9051
|
+
}
|
|
9037
9052
|
if (error?.message?.includes("streaming buffer") && attempt < maxRetries) {
|
|
9038
9053
|
const delaySeconds = 30;
|
|
9054
|
+
if (this.config.verbose) {
|
|
9055
|
+
console.warn(`[BigqueryReplicator] Retrying in ${delaySeconds} seconds due to streaming buffer issue`);
|
|
9056
|
+
}
|
|
9039
9057
|
await new Promise((resolve) => setTimeout(resolve, delaySeconds * 1e3));
|
|
9040
9058
|
continue;
|
|
9041
9059
|
}
|
|
@@ -9102,6 +9120,9 @@ class BigqueryReplicator extends base_replicator_class_default {
|
|
|
9102
9120
|
};
|
|
9103
9121
|
});
|
|
9104
9122
|
if (ok) return result;
|
|
9123
|
+
if (this.config.verbose) {
|
|
9124
|
+
console.warn(`[BigqueryReplicator] Replication failed for ${resourceName}: ${err.message}`);
|
|
9125
|
+
}
|
|
9105
9126
|
this.emit("replicator_error", {
|
|
9106
9127
|
replicator: this.name,
|
|
9107
9128
|
resourceName,
|
|
@@ -9122,8 +9143,14 @@ class BigqueryReplicator extends base_replicator_class_default {
|
|
|
9122
9143
|
record.id,
|
|
9123
9144
|
record.beforeData
|
|
9124
9145
|
));
|
|
9125
|
-
if (ok)
|
|
9126
|
-
|
|
9146
|
+
if (ok) {
|
|
9147
|
+
results.push(res);
|
|
9148
|
+
} else {
|
|
9149
|
+
if (this.config.verbose) {
|
|
9150
|
+
console.warn(`[BigqueryReplicator] Batch replication failed for record ${record.id}: ${err.message}`);
|
|
9151
|
+
}
|
|
9152
|
+
errors.push({ id: record.id, error: err.message });
|
|
9153
|
+
}
|
|
9127
9154
|
}
|
|
9128
9155
|
return {
|
|
9129
9156
|
success: errors.length === 0,
|
|
@@ -9139,6 +9166,9 @@ class BigqueryReplicator extends base_replicator_class_default {
|
|
|
9139
9166
|
return true;
|
|
9140
9167
|
});
|
|
9141
9168
|
if (ok) return true;
|
|
9169
|
+
if (this.config.verbose) {
|
|
9170
|
+
console.warn(`[BigqueryReplicator] Connection test failed: ${err.message}`);
|
|
9171
|
+
}
|
|
9142
9172
|
this.emit("connection_error", { replicator: this.name, error: err.message });
|
|
9143
9173
|
return false;
|
|
9144
9174
|
}
|
|
@@ -9226,6 +9256,9 @@ class PostgresReplicator extends base_replicator_class_default {
|
|
|
9226
9256
|
await super.initialize(database);
|
|
9227
9257
|
const [ok, err, sdk] = await try_fn_default(() => import('pg'));
|
|
9228
9258
|
if (!ok) {
|
|
9259
|
+
if (this.config.verbose) {
|
|
9260
|
+
console.warn(`[PostgresReplicator] Failed to import pg SDK: ${err.message}`);
|
|
9261
|
+
}
|
|
9229
9262
|
this.emit("initialization_error", {
|
|
9230
9263
|
replicator: this.name,
|
|
9231
9264
|
error: err.message
|
|
@@ -9367,6 +9400,9 @@ class PostgresReplicator extends base_replicator_class_default {
|
|
|
9367
9400
|
};
|
|
9368
9401
|
});
|
|
9369
9402
|
if (ok) return result;
|
|
9403
|
+
if (this.config.verbose) {
|
|
9404
|
+
console.warn(`[PostgresReplicator] Replication failed for ${resourceName}: ${err.message}`);
|
|
9405
|
+
}
|
|
9370
9406
|
this.emit("replicator_error", {
|
|
9371
9407
|
replicator: this.name,
|
|
9372
9408
|
resourceName,
|
|
@@ -9387,8 +9423,14 @@ class PostgresReplicator extends base_replicator_class_default {
|
|
|
9387
9423
|
record.id,
|
|
9388
9424
|
record.beforeData
|
|
9389
9425
|
));
|
|
9390
|
-
if (ok)
|
|
9391
|
-
|
|
9426
|
+
if (ok) {
|
|
9427
|
+
results.push(res);
|
|
9428
|
+
} else {
|
|
9429
|
+
if (this.config.verbose) {
|
|
9430
|
+
console.warn(`[PostgresReplicator] Batch replication failed for record ${record.id}: ${err.message}`);
|
|
9431
|
+
}
|
|
9432
|
+
errors.push({ id: record.id, error: err.message });
|
|
9433
|
+
}
|
|
9392
9434
|
}
|
|
9393
9435
|
return {
|
|
9394
9436
|
success: errors.length === 0,
|
|
@@ -9403,6 +9445,9 @@ class PostgresReplicator extends base_replicator_class_default {
|
|
|
9403
9445
|
return true;
|
|
9404
9446
|
});
|
|
9405
9447
|
if (ok) return true;
|
|
9448
|
+
if (this.config.verbose) {
|
|
9449
|
+
console.warn(`[PostgresReplicator] Connection test failed: ${err.message}`);
|
|
9450
|
+
}
|
|
9406
9451
|
this.emit("connection_error", { replicator: this.name, error: err.message });
|
|
9407
9452
|
return false;
|
|
9408
9453
|
}
|
|
@@ -13137,7 +13182,7 @@ class Database extends EventEmitter {
|
|
|
13137
13182
|
super();
|
|
13138
13183
|
this.version = "1";
|
|
13139
13184
|
this.s3dbVersion = (() => {
|
|
13140
|
-
const [ok, err, version] = try_fn_default(() => true ? "7.3.
|
|
13185
|
+
const [ok, err, version] = try_fn_default(() => true ? "7.3.6" : "latest");
|
|
13141
13186
|
return ok ? version : "latest";
|
|
13142
13187
|
})();
|
|
13143
13188
|
this.resources = {};
|
|
@@ -13641,9 +13686,8 @@ class S3dbReplicator extends base_replicator_class_default {
|
|
|
13641
13686
|
const map = {};
|
|
13642
13687
|
for (const res of resources) {
|
|
13643
13688
|
if (typeof res === "string") map[normalizeResourceName$1(res)] = res;
|
|
13644
|
-
else if (Array.isArray(res) && typeof res[0] === "string") map[normalizeResourceName$1(res[0])] = res;
|
|
13645
13689
|
else if (typeof res === "object" && res.resource) {
|
|
13646
|
-
map[normalizeResourceName$1(res.resource)] =
|
|
13690
|
+
map[normalizeResourceName$1(res.resource)] = res;
|
|
13647
13691
|
}
|
|
13648
13692
|
}
|
|
13649
13693
|
return map;
|
|
@@ -13656,15 +13700,14 @@ class S3dbReplicator extends base_replicator_class_default {
|
|
|
13656
13700
|
else if (Array.isArray(dest)) {
|
|
13657
13701
|
map[normSrc] = dest.map((item) => {
|
|
13658
13702
|
if (typeof item === "string") return item;
|
|
13659
|
-
if (typeof item === "function") return item;
|
|
13660
13703
|
if (typeof item === "object" && item.resource) {
|
|
13661
|
-
return
|
|
13704
|
+
return item;
|
|
13662
13705
|
}
|
|
13663
13706
|
return item;
|
|
13664
13707
|
});
|
|
13665
13708
|
} else if (typeof dest === "function") map[normSrc] = dest;
|
|
13666
13709
|
else if (typeof dest === "object" && dest.resource) {
|
|
13667
|
-
map[normSrc] =
|
|
13710
|
+
map[normSrc] = dest;
|
|
13668
13711
|
}
|
|
13669
13712
|
}
|
|
13670
13713
|
return map;
|
|
@@ -13672,10 +13715,6 @@ class S3dbReplicator extends base_replicator_class_default {
|
|
|
13672
13715
|
if (typeof resources === "function") {
|
|
13673
13716
|
return resources;
|
|
13674
13717
|
}
|
|
13675
|
-
if (typeof resources === "string") {
|
|
13676
|
-
const map = { [normalizeResourceName$1(resources)]: resources };
|
|
13677
|
-
return map;
|
|
13678
|
-
}
|
|
13679
13718
|
return {};
|
|
13680
13719
|
}
|
|
13681
13720
|
validateConfig() {
|
|
@@ -13689,8 +13728,8 @@ class S3dbReplicator extends base_replicator_class_default {
|
|
|
13689
13728
|
return { isValid: errors.length === 0, errors };
|
|
13690
13729
|
}
|
|
13691
13730
|
async initialize(database) {
|
|
13692
|
-
|
|
13693
|
-
|
|
13731
|
+
await super.initialize(database);
|
|
13732
|
+
const [ok, err] = await try_fn_default(async () => {
|
|
13694
13733
|
if (this.client) {
|
|
13695
13734
|
this.targetDatabase = this.client;
|
|
13696
13735
|
} else if (this.connectionString) {
|
|
@@ -13709,7 +13748,11 @@ class S3dbReplicator extends base_replicator_class_default {
|
|
|
13709
13748
|
replicator: this.name,
|
|
13710
13749
|
target: this.connectionString || "client-provided"
|
|
13711
13750
|
});
|
|
13712
|
-
}
|
|
13751
|
+
});
|
|
13752
|
+
if (!ok) {
|
|
13753
|
+
if (this.config.verbose) {
|
|
13754
|
+
console.warn(`[S3dbReplicator] Initialization failed: ${err.message}`);
|
|
13755
|
+
}
|
|
13713
13756
|
throw err;
|
|
13714
13757
|
}
|
|
13715
13758
|
}
|
|
@@ -13728,18 +13771,77 @@ class S3dbReplicator extends base_replicator_class_default {
|
|
|
13728
13771
|
id = recordId;
|
|
13729
13772
|
}
|
|
13730
13773
|
const normResource = normalizeResourceName$1(resource);
|
|
13731
|
-
const
|
|
13732
|
-
|
|
13733
|
-
|
|
13774
|
+
const entry = this.resourcesMap[normResource];
|
|
13775
|
+
if (!entry) {
|
|
13776
|
+
throw new Error(`[S3dbReplicator] Resource not configured: ${resource}`);
|
|
13777
|
+
}
|
|
13778
|
+
if (Array.isArray(entry)) {
|
|
13779
|
+
const results = [];
|
|
13780
|
+
for (const destConfig of entry) {
|
|
13781
|
+
const [ok, error, result] = await try_fn_default(async () => {
|
|
13782
|
+
return await this._replicateToSingleDestination(destConfig, normResource, op, payload, id);
|
|
13783
|
+
});
|
|
13784
|
+
if (!ok) {
|
|
13785
|
+
if (this.config && this.config.verbose) {
|
|
13786
|
+
console.warn(`[S3dbReplicator] Failed to replicate to destination ${JSON.stringify(destConfig)}: ${error.message}`);
|
|
13787
|
+
}
|
|
13788
|
+
throw error;
|
|
13789
|
+
}
|
|
13790
|
+
results.push(result);
|
|
13791
|
+
}
|
|
13792
|
+
return results;
|
|
13793
|
+
} else {
|
|
13794
|
+
const [ok, error, result] = await try_fn_default(async () => {
|
|
13795
|
+
return await this._replicateToSingleDestination(entry, normResource, op, payload, id);
|
|
13796
|
+
});
|
|
13797
|
+
if (!ok) {
|
|
13798
|
+
if (this.config && this.config.verbose) {
|
|
13799
|
+
console.warn(`[S3dbReplicator] Failed to replicate to destination ${JSON.stringify(entry)}: ${error.message}`);
|
|
13800
|
+
}
|
|
13801
|
+
throw error;
|
|
13802
|
+
}
|
|
13803
|
+
return result;
|
|
13804
|
+
}
|
|
13805
|
+
}
|
|
13806
|
+
async _replicateToSingleDestination(destConfig, sourceResource, operation, data, recordId) {
|
|
13807
|
+
let destResourceName;
|
|
13808
|
+
if (typeof destConfig === "string") {
|
|
13809
|
+
destResourceName = destConfig;
|
|
13810
|
+
} else if (typeof destConfig === "object" && destConfig.resource) {
|
|
13811
|
+
destResourceName = destConfig.resource;
|
|
13812
|
+
} else {
|
|
13813
|
+
destResourceName = sourceResource;
|
|
13814
|
+
}
|
|
13815
|
+
if (typeof destConfig === "object" && destConfig.actions && Array.isArray(destConfig.actions)) {
|
|
13816
|
+
if (!destConfig.actions.includes(operation)) {
|
|
13817
|
+
return { skipped: true, reason: "action_not_supported", action: operation, destination: destResourceName };
|
|
13818
|
+
}
|
|
13819
|
+
}
|
|
13820
|
+
const destResourceObj = this._getDestResourceObj(destResourceName);
|
|
13821
|
+
let transformedData;
|
|
13822
|
+
if (typeof destConfig === "object" && destConfig.transform && typeof destConfig.transform === "function") {
|
|
13823
|
+
transformedData = destConfig.transform(data);
|
|
13824
|
+
if (transformedData && data && data.id && !transformedData.id) {
|
|
13825
|
+
transformedData.id = data.id;
|
|
13826
|
+
}
|
|
13827
|
+
} else if (typeof destConfig === "object" && destConfig.transformer && typeof destConfig.transformer === "function") {
|
|
13828
|
+
transformedData = destConfig.transformer(data);
|
|
13829
|
+
if (transformedData && data && data.id && !transformedData.id) {
|
|
13830
|
+
transformedData.id = data.id;
|
|
13831
|
+
}
|
|
13832
|
+
} else {
|
|
13833
|
+
transformedData = data;
|
|
13834
|
+
}
|
|
13835
|
+
if (!transformedData && data) transformedData = data;
|
|
13734
13836
|
let result;
|
|
13735
|
-
if (
|
|
13837
|
+
if (operation === "insert") {
|
|
13736
13838
|
result = await destResourceObj.insert(transformedData);
|
|
13737
|
-
} else if (
|
|
13738
|
-
result = await destResourceObj.update(
|
|
13739
|
-
} else if (
|
|
13740
|
-
result = await destResourceObj.delete(
|
|
13839
|
+
} else if (operation === "update") {
|
|
13840
|
+
result = await destResourceObj.update(recordId, transformedData);
|
|
13841
|
+
} else if (operation === "delete") {
|
|
13842
|
+
result = await destResourceObj.delete(recordId);
|
|
13741
13843
|
} else {
|
|
13742
|
-
throw new Error(`Invalid operation: ${
|
|
13844
|
+
throw new Error(`Invalid operation: ${operation}. Supported operations are: insert, update, delete`);
|
|
13743
13845
|
}
|
|
13744
13846
|
return result;
|
|
13745
13847
|
}
|
|
@@ -13748,13 +13850,25 @@ class S3dbReplicator extends base_replicator_class_default {
|
|
|
13748
13850
|
const entry = this.resourcesMap[normResource];
|
|
13749
13851
|
let result;
|
|
13750
13852
|
if (!entry) return data;
|
|
13751
|
-
if (Array.isArray(entry)
|
|
13752
|
-
|
|
13853
|
+
if (Array.isArray(entry)) {
|
|
13854
|
+
for (const item of entry) {
|
|
13855
|
+
if (typeof item === "object" && item.transform && typeof item.transform === "function") {
|
|
13856
|
+
result = item.transform(data);
|
|
13857
|
+
break;
|
|
13858
|
+
} else if (typeof item === "object" && item.transformer && typeof item.transformer === "function") {
|
|
13859
|
+
result = item.transformer(data);
|
|
13860
|
+
break;
|
|
13861
|
+
}
|
|
13862
|
+
}
|
|
13863
|
+
if (!result) result = data;
|
|
13864
|
+
} else if (typeof entry === "object") {
|
|
13865
|
+
if (typeof entry.transform === "function") {
|
|
13866
|
+
result = entry.transform(data);
|
|
13867
|
+
} else if (typeof entry.transformer === "function") {
|
|
13868
|
+
result = entry.transformer(data);
|
|
13869
|
+
}
|
|
13753
13870
|
} else if (typeof entry === "function") {
|
|
13754
13871
|
result = entry(data);
|
|
13755
|
-
} else if (typeof entry === "object") {
|
|
13756
|
-
if (typeof entry.transform === "function") result = entry.transform(data);
|
|
13757
|
-
else if (typeof entry.transformer === "function") result = entry.transformer(data);
|
|
13758
13872
|
} else {
|
|
13759
13873
|
result = data;
|
|
13760
13874
|
}
|
|
@@ -13767,9 +13881,11 @@ class S3dbReplicator extends base_replicator_class_default {
|
|
|
13767
13881
|
const entry = this.resourcesMap[normResource];
|
|
13768
13882
|
if (!entry) return resource;
|
|
13769
13883
|
if (Array.isArray(entry)) {
|
|
13770
|
-
|
|
13771
|
-
|
|
13772
|
-
|
|
13884
|
+
for (const item of entry) {
|
|
13885
|
+
if (typeof item === "string") return item;
|
|
13886
|
+
if (typeof item === "object" && item.resource) return item.resource;
|
|
13887
|
+
}
|
|
13888
|
+
return resource;
|
|
13773
13889
|
}
|
|
13774
13890
|
if (typeof entry === "string") return entry;
|
|
13775
13891
|
if (typeof entry === "function") return resource;
|
|
@@ -13777,8 +13893,7 @@ class S3dbReplicator extends base_replicator_class_default {
|
|
|
13777
13893
|
return resource;
|
|
13778
13894
|
}
|
|
13779
13895
|
_getDestResourceObj(resource) {
|
|
13780
|
-
|
|
13781
|
-
const available = Object.keys(this.client.resources);
|
|
13896
|
+
const available = Object.keys(this.client.resources || {});
|
|
13782
13897
|
const norm = normalizeResourceName$1(resource);
|
|
13783
13898
|
const found = available.find((r) => normalizeResourceName$1(r) === norm);
|
|
13784
13899
|
if (!found) {
|
|
@@ -13800,8 +13915,14 @@ class S3dbReplicator extends base_replicator_class_default {
|
|
|
13800
13915
|
data: record.data,
|
|
13801
13916
|
beforeData: record.beforeData
|
|
13802
13917
|
}));
|
|
13803
|
-
if (ok)
|
|
13804
|
-
|
|
13918
|
+
if (ok) {
|
|
13919
|
+
results.push(result);
|
|
13920
|
+
} else {
|
|
13921
|
+
if (this.config.verbose) {
|
|
13922
|
+
console.warn(`[S3dbReplicator] Batch replication failed for record ${record.id}: ${err.message}`);
|
|
13923
|
+
}
|
|
13924
|
+
errors.push({ id: record.id, error: err.message });
|
|
13925
|
+
}
|
|
13805
13926
|
}
|
|
13806
13927
|
this.emit("batch_replicated", {
|
|
13807
13928
|
replicator: this.name,
|
|
@@ -13819,18 +13940,20 @@ class S3dbReplicator extends base_replicator_class_default {
|
|
|
13819
13940
|
}
|
|
13820
13941
|
async testConnection() {
|
|
13821
13942
|
const [ok, err] = await try_fn_default(async () => {
|
|
13822
|
-
if (!this.targetDatabase)
|
|
13823
|
-
|
|
13943
|
+
if (!this.targetDatabase) throw new Error("No target database configured");
|
|
13944
|
+
if (typeof this.targetDatabase.connect === "function") {
|
|
13945
|
+
await this.targetDatabase.connect();
|
|
13824
13946
|
}
|
|
13825
|
-
await this.targetDatabase.listResources();
|
|
13826
13947
|
return true;
|
|
13827
13948
|
});
|
|
13828
|
-
if (ok)
|
|
13829
|
-
|
|
13830
|
-
|
|
13831
|
-
|
|
13832
|
-
|
|
13833
|
-
|
|
13949
|
+
if (!ok) {
|
|
13950
|
+
if (this.config.verbose) {
|
|
13951
|
+
console.warn(`[S3dbReplicator] Connection test failed: ${err.message}`);
|
|
13952
|
+
}
|
|
13953
|
+
this.emit("connection_error", { replicator: this.name, error: err.message });
|
|
13954
|
+
return false;
|
|
13955
|
+
}
|
|
13956
|
+
return true;
|
|
13834
13957
|
}
|
|
13835
13958
|
async getStatus() {
|
|
13836
13959
|
const baseStatus = await super.getStatus();
|
|
@@ -13862,7 +13985,7 @@ class S3dbReplicator extends base_replicator_class_default {
|
|
|
13862
13985
|
} else {
|
|
13863
13986
|
return true;
|
|
13864
13987
|
}
|
|
13865
|
-
} else if (typeof item === "string"
|
|
13988
|
+
} else if (typeof item === "string") {
|
|
13866
13989
|
return true;
|
|
13867
13990
|
}
|
|
13868
13991
|
}
|
|
@@ -13989,6 +14112,9 @@ class SqsReplicator extends base_replicator_class_default {
|
|
|
13989
14112
|
if (!this.sqsClient) {
|
|
13990
14113
|
const [ok, err, sdk] = await try_fn_default(() => import('@aws-sdk/client-sqs'));
|
|
13991
14114
|
if (!ok) {
|
|
14115
|
+
if (this.config.verbose) {
|
|
14116
|
+
console.warn(`[SqsReplicator] Failed to import SQS SDK: ${err.message}`);
|
|
14117
|
+
}
|
|
13992
14118
|
this.emit("initialization_error", {
|
|
13993
14119
|
replicator: this.name,
|
|
13994
14120
|
error: err.message
|
|
@@ -14040,6 +14166,9 @@ class SqsReplicator extends base_replicator_class_default {
|
|
|
14040
14166
|
return { success: true, results };
|
|
14041
14167
|
});
|
|
14042
14168
|
if (ok) return result;
|
|
14169
|
+
if (this.config.verbose) {
|
|
14170
|
+
console.warn(`[SqsReplicator] Replication failed for ${resource}: ${err.message}`);
|
|
14171
|
+
}
|
|
14043
14172
|
this.emit("replicator_error", {
|
|
14044
14173
|
replicator: this.name,
|
|
14045
14174
|
resource,
|
|
@@ -14112,6 +14241,9 @@ class SqsReplicator extends base_replicator_class_default {
|
|
|
14112
14241
|
});
|
|
14113
14242
|
if (ok) return result;
|
|
14114
14243
|
const errorMessage = err?.message || err || "Unknown error";
|
|
14244
|
+
if (this.config.verbose) {
|
|
14245
|
+
console.warn(`[SqsReplicator] Batch replication failed for ${resource}: ${errorMessage}`);
|
|
14246
|
+
}
|
|
14115
14247
|
this.emit("batch_replicator_error", {
|
|
14116
14248
|
replicator: this.name,
|
|
14117
14249
|
resource,
|
|
@@ -14133,6 +14265,9 @@ class SqsReplicator extends base_replicator_class_default {
|
|
|
14133
14265
|
return true;
|
|
14134
14266
|
});
|
|
14135
14267
|
if (ok) return true;
|
|
14268
|
+
if (this.config.verbose) {
|
|
14269
|
+
console.warn(`[SqsReplicator] Connection test failed: ${err.message}`);
|
|
14270
|
+
}
|
|
14136
14271
|
this.emit("connection_error", {
|
|
14137
14272
|
replicator: this.name,
|
|
14138
14273
|
error: err.message
|
|
@@ -14229,25 +14364,37 @@ class ReplicatorPlugin extends plugin_class_default {
|
|
|
14229
14364
|
return;
|
|
14230
14365
|
}
|
|
14231
14366
|
resource.on("insert", async (data) => {
|
|
14232
|
-
|
|
14367
|
+
const [ok, error] = await try_fn_default(async () => {
|
|
14233
14368
|
const completeData = { ...data, createdAt: (/* @__PURE__ */ new Date()).toISOString() };
|
|
14234
14369
|
await plugin.processReplicatorEvent("insert", resource.name, completeData.id, completeData);
|
|
14235
|
-
}
|
|
14370
|
+
});
|
|
14371
|
+
if (!ok) {
|
|
14372
|
+
if (this.config.verbose) {
|
|
14373
|
+
console.warn(`[ReplicatorPlugin] Insert event failed for resource ${resource.name}: ${error.message}`);
|
|
14374
|
+
}
|
|
14236
14375
|
this.emit("error", { operation: "insert", error: error.message, resource: resource.name });
|
|
14237
14376
|
}
|
|
14238
14377
|
});
|
|
14239
14378
|
resource.on("update", async (data, beforeData) => {
|
|
14240
|
-
|
|
14379
|
+
const [ok, error] = await try_fn_default(async () => {
|
|
14241
14380
|
const completeData = { ...data, updatedAt: (/* @__PURE__ */ new Date()).toISOString() };
|
|
14242
14381
|
await plugin.processReplicatorEvent("update", resource.name, completeData.id, completeData, beforeData);
|
|
14243
|
-
}
|
|
14382
|
+
});
|
|
14383
|
+
if (!ok) {
|
|
14384
|
+
if (this.config.verbose) {
|
|
14385
|
+
console.warn(`[ReplicatorPlugin] Update event failed for resource ${resource.name}: ${error.message}`);
|
|
14386
|
+
}
|
|
14244
14387
|
this.emit("error", { operation: "update", error: error.message, resource: resource.name });
|
|
14245
14388
|
}
|
|
14246
14389
|
});
|
|
14247
14390
|
resource.on("delete", async (data) => {
|
|
14248
|
-
|
|
14391
|
+
const [ok, error] = await try_fn_default(async () => {
|
|
14249
14392
|
await plugin.processReplicatorEvent("delete", resource.name, data.id, data);
|
|
14250
|
-
}
|
|
14393
|
+
});
|
|
14394
|
+
if (!ok) {
|
|
14395
|
+
if (this.config.verbose) {
|
|
14396
|
+
console.warn(`[ReplicatorPlugin] Delete event failed for resource ${resource.name}: ${error.message}`);
|
|
14397
|
+
}
|
|
14251
14398
|
this.emit("error", { operation: "delete", error: error.message, resource: resource.name });
|
|
14252
14399
|
}
|
|
14253
14400
|
});
|
|
@@ -14263,13 +14410,17 @@ class ReplicatorPlugin extends plugin_class_default {
|
|
|
14263
14410
|
}
|
|
14264
14411
|
async setup(database) {
|
|
14265
14412
|
this.database = database;
|
|
14266
|
-
|
|
14413
|
+
const [initOk, initError] = await try_fn_default(async () => {
|
|
14267
14414
|
await this.initializeReplicators(database);
|
|
14268
|
-
}
|
|
14269
|
-
|
|
14270
|
-
|
|
14415
|
+
});
|
|
14416
|
+
if (!initOk) {
|
|
14417
|
+
if (this.config.verbose) {
|
|
14418
|
+
console.warn(`[ReplicatorPlugin] Replicator initialization failed: ${initError.message}`);
|
|
14419
|
+
}
|
|
14420
|
+
this.emit("error", { operation: "setup", error: initError.message });
|
|
14421
|
+
throw initError;
|
|
14271
14422
|
}
|
|
14272
|
-
|
|
14423
|
+
const [logOk, logError] = await try_fn_default(async () => {
|
|
14273
14424
|
if (this.config.replicatorLogResource) {
|
|
14274
14425
|
const logRes = await database.createResource({
|
|
14275
14426
|
name: this.config.replicatorLogResource,
|
|
@@ -14286,7 +14437,15 @@ class ReplicatorPlugin extends plugin_class_default {
|
|
|
14286
14437
|
}
|
|
14287
14438
|
});
|
|
14288
14439
|
}
|
|
14289
|
-
}
|
|
14440
|
+
});
|
|
14441
|
+
if (!logOk) {
|
|
14442
|
+
if (this.config.verbose) {
|
|
14443
|
+
console.warn(`[ReplicatorPlugin] Failed to create log resource ${this.config.replicatorLogResource}: ${logError.message}`);
|
|
14444
|
+
}
|
|
14445
|
+
this.emit("replicator_log_resource_creation_error", {
|
|
14446
|
+
resourceName: this.config.replicatorLogResource,
|
|
14447
|
+
error: logError.message
|
|
14448
|
+
});
|
|
14290
14449
|
}
|
|
14291
14450
|
await this.uploadMetadataFile(database);
|
|
14292
14451
|
const originalCreateResource = database.createResource.bind(database);
|
|
@@ -14329,21 +14488,28 @@ class ReplicatorPlugin extends plugin_class_default {
|
|
|
14329
14488
|
async retryWithBackoff(operation, maxRetries = 3) {
|
|
14330
14489
|
let lastError;
|
|
14331
14490
|
for (let attempt = 1; attempt <= maxRetries; attempt++) {
|
|
14332
|
-
|
|
14333
|
-
|
|
14334
|
-
|
|
14491
|
+
const [ok, error] = await try_fn_default(operation);
|
|
14492
|
+
if (ok) {
|
|
14493
|
+
return ok;
|
|
14494
|
+
} else {
|
|
14335
14495
|
lastError = error;
|
|
14496
|
+
if (this.config.verbose) {
|
|
14497
|
+
console.warn(`[ReplicatorPlugin] Retry attempt ${attempt}/${maxRetries} failed: ${error.message}`);
|
|
14498
|
+
}
|
|
14336
14499
|
if (attempt === maxRetries) {
|
|
14337
14500
|
throw error;
|
|
14338
14501
|
}
|
|
14339
14502
|
const delay = Math.pow(2, attempt - 1) * 1e3;
|
|
14503
|
+
if (this.config.verbose) {
|
|
14504
|
+
console.warn(`[ReplicatorPlugin] Waiting ${delay}ms before retry...`);
|
|
14505
|
+
}
|
|
14340
14506
|
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
14341
14507
|
}
|
|
14342
14508
|
}
|
|
14343
14509
|
throw lastError;
|
|
14344
14510
|
}
|
|
14345
14511
|
async logError(replicator, resourceName, operation, recordId, data, error) {
|
|
14346
|
-
|
|
14512
|
+
const [ok, logError] = await try_fn_default(async () => {
|
|
14347
14513
|
const logResourceName = this.config.replicatorLogResource;
|
|
14348
14514
|
if (this.database && this.database.resources && this.database.resources[logResourceName]) {
|
|
14349
14515
|
const logResource = this.database.resources[logResourceName];
|
|
@@ -14358,7 +14524,19 @@ class ReplicatorPlugin extends plugin_class_default {
|
|
|
14358
14524
|
status: "error"
|
|
14359
14525
|
});
|
|
14360
14526
|
}
|
|
14361
|
-
}
|
|
14527
|
+
});
|
|
14528
|
+
if (!ok) {
|
|
14529
|
+
if (this.config.verbose) {
|
|
14530
|
+
console.warn(`[ReplicatorPlugin] Failed to log error for ${resourceName}: ${logError.message}`);
|
|
14531
|
+
}
|
|
14532
|
+
this.emit("replicator_log_error", {
|
|
14533
|
+
replicator: replicator.name || replicator.id,
|
|
14534
|
+
resourceName,
|
|
14535
|
+
operation,
|
|
14536
|
+
recordId,
|
|
14537
|
+
originalError: error.message,
|
|
14538
|
+
logError: logError.message
|
|
14539
|
+
});
|
|
14362
14540
|
}
|
|
14363
14541
|
}
|
|
14364
14542
|
async processReplicatorEvent(operation, resourceName, recordId, data, beforeData = null) {
|
|
@@ -14371,8 +14549,8 @@ class ReplicatorPlugin extends plugin_class_default {
|
|
|
14371
14549
|
return;
|
|
14372
14550
|
}
|
|
14373
14551
|
const promises = applicableReplicators.map(async (replicator) => {
|
|
14374
|
-
|
|
14375
|
-
const
|
|
14552
|
+
const [ok, error, result] = await try_fn_default(async () => {
|
|
14553
|
+
const result2 = await this.retryWithBackoff(
|
|
14376
14554
|
() => replicator.replicate(resourceName, operation, data, recordId, beforeData),
|
|
14377
14555
|
this.config.maxRetries
|
|
14378
14556
|
);
|
|
@@ -14381,11 +14559,17 @@ class ReplicatorPlugin extends plugin_class_default {
|
|
|
14381
14559
|
resourceName,
|
|
14382
14560
|
operation,
|
|
14383
14561
|
recordId,
|
|
14384
|
-
result,
|
|
14562
|
+
result: result2,
|
|
14385
14563
|
success: true
|
|
14386
14564
|
});
|
|
14565
|
+
return result2;
|
|
14566
|
+
});
|
|
14567
|
+
if (ok) {
|
|
14387
14568
|
return result;
|
|
14388
|
-
}
|
|
14569
|
+
} else {
|
|
14570
|
+
if (this.config.verbose) {
|
|
14571
|
+
console.warn(`[ReplicatorPlugin] Replication failed for ${replicator.name || replicator.id} on ${resourceName}: ${error.message}`);
|
|
14572
|
+
}
|
|
14389
14573
|
this.emit("replicator_error", {
|
|
14390
14574
|
replicator: replicator.name || replicator.id,
|
|
14391
14575
|
resourceName,
|
|
@@ -14410,11 +14594,14 @@ class ReplicatorPlugin extends plugin_class_default {
|
|
|
14410
14594
|
return;
|
|
14411
14595
|
}
|
|
14412
14596
|
const promises = applicableReplicators.map(async (replicator) => {
|
|
14413
|
-
|
|
14597
|
+
const [wrapperOk, wrapperError] = await try_fn_default(async () => {
|
|
14414
14598
|
const [ok, err, result] = await try_fn_default(
|
|
14415
14599
|
() => replicator.replicate(item.resourceName, item.operation, item.data, item.recordId, item.beforeData)
|
|
14416
14600
|
);
|
|
14417
14601
|
if (!ok) {
|
|
14602
|
+
if (this.config.verbose) {
|
|
14603
|
+
console.warn(`[ReplicatorPlugin] Replicator item processing failed for ${replicator.name || replicator.id} on ${item.resourceName}: ${err.message}`);
|
|
14604
|
+
}
|
|
14418
14605
|
this.emit("replicator_error", {
|
|
14419
14606
|
replicator: replicator.name || replicator.id,
|
|
14420
14607
|
resourceName: item.resourceName,
|
|
@@ -14436,18 +14623,24 @@ class ReplicatorPlugin extends plugin_class_default {
|
|
|
14436
14623
|
success: true
|
|
14437
14624
|
});
|
|
14438
14625
|
return { success: true, result };
|
|
14439
|
-
}
|
|
14626
|
+
});
|
|
14627
|
+
if (wrapperOk) {
|
|
14628
|
+
return wrapperOk;
|
|
14629
|
+
} else {
|
|
14630
|
+
if (this.config.verbose) {
|
|
14631
|
+
console.warn(`[ReplicatorPlugin] Wrapper processing failed for ${replicator.name || replicator.id} on ${item.resourceName}: ${wrapperError.message}`);
|
|
14632
|
+
}
|
|
14440
14633
|
this.emit("replicator_error", {
|
|
14441
14634
|
replicator: replicator.name || replicator.id,
|
|
14442
14635
|
resourceName: item.resourceName,
|
|
14443
14636
|
operation: item.operation,
|
|
14444
14637
|
recordId: item.recordId,
|
|
14445
|
-
error:
|
|
14638
|
+
error: wrapperError.message
|
|
14446
14639
|
});
|
|
14447
14640
|
if (this.config.logErrors && this.database) {
|
|
14448
|
-
await this.logError(replicator, item.resourceName, item.operation, item.recordId, item.data,
|
|
14641
|
+
await this.logError(replicator, item.resourceName, item.operation, item.recordId, item.data, wrapperError);
|
|
14449
14642
|
}
|
|
14450
|
-
return { success: false, error:
|
|
14643
|
+
return { success: false, error: wrapperError.message };
|
|
14451
14644
|
}
|
|
14452
14645
|
});
|
|
14453
14646
|
return Promise.allSettled(promises);
|
|
@@ -14469,9 +14662,13 @@ class ReplicatorPlugin extends plugin_class_default {
|
|
|
14469
14662
|
timestamp: typeof item.timestamp === "number" ? item.timestamp : Date.now(),
|
|
14470
14663
|
createdAt: item.createdAt || (/* @__PURE__ */ new Date()).toISOString().slice(0, 10)
|
|
14471
14664
|
};
|
|
14472
|
-
|
|
14665
|
+
const [ok, err] = await try_fn_default(async () => {
|
|
14473
14666
|
await logRes.insert(logItem);
|
|
14474
|
-
}
|
|
14667
|
+
});
|
|
14668
|
+
if (!ok) {
|
|
14669
|
+
if (this.config.verbose) {
|
|
14670
|
+
console.warn(`[ReplicatorPlugin] Failed to log replicator item: ${err.message}`);
|
|
14671
|
+
}
|
|
14475
14672
|
this.emit("replicator.log.failed", { error: err, item });
|
|
14476
14673
|
}
|
|
14477
14674
|
}
|
|
@@ -14577,14 +14774,23 @@ class ReplicatorPlugin extends plugin_class_default {
|
|
|
14577
14774
|
this.emit("replicator.sync.completed", { replicatorId, stats: this.stats });
|
|
14578
14775
|
}
|
|
14579
14776
|
async cleanup() {
|
|
14580
|
-
|
|
14777
|
+
const [ok, error] = await try_fn_default(async () => {
|
|
14581
14778
|
if (this.replicators && this.replicators.length > 0) {
|
|
14582
14779
|
const cleanupPromises = this.replicators.map(async (replicator) => {
|
|
14583
|
-
|
|
14780
|
+
const [replicatorOk, replicatorError] = await try_fn_default(async () => {
|
|
14584
14781
|
if (replicator && typeof replicator.cleanup === "function") {
|
|
14585
14782
|
await replicator.cleanup();
|
|
14586
14783
|
}
|
|
14587
|
-
}
|
|
14784
|
+
});
|
|
14785
|
+
if (!replicatorOk) {
|
|
14786
|
+
if (this.config.verbose) {
|
|
14787
|
+
console.warn(`[ReplicatorPlugin] Failed to cleanup replicator ${replicator.name || replicator.id}: ${replicatorError.message}`);
|
|
14788
|
+
}
|
|
14789
|
+
this.emit("replicator_cleanup_error", {
|
|
14790
|
+
replicator: replicator.name || replicator.id || "unknown",
|
|
14791
|
+
driver: replicator.driver || "unknown",
|
|
14792
|
+
error: replicatorError.message
|
|
14793
|
+
});
|
|
14588
14794
|
}
|
|
14589
14795
|
});
|
|
14590
14796
|
await Promise.allSettled(cleanupPromises);
|
|
@@ -14593,7 +14799,14 @@ class ReplicatorPlugin extends plugin_class_default {
|
|
|
14593
14799
|
this.database = null;
|
|
14594
14800
|
this.eventListenersInstalled.clear();
|
|
14595
14801
|
this.removeAllListeners();
|
|
14596
|
-
}
|
|
14802
|
+
});
|
|
14803
|
+
if (!ok) {
|
|
14804
|
+
if (this.config.verbose) {
|
|
14805
|
+
console.warn(`[ReplicatorPlugin] Failed to cleanup plugin: ${error.message}`);
|
|
14806
|
+
}
|
|
14807
|
+
this.emit("replicator_plugin_cleanup_error", {
|
|
14808
|
+
error: error.message
|
|
14809
|
+
});
|
|
14597
14810
|
}
|
|
14598
14811
|
}
|
|
14599
14812
|
}
|