s3db.js 7.3.1 → 7.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/s3db.cjs.js +2394 -191
- package/dist/s3db.cjs.min.js +1 -1
- package/dist/s3db.es.js +2394 -191
- package/dist/s3db.es.min.js +1 -1
- package/dist/s3db.iife.js +2393 -190
- package/dist/s3db.iife.min.js +1 -1
- package/package.json +25 -26
- package/src/plugins/cache/filesystem-cache.class.js +1 -1
- package/src/plugins/cache/partition-aware-filesystem-cache.class.js +1 -1
- package/src/plugins/cache/s3-cache.class.js +1 -1
package/dist/s3db.cjs.js
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
Object.defineProperty(exports, '__esModule', { value: true });
|
|
4
4
|
|
|
5
5
|
var nanoid = require('nanoid');
|
|
6
|
-
var zlib = require('zlib');
|
|
6
|
+
var zlib = require('node:zlib');
|
|
7
7
|
var promisePool = require('@supercharge/promise-pool');
|
|
8
8
|
var web = require('node:stream/web');
|
|
9
9
|
var lodashEs = require('lodash-es');
|
|
@@ -1345,7 +1345,8 @@ class AuditPlugin extends plugin_class_default {
|
|
|
1345
1345
|
version: "2.0"
|
|
1346
1346
|
})
|
|
1347
1347
|
};
|
|
1348
|
-
this.logAudit(auditRecord).catch(
|
|
1348
|
+
this.logAudit(auditRecord).catch(() => {
|
|
1349
|
+
});
|
|
1349
1350
|
});
|
|
1350
1351
|
resource.on("update", async (data) => {
|
|
1351
1352
|
const recordId = data.id;
|
|
@@ -1371,7 +1372,8 @@ class AuditPlugin extends plugin_class_default {
|
|
|
1371
1372
|
version: "2.0"
|
|
1372
1373
|
})
|
|
1373
1374
|
};
|
|
1374
|
-
this.logAudit(auditRecord).catch(
|
|
1375
|
+
this.logAudit(auditRecord).catch(() => {
|
|
1376
|
+
});
|
|
1375
1377
|
});
|
|
1376
1378
|
resource.on("delete", async (data) => {
|
|
1377
1379
|
const recordId = data.id;
|
|
@@ -1397,7 +1399,8 @@ class AuditPlugin extends plugin_class_default {
|
|
|
1397
1399
|
version: "2.0"
|
|
1398
1400
|
})
|
|
1399
1401
|
};
|
|
1400
|
-
this.logAudit(auditRecord).catch(
|
|
1402
|
+
this.logAudit(auditRecord).catch(() => {
|
|
1403
|
+
});
|
|
1401
1404
|
});
|
|
1402
1405
|
resource.useMiddleware("deleteMany", async (ctx, next) => {
|
|
1403
1406
|
const ids = ctx.args[0];
|
|
@@ -1430,7 +1433,8 @@ class AuditPlugin extends plugin_class_default {
|
|
|
1430
1433
|
batchOperation: true
|
|
1431
1434
|
})
|
|
1432
1435
|
};
|
|
1433
|
-
this.logAudit(auditRecord).catch(
|
|
1436
|
+
this.logAudit(auditRecord).catch(() => {
|
|
1437
|
+
});
|
|
1434
1438
|
}
|
|
1435
1439
|
}
|
|
1436
1440
|
return result;
|
|
@@ -2077,6 +2081,16 @@ if (typeof Object.create === 'function'){
|
|
|
2077
2081
|
// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
|
|
2078
2082
|
// USE OR OTHER DEALINGS IN THE SOFTWARE.
|
|
2079
2083
|
|
|
2084
|
+
var getOwnPropertyDescriptors = Object.getOwnPropertyDescriptors ||
|
|
2085
|
+
function getOwnPropertyDescriptors(obj) {
|
|
2086
|
+
var keys = Object.keys(obj);
|
|
2087
|
+
var descriptors = {};
|
|
2088
|
+
for (var i = 0; i < keys.length; i++) {
|
|
2089
|
+
descriptors[keys[i]] = Object.getOwnPropertyDescriptor(obj, keys[i]);
|
|
2090
|
+
}
|
|
2091
|
+
return descriptors;
|
|
2092
|
+
};
|
|
2093
|
+
|
|
2080
2094
|
var formatRegExp = /%[sdj%]/g;
|
|
2081
2095
|
function format(f) {
|
|
2082
2096
|
if (!isString(f)) {
|
|
@@ -2562,6 +2576,64 @@ function hasOwnProperty(obj, prop) {
|
|
|
2562
2576
|
return Object.prototype.hasOwnProperty.call(obj, prop);
|
|
2563
2577
|
}
|
|
2564
2578
|
|
|
2579
|
+
var kCustomPromisifiedSymbol = typeof Symbol !== 'undefined' ? Symbol('util.promisify.custom') : undefined;
|
|
2580
|
+
|
|
2581
|
+
function promisify(original) {
|
|
2582
|
+
if (typeof original !== 'function')
|
|
2583
|
+
throw new TypeError('The "original" argument must be of type Function');
|
|
2584
|
+
|
|
2585
|
+
if (kCustomPromisifiedSymbol && original[kCustomPromisifiedSymbol]) {
|
|
2586
|
+
var fn = original[kCustomPromisifiedSymbol];
|
|
2587
|
+
if (typeof fn !== 'function') {
|
|
2588
|
+
throw new TypeError('The "util.promisify.custom" argument must be of type Function');
|
|
2589
|
+
}
|
|
2590
|
+
Object.defineProperty(fn, kCustomPromisifiedSymbol, {
|
|
2591
|
+
value: fn, enumerable: false, writable: false, configurable: true
|
|
2592
|
+
});
|
|
2593
|
+
return fn;
|
|
2594
|
+
}
|
|
2595
|
+
|
|
2596
|
+
function fn() {
|
|
2597
|
+
var promiseResolve, promiseReject;
|
|
2598
|
+
var promise = new Promise(function (resolve, reject) {
|
|
2599
|
+
promiseResolve = resolve;
|
|
2600
|
+
promiseReject = reject;
|
|
2601
|
+
});
|
|
2602
|
+
|
|
2603
|
+
var args = [];
|
|
2604
|
+
for (var i = 0; i < arguments.length; i++) {
|
|
2605
|
+
args.push(arguments[i]);
|
|
2606
|
+
}
|
|
2607
|
+
args.push(function (err, value) {
|
|
2608
|
+
if (err) {
|
|
2609
|
+
promiseReject(err);
|
|
2610
|
+
} else {
|
|
2611
|
+
promiseResolve(value);
|
|
2612
|
+
}
|
|
2613
|
+
});
|
|
2614
|
+
|
|
2615
|
+
try {
|
|
2616
|
+
original.apply(this, args);
|
|
2617
|
+
} catch (err) {
|
|
2618
|
+
promiseReject(err);
|
|
2619
|
+
}
|
|
2620
|
+
|
|
2621
|
+
return promise;
|
|
2622
|
+
}
|
|
2623
|
+
|
|
2624
|
+
Object.setPrototypeOf(fn, Object.getPrototypeOf(original));
|
|
2625
|
+
|
|
2626
|
+
if (kCustomPromisifiedSymbol) Object.defineProperty(fn, kCustomPromisifiedSymbol, {
|
|
2627
|
+
value: fn, enumerable: false, writable: false, configurable: true
|
|
2628
|
+
});
|
|
2629
|
+
return Object.defineProperties(
|
|
2630
|
+
fn,
|
|
2631
|
+
getOwnPropertyDescriptors(original)
|
|
2632
|
+
);
|
|
2633
|
+
}
|
|
2634
|
+
|
|
2635
|
+
promisify.custom = kCustomPromisifiedSymbol;
|
|
2636
|
+
|
|
2565
2637
|
var lookup = [];
|
|
2566
2638
|
var revLookup = [];
|
|
2567
2639
|
var Arr = typeof Uint8Array !== 'undefined' ? Uint8Array : Array;
|
|
@@ -6812,186 +6884,1026 @@ class MemoryCache extends Cache {
|
|
|
6812
6884
|
}
|
|
6813
6885
|
var memory_cache_class_default = MemoryCache;
|
|
6814
6886
|
|
|
6815
|
-
|
|
6816
|
-
|
|
6817
|
-
|
|
6818
|
-
|
|
6819
|
-
|
|
6820
|
-
|
|
6821
|
-
|
|
6887
|
+
var fs = {};
|
|
6888
|
+
|
|
6889
|
+
const readFile$1 = promisify(fs.readFile);
|
|
6890
|
+
const writeFile$1 = promisify(fs.writeFile);
|
|
6891
|
+
const unlink = promisify(fs.unlink);
|
|
6892
|
+
const readdir$1 = promisify(fs.readdir);
|
|
6893
|
+
const stat$1 = promisify(fs.stat);
|
|
6894
|
+
const mkdir = promisify(fs.mkdir);
|
|
6895
|
+
class FilesystemCache extends Cache {
|
|
6896
|
+
constructor({
|
|
6897
|
+
directory,
|
|
6898
|
+
prefix = "cache",
|
|
6899
|
+
ttl = 36e5,
|
|
6900
|
+
enableCompression = true,
|
|
6901
|
+
compressionThreshold = 1024,
|
|
6902
|
+
createDirectory = true,
|
|
6903
|
+
fileExtension = ".cache",
|
|
6904
|
+
enableMetadata = true,
|
|
6905
|
+
maxFileSize = 10485760,
|
|
6906
|
+
// 10MB
|
|
6907
|
+
enableStats = false,
|
|
6908
|
+
enableCleanup = true,
|
|
6909
|
+
cleanupInterval = 3e5,
|
|
6910
|
+
// 5 minutes
|
|
6911
|
+
encoding = "utf8",
|
|
6912
|
+
fileMode = 420,
|
|
6913
|
+
enableBackup = false,
|
|
6914
|
+
backupSuffix = ".bak",
|
|
6915
|
+
enableLocking = false,
|
|
6916
|
+
lockTimeout = 5e3,
|
|
6917
|
+
enableJournal = false,
|
|
6918
|
+
journalFile = "cache.journal",
|
|
6919
|
+
...config
|
|
6920
|
+
}) {
|
|
6921
|
+
super(config);
|
|
6922
|
+
if (!directory) {
|
|
6923
|
+
throw new Error("FilesystemCache: directory parameter is required");
|
|
6924
|
+
}
|
|
6925
|
+
this.directory = path.resolve(directory);
|
|
6926
|
+
this.prefix = prefix;
|
|
6927
|
+
this.ttl = ttl;
|
|
6928
|
+
this.enableCompression = enableCompression;
|
|
6929
|
+
this.compressionThreshold = compressionThreshold;
|
|
6930
|
+
this.createDirectory = createDirectory;
|
|
6931
|
+
this.fileExtension = fileExtension;
|
|
6932
|
+
this.enableMetadata = enableMetadata;
|
|
6933
|
+
this.maxFileSize = maxFileSize;
|
|
6934
|
+
this.enableStats = enableStats;
|
|
6935
|
+
this.enableCleanup = enableCleanup;
|
|
6936
|
+
this.cleanupInterval = cleanupInterval;
|
|
6937
|
+
this.encoding = encoding;
|
|
6938
|
+
this.fileMode = fileMode;
|
|
6939
|
+
this.enableBackup = enableBackup;
|
|
6940
|
+
this.backupSuffix = backupSuffix;
|
|
6941
|
+
this.enableLocking = enableLocking;
|
|
6942
|
+
this.lockTimeout = lockTimeout;
|
|
6943
|
+
this.enableJournal = enableJournal;
|
|
6944
|
+
this.journalFile = path.join(this.directory, journalFile);
|
|
6945
|
+
this.stats = {
|
|
6946
|
+
hits: 0,
|
|
6947
|
+
misses: 0,
|
|
6948
|
+
sets: 0,
|
|
6949
|
+
deletes: 0,
|
|
6950
|
+
clears: 0,
|
|
6951
|
+
errors: 0
|
|
6822
6952
|
};
|
|
6823
|
-
|
|
6824
|
-
|
|
6825
|
-
|
|
6826
|
-
}
|
|
6827
|
-
async
|
|
6828
|
-
if (this.
|
|
6829
|
-
this.
|
|
6830
|
-
}
|
|
6831
|
-
|
|
6832
|
-
|
|
6833
|
-
|
|
6953
|
+
this.locks = /* @__PURE__ */ new Map();
|
|
6954
|
+
this.cleanupTimer = null;
|
|
6955
|
+
this._init();
|
|
6956
|
+
}
|
|
6957
|
+
async _init() {
|
|
6958
|
+
if (this.createDirectory) {
|
|
6959
|
+
await this._ensureDirectory(this.directory);
|
|
6960
|
+
}
|
|
6961
|
+
if (this.enableCleanup && this.cleanupInterval > 0) {
|
|
6962
|
+
this.cleanupTimer = setInterval(() => {
|
|
6963
|
+
this._cleanup().catch((err) => {
|
|
6964
|
+
console.warn("FilesystemCache cleanup error:", err.message);
|
|
6965
|
+
});
|
|
6966
|
+
}, this.cleanupInterval);
|
|
6834
6967
|
}
|
|
6835
|
-
this.installDatabaseProxy();
|
|
6836
|
-
this.installResourceHooks();
|
|
6837
6968
|
}
|
|
6838
|
-
async
|
|
6969
|
+
async _ensureDirectory(dir) {
|
|
6970
|
+
const [ok, err] = await try_fn_default(async () => {
|
|
6971
|
+
await mkdir(dir, { recursive: true });
|
|
6972
|
+
});
|
|
6973
|
+
if (!ok && err.code !== "EEXIST") {
|
|
6974
|
+
throw new Error(`Failed to create cache directory: ${err.message}`);
|
|
6975
|
+
}
|
|
6839
6976
|
}
|
|
6840
|
-
|
|
6977
|
+
_getFilePath(key) {
|
|
6978
|
+
const sanitizedKey = key.replace(/[<>:"/\\|?*]/g, "_");
|
|
6979
|
+
const filename = `${this.prefix}_${sanitizedKey}${this.fileExtension}`;
|
|
6980
|
+
return path.join(this.directory, filename);
|
|
6841
6981
|
}
|
|
6842
|
-
|
|
6843
|
-
|
|
6844
|
-
return;
|
|
6845
|
-
}
|
|
6846
|
-
const installResourceHooks = this.installResourceHooks.bind(this);
|
|
6847
|
-
this.database._originalCreateResourceForCache = this.database.createResource;
|
|
6848
|
-
this.database.createResource = async function(...args) {
|
|
6849
|
-
const resource = await this._originalCreateResourceForCache(...args);
|
|
6850
|
-
installResourceHooks(resource);
|
|
6851
|
-
return resource;
|
|
6852
|
-
};
|
|
6853
|
-
this.database._cacheProxyInstalled = true;
|
|
6982
|
+
_getMetadataPath(filePath) {
|
|
6983
|
+
return filePath + ".meta";
|
|
6854
6984
|
}
|
|
6855
|
-
|
|
6856
|
-
|
|
6857
|
-
|
|
6985
|
+
async _set(key, data) {
|
|
6986
|
+
const filePath = this._getFilePath(key);
|
|
6987
|
+
try {
|
|
6988
|
+
let serialized = JSON.stringify(data);
|
|
6989
|
+
const originalSize = Buffer.byteLength(serialized, this.encoding);
|
|
6990
|
+
if (originalSize > this.maxFileSize) {
|
|
6991
|
+
throw new Error(`Cache data exceeds maximum file size: ${originalSize} > ${this.maxFileSize}`);
|
|
6992
|
+
}
|
|
6993
|
+
let compressed = false;
|
|
6994
|
+
let finalData = serialized;
|
|
6995
|
+
if (this.enableCompression && originalSize >= this.compressionThreshold) {
|
|
6996
|
+
const compressedBuffer = zlib.gzipSync(Buffer.from(serialized, this.encoding));
|
|
6997
|
+
finalData = compressedBuffer.toString("base64");
|
|
6998
|
+
compressed = true;
|
|
6999
|
+
}
|
|
7000
|
+
if (this.enableBackup && await this._fileExists(filePath)) {
|
|
7001
|
+
const backupPath = filePath + this.backupSuffix;
|
|
7002
|
+
await this._copyFile(filePath, backupPath);
|
|
7003
|
+
}
|
|
7004
|
+
if (this.enableLocking) {
|
|
7005
|
+
await this._acquireLock(filePath);
|
|
7006
|
+
}
|
|
7007
|
+
try {
|
|
7008
|
+
await writeFile$1(filePath, finalData, {
|
|
7009
|
+
encoding: compressed ? "utf8" : this.encoding,
|
|
7010
|
+
mode: this.fileMode
|
|
7011
|
+
});
|
|
7012
|
+
if (this.enableMetadata) {
|
|
7013
|
+
const metadata = {
|
|
7014
|
+
key,
|
|
7015
|
+
timestamp: Date.now(),
|
|
7016
|
+
ttl: this.ttl,
|
|
7017
|
+
compressed,
|
|
7018
|
+
originalSize,
|
|
7019
|
+
compressedSize: compressed ? Buffer.byteLength(finalData, "utf8") : originalSize,
|
|
7020
|
+
compressionRatio: compressed ? (Buffer.byteLength(finalData, "utf8") / originalSize).toFixed(2) : 1
|
|
7021
|
+
};
|
|
7022
|
+
await writeFile$1(this._getMetadataPath(filePath), JSON.stringify(metadata), {
|
|
7023
|
+
encoding: this.encoding,
|
|
7024
|
+
mode: this.fileMode
|
|
7025
|
+
});
|
|
7026
|
+
}
|
|
7027
|
+
if (this.enableStats) {
|
|
7028
|
+
this.stats.sets++;
|
|
7029
|
+
}
|
|
7030
|
+
if (this.enableJournal) {
|
|
7031
|
+
await this._journalOperation("set", key, { size: originalSize, compressed });
|
|
7032
|
+
}
|
|
7033
|
+
} finally {
|
|
7034
|
+
if (this.enableLocking) {
|
|
7035
|
+
this._releaseLock(filePath);
|
|
7036
|
+
}
|
|
7037
|
+
}
|
|
7038
|
+
return data;
|
|
7039
|
+
} catch (error) {
|
|
7040
|
+
if (this.enableStats) {
|
|
7041
|
+
this.stats.errors++;
|
|
7042
|
+
}
|
|
7043
|
+
throw new Error(`Failed to set cache key '${key}': ${error.message}`);
|
|
6858
7044
|
}
|
|
6859
7045
|
}
|
|
6860
|
-
|
|
6861
|
-
|
|
6862
|
-
|
|
6863
|
-
|
|
6864
|
-
|
|
6865
|
-
|
|
6866
|
-
enumerable: false
|
|
6867
|
-
});
|
|
6868
|
-
resource.cacheKeyFor = async (options = {}) => {
|
|
6869
|
-
const { action, params = {}, partition, partitionValues } = options;
|
|
6870
|
-
return this.generateCacheKey(resource, action, params, partition, partitionValues);
|
|
6871
|
-
};
|
|
6872
|
-
const cacheMethods = [
|
|
6873
|
-
"count",
|
|
6874
|
-
"listIds",
|
|
6875
|
-
"getMany",
|
|
6876
|
-
"getAll",
|
|
6877
|
-
"page",
|
|
6878
|
-
"list",
|
|
6879
|
-
"get"
|
|
6880
|
-
];
|
|
6881
|
-
for (const method of cacheMethods) {
|
|
6882
|
-
resource.useMiddleware(method, async (ctx, next) => {
|
|
6883
|
-
let key;
|
|
6884
|
-
if (method === "getMany") {
|
|
6885
|
-
key = await resource.cacheKeyFor({ action: method, params: { ids: ctx.args[0] } });
|
|
6886
|
-
} else if (method === "page") {
|
|
6887
|
-
const { offset, size, partition, partitionValues } = ctx.args[0] || {};
|
|
6888
|
-
key = await resource.cacheKeyFor({ action: method, params: { offset, size }, partition, partitionValues });
|
|
6889
|
-
} else if (method === "list" || method === "listIds" || method === "count") {
|
|
6890
|
-
const { partition, partitionValues } = ctx.args[0] || {};
|
|
6891
|
-
key = await resource.cacheKeyFor({ action: method, partition, partitionValues });
|
|
6892
|
-
} else if (method === "getAll") {
|
|
6893
|
-
key = await resource.cacheKeyFor({ action: method });
|
|
6894
|
-
} else if (method === "get") {
|
|
6895
|
-
key = await resource.cacheKeyFor({ action: method, params: { id: ctx.args[0] } });
|
|
7046
|
+
async _get(key) {
|
|
7047
|
+
const filePath = this._getFilePath(key);
|
|
7048
|
+
try {
|
|
7049
|
+
if (!await this._fileExists(filePath)) {
|
|
7050
|
+
if (this.enableStats) {
|
|
7051
|
+
this.stats.misses++;
|
|
6896
7052
|
}
|
|
6897
|
-
|
|
6898
|
-
|
|
6899
|
-
|
|
6900
|
-
|
|
6901
|
-
|
|
6902
|
-
|
|
6903
|
-
|
|
6904
|
-
|
|
6905
|
-
|
|
6906
|
-
|
|
6907
|
-
|
|
6908
|
-
|
|
6909
|
-
|
|
6910
|
-
await this.clearCacheForResource(resource, ctx.args[0]);
|
|
6911
|
-
} else if (method === "update") {
|
|
6912
|
-
await this.clearCacheForResource(resource, { id: ctx.args[0], ...ctx.args[1] });
|
|
6913
|
-
} else if (method === "delete") {
|
|
6914
|
-
let data = { id: ctx.args[0] };
|
|
6915
|
-
if (typeof resource.get === "function") {
|
|
6916
|
-
const [ok, err, full] = await try_fn_default(() => resource.get(ctx.args[0]));
|
|
6917
|
-
if (ok && full) data = full;
|
|
7053
|
+
return null;
|
|
7054
|
+
}
|
|
7055
|
+
let isExpired = false;
|
|
7056
|
+
if (this.enableMetadata) {
|
|
7057
|
+
const metadataPath = this._getMetadataPath(filePath);
|
|
7058
|
+
if (await this._fileExists(metadataPath)) {
|
|
7059
|
+
const [ok, err, metadata] = await try_fn_default(async () => {
|
|
7060
|
+
const metaContent = await readFile$1(metadataPath, this.encoding);
|
|
7061
|
+
return JSON.parse(metaContent);
|
|
7062
|
+
});
|
|
7063
|
+
if (ok && metadata.ttl > 0) {
|
|
7064
|
+
const age = Date.now() - metadata.timestamp;
|
|
7065
|
+
isExpired = age > metadata.ttl;
|
|
6918
7066
|
}
|
|
6919
|
-
await this.clearCacheForResource(resource, data);
|
|
6920
|
-
} else if (method === "deleteMany") {
|
|
6921
|
-
await this.clearCacheForResource(resource);
|
|
6922
7067
|
}
|
|
6923
|
-
|
|
6924
|
-
|
|
6925
|
-
|
|
6926
|
-
|
|
6927
|
-
|
|
6928
|
-
|
|
6929
|
-
|
|
6930
|
-
|
|
6931
|
-
|
|
6932
|
-
if (!data) {
|
|
6933
|
-
for (const partitionName of Object.keys(resource.config.partitions)) {
|
|
6934
|
-
const partitionKeyPrefix = join(keyPrefix, `partition=${partitionName}`);
|
|
6935
|
-
await resource.cache.clear(partitionKeyPrefix);
|
|
7068
|
+
} else if (this.ttl > 0) {
|
|
7069
|
+
const stats = await stat$1(filePath);
|
|
7070
|
+
const age = Date.now() - stats.mtime.getTime();
|
|
7071
|
+
isExpired = age > this.ttl;
|
|
7072
|
+
}
|
|
7073
|
+
if (isExpired) {
|
|
7074
|
+
await this._del(key);
|
|
7075
|
+
if (this.enableStats) {
|
|
7076
|
+
this.stats.misses++;
|
|
6936
7077
|
}
|
|
6937
|
-
|
|
6938
|
-
|
|
6939
|
-
|
|
6940
|
-
|
|
6941
|
-
|
|
6942
|
-
|
|
7078
|
+
return null;
|
|
7079
|
+
}
|
|
7080
|
+
if (this.enableLocking) {
|
|
7081
|
+
await this._acquireLock(filePath);
|
|
7082
|
+
}
|
|
7083
|
+
try {
|
|
7084
|
+
const content = await readFile$1(filePath, this.encoding);
|
|
7085
|
+
let isCompressed = false;
|
|
7086
|
+
if (this.enableMetadata) {
|
|
7087
|
+
const metadataPath = this._getMetadataPath(filePath);
|
|
7088
|
+
if (await this._fileExists(metadataPath)) {
|
|
7089
|
+
const [ok, err, metadata] = await try_fn_default(async () => {
|
|
7090
|
+
const metaContent = await readFile$1(metadataPath, this.encoding);
|
|
7091
|
+
return JSON.parse(metaContent);
|
|
7092
|
+
});
|
|
7093
|
+
if (ok) {
|
|
7094
|
+
isCompressed = metadata.compressed;
|
|
7095
|
+
}
|
|
7096
|
+
}
|
|
7097
|
+
}
|
|
7098
|
+
let finalContent = content;
|
|
7099
|
+
if (isCompressed || this.enableCompression && content.match(/^[A-Za-z0-9+/=]+$/)) {
|
|
7100
|
+
try {
|
|
7101
|
+
const compressedBuffer = Buffer.from(content, "base64");
|
|
7102
|
+
finalContent = zlib.gunzipSync(compressedBuffer).toString(this.encoding);
|
|
7103
|
+
} catch (decompressError) {
|
|
7104
|
+
finalContent = content;
|
|
6943
7105
|
}
|
|
6944
7106
|
}
|
|
7107
|
+
const data = JSON.parse(finalContent);
|
|
7108
|
+
if (this.enableStats) {
|
|
7109
|
+
this.stats.hits++;
|
|
7110
|
+
}
|
|
7111
|
+
return data;
|
|
7112
|
+
} finally {
|
|
7113
|
+
if (this.enableLocking) {
|
|
7114
|
+
this._releaseLock(filePath);
|
|
7115
|
+
}
|
|
6945
7116
|
}
|
|
7117
|
+
} catch (error) {
|
|
7118
|
+
if (this.enableStats) {
|
|
7119
|
+
this.stats.errors++;
|
|
7120
|
+
}
|
|
7121
|
+
await this._del(key);
|
|
7122
|
+
return null;
|
|
6946
7123
|
}
|
|
6947
7124
|
}
|
|
6948
|
-
async
|
|
6949
|
-
const
|
|
6950
|
-
|
|
6951
|
-
|
|
6952
|
-
|
|
6953
|
-
|
|
6954
|
-
|
|
6955
|
-
|
|
6956
|
-
if (
|
|
6957
|
-
|
|
7125
|
+
async _del(key) {
|
|
7126
|
+
const filePath = this._getFilePath(key);
|
|
7127
|
+
try {
|
|
7128
|
+
if (await this._fileExists(filePath)) {
|
|
7129
|
+
await unlink(filePath);
|
|
7130
|
+
}
|
|
7131
|
+
if (this.enableMetadata) {
|
|
7132
|
+
const metadataPath = this._getMetadataPath(filePath);
|
|
7133
|
+
if (await this._fileExists(metadataPath)) {
|
|
7134
|
+
await unlink(metadataPath);
|
|
6958
7135
|
}
|
|
6959
7136
|
}
|
|
7137
|
+
if (this.enableBackup) {
|
|
7138
|
+
const backupPath = filePath + this.backupSuffix;
|
|
7139
|
+
if (await this._fileExists(backupPath)) {
|
|
7140
|
+
await unlink(backupPath);
|
|
7141
|
+
}
|
|
7142
|
+
}
|
|
7143
|
+
if (this.enableStats) {
|
|
7144
|
+
this.stats.deletes++;
|
|
7145
|
+
}
|
|
7146
|
+
if (this.enableJournal) {
|
|
7147
|
+
await this._journalOperation("delete", key);
|
|
7148
|
+
}
|
|
7149
|
+
return true;
|
|
7150
|
+
} catch (error) {
|
|
7151
|
+
if (this.enableStats) {
|
|
7152
|
+
this.stats.errors++;
|
|
7153
|
+
}
|
|
7154
|
+
throw new Error(`Failed to delete cache key '${key}': ${error.message}`);
|
|
6960
7155
|
}
|
|
6961
|
-
if (Object.keys(params).length > 0) {
|
|
6962
|
-
const paramsHash = await this.hashParams(params);
|
|
6963
|
-
keyParts.push(paramsHash);
|
|
6964
|
-
}
|
|
6965
|
-
return join(...keyParts) + ".json.gz";
|
|
6966
|
-
}
|
|
6967
|
-
async hashParams(params) {
|
|
6968
|
-
const sortedParams = Object.keys(params).sort().map((key) => `${key}:${params[key]}`).join("|") || "empty";
|
|
6969
|
-
return await sha256(sortedParams);
|
|
6970
|
-
}
|
|
6971
|
-
// Utility methods
|
|
6972
|
-
async getCacheStats() {
|
|
6973
|
-
if (!this.driver) return null;
|
|
6974
|
-
return {
|
|
6975
|
-
size: await this.driver.size(),
|
|
6976
|
-
keys: await this.driver.keys(),
|
|
6977
|
-
driver: this.driver.constructor.name
|
|
6978
|
-
};
|
|
6979
7156
|
}
|
|
6980
|
-
async
|
|
6981
|
-
|
|
6982
|
-
|
|
6983
|
-
|
|
6984
|
-
|
|
6985
|
-
|
|
7157
|
+
async _clear(prefix) {
|
|
7158
|
+
try {
|
|
7159
|
+
const files = await readdir$1(this.directory);
|
|
7160
|
+
const cacheFiles = files.filter((file) => {
|
|
7161
|
+
if (!file.startsWith(this.prefix)) return false;
|
|
7162
|
+
if (!file.endsWith(this.fileExtension)) return false;
|
|
7163
|
+
if (prefix) {
|
|
7164
|
+
const keyPart = file.slice(this.prefix.length + 1, -this.fileExtension.length);
|
|
7165
|
+
return keyPart.startsWith(prefix);
|
|
7166
|
+
}
|
|
7167
|
+
return true;
|
|
7168
|
+
});
|
|
7169
|
+
for (const file of cacheFiles) {
|
|
7170
|
+
const filePath = path.join(this.directory, file);
|
|
7171
|
+
if (await this._fileExists(filePath)) {
|
|
7172
|
+
await unlink(filePath);
|
|
7173
|
+
}
|
|
7174
|
+
if (this.enableMetadata) {
|
|
7175
|
+
const metadataPath = this._getMetadataPath(filePath);
|
|
7176
|
+
if (await this._fileExists(metadataPath)) {
|
|
7177
|
+
await unlink(metadataPath);
|
|
7178
|
+
}
|
|
7179
|
+
}
|
|
7180
|
+
if (this.enableBackup) {
|
|
7181
|
+
const backupPath = filePath + this.backupSuffix;
|
|
7182
|
+
if (await this._fileExists(backupPath)) {
|
|
7183
|
+
await unlink(backupPath);
|
|
7184
|
+
}
|
|
7185
|
+
}
|
|
7186
|
+
}
|
|
7187
|
+
if (this.enableStats) {
|
|
7188
|
+
this.stats.clears++;
|
|
7189
|
+
}
|
|
7190
|
+
if (this.enableJournal) {
|
|
7191
|
+
await this._journalOperation("clear", prefix || "all", { count: cacheFiles.length });
|
|
7192
|
+
}
|
|
7193
|
+
return true;
|
|
7194
|
+
} catch (error) {
|
|
7195
|
+
if (this.enableStats) {
|
|
7196
|
+
this.stats.errors++;
|
|
6986
7197
|
}
|
|
7198
|
+
throw new Error(`Failed to clear cache: ${error.message}`);
|
|
6987
7199
|
}
|
|
6988
7200
|
}
|
|
6989
|
-
async
|
|
6990
|
-
const
|
|
6991
|
-
|
|
6992
|
-
|
|
6993
|
-
|
|
6994
|
-
|
|
7201
|
+
async size() {
|
|
7202
|
+
const keys = await this.keys();
|
|
7203
|
+
return keys.length;
|
|
7204
|
+
}
|
|
7205
|
+
async keys() {
|
|
7206
|
+
try {
|
|
7207
|
+
const files = await readdir$1(this.directory);
|
|
7208
|
+
const cacheFiles = files.filter(
|
|
7209
|
+
(file) => file.startsWith(this.prefix) && file.endsWith(this.fileExtension)
|
|
7210
|
+
);
|
|
7211
|
+
const keys = cacheFiles.map((file) => {
|
|
7212
|
+
const keyPart = file.slice(this.prefix.length + 1, -this.fileExtension.length);
|
|
7213
|
+
return keyPart;
|
|
7214
|
+
});
|
|
7215
|
+
return keys;
|
|
7216
|
+
} catch (error) {
|
|
7217
|
+
console.warn("FilesystemCache: Failed to list keys:", error.message);
|
|
7218
|
+
return [];
|
|
7219
|
+
}
|
|
7220
|
+
}
|
|
7221
|
+
// Helper methods
|
|
7222
|
+
async _fileExists(filePath) {
|
|
7223
|
+
const [ok] = await try_fn_default(async () => {
|
|
7224
|
+
await stat$1(filePath);
|
|
7225
|
+
});
|
|
7226
|
+
return ok;
|
|
7227
|
+
}
|
|
7228
|
+
async _copyFile(src, dest) {
|
|
7229
|
+
const [ok, err] = await try_fn_default(async () => {
|
|
7230
|
+
const content = await readFile$1(src);
|
|
7231
|
+
await writeFile$1(dest, content);
|
|
7232
|
+
});
|
|
7233
|
+
if (!ok) {
|
|
7234
|
+
console.warn("FilesystemCache: Failed to create backup:", err.message);
|
|
7235
|
+
}
|
|
7236
|
+
}
|
|
7237
|
+
async _cleanup() {
|
|
7238
|
+
if (!this.ttl || this.ttl <= 0) return;
|
|
7239
|
+
try {
|
|
7240
|
+
const files = await readdir$1(this.directory);
|
|
7241
|
+
const now = Date.now();
|
|
7242
|
+
for (const file of files) {
|
|
7243
|
+
if (!file.startsWith(this.prefix) || !file.endsWith(this.fileExtension)) {
|
|
7244
|
+
continue;
|
|
7245
|
+
}
|
|
7246
|
+
const filePath = path.join(this.directory, file);
|
|
7247
|
+
let shouldDelete = false;
|
|
7248
|
+
if (this.enableMetadata) {
|
|
7249
|
+
const metadataPath = this._getMetadataPath(filePath);
|
|
7250
|
+
if (await this._fileExists(metadataPath)) {
|
|
7251
|
+
const [ok, err, metadata] = await try_fn_default(async () => {
|
|
7252
|
+
const metaContent = await readFile$1(metadataPath, this.encoding);
|
|
7253
|
+
return JSON.parse(metaContent);
|
|
7254
|
+
});
|
|
7255
|
+
if (ok && metadata.ttl > 0) {
|
|
7256
|
+
const age = now - metadata.timestamp;
|
|
7257
|
+
shouldDelete = age > metadata.ttl;
|
|
7258
|
+
}
|
|
7259
|
+
}
|
|
7260
|
+
} else {
|
|
7261
|
+
const [ok, err, stats] = await try_fn_default(async () => {
|
|
7262
|
+
return await stat$1(filePath);
|
|
7263
|
+
});
|
|
7264
|
+
if (ok) {
|
|
7265
|
+
const age = now - stats.mtime.getTime();
|
|
7266
|
+
shouldDelete = age > this.ttl;
|
|
7267
|
+
}
|
|
7268
|
+
}
|
|
7269
|
+
if (shouldDelete) {
|
|
7270
|
+
const keyPart = file.slice(this.prefix.length + 1, -this.fileExtension.length);
|
|
7271
|
+
await this._del(keyPart);
|
|
7272
|
+
}
|
|
7273
|
+
}
|
|
7274
|
+
} catch (error) {
|
|
7275
|
+
console.warn("FilesystemCache cleanup error:", error.message);
|
|
7276
|
+
}
|
|
7277
|
+
}
|
|
7278
|
+
async _acquireLock(filePath) {
|
|
7279
|
+
if (!this.enableLocking) return;
|
|
7280
|
+
const lockKey = filePath;
|
|
7281
|
+
const startTime = Date.now();
|
|
7282
|
+
while (this.locks.has(lockKey)) {
|
|
7283
|
+
if (Date.now() - startTime > this.lockTimeout) {
|
|
7284
|
+
throw new Error(`Lock timeout for file: ${filePath}`);
|
|
7285
|
+
}
|
|
7286
|
+
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
7287
|
+
}
|
|
7288
|
+
this.locks.set(lockKey, Date.now());
|
|
7289
|
+
}
|
|
7290
|
+
_releaseLock(filePath) {
|
|
7291
|
+
if (!this.enableLocking) return;
|
|
7292
|
+
this.locks.delete(filePath);
|
|
7293
|
+
}
|
|
7294
|
+
async _journalOperation(operation, key, metadata = {}) {
|
|
7295
|
+
if (!this.enableJournal) return;
|
|
7296
|
+
const entry = {
|
|
7297
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
7298
|
+
operation,
|
|
7299
|
+
key,
|
|
7300
|
+
metadata
|
|
7301
|
+
};
|
|
7302
|
+
const [ok, err] = await try_fn_default(async () => {
|
|
7303
|
+
const line = JSON.stringify(entry) + "\n";
|
|
7304
|
+
await fs.promises.appendFile(this.journalFile, line, this.encoding);
|
|
7305
|
+
});
|
|
7306
|
+
if (!ok) {
|
|
7307
|
+
console.warn("FilesystemCache journal error:", err.message);
|
|
7308
|
+
}
|
|
7309
|
+
}
|
|
7310
|
+
// Cleanup on process exit
|
|
7311
|
+
destroy() {
|
|
7312
|
+
if (this.cleanupTimer) {
|
|
7313
|
+
clearInterval(this.cleanupTimer);
|
|
7314
|
+
this.cleanupTimer = null;
|
|
7315
|
+
}
|
|
7316
|
+
}
|
|
7317
|
+
// Get cache statistics
|
|
7318
|
+
getStats() {
|
|
7319
|
+
return {
|
|
7320
|
+
...this.stats,
|
|
7321
|
+
directory: this.directory,
|
|
7322
|
+
ttl: this.ttl,
|
|
7323
|
+
compression: this.enableCompression,
|
|
7324
|
+
metadata: this.enableMetadata,
|
|
7325
|
+
cleanup: this.enableCleanup,
|
|
7326
|
+
locking: this.enableLocking,
|
|
7327
|
+
journal: this.enableJournal
|
|
7328
|
+
};
|
|
7329
|
+
}
|
|
7330
|
+
}
|
|
7331
|
+
|
|
7332
|
+
promisify(fs.mkdir);
|
|
7333
|
+
const rmdir = promisify(fs.rm);
|
|
7334
|
+
const readdir = promisify(fs.readdir);
|
|
7335
|
+
const stat = promisify(fs.stat);
|
|
7336
|
+
const writeFile = promisify(fs.writeFile);
|
|
7337
|
+
const readFile = promisify(fs.readFile);
|
|
7338
|
+
class PartitionAwareFilesystemCache extends FilesystemCache {
|
|
7339
|
+
constructor({
|
|
7340
|
+
partitionStrategy = "hierarchical",
|
|
7341
|
+
// 'hierarchical', 'flat', 'temporal'
|
|
7342
|
+
trackUsage = true,
|
|
7343
|
+
preloadRelated = false,
|
|
7344
|
+
preloadThreshold = 10,
|
|
7345
|
+
maxCacheSize = null,
|
|
7346
|
+
usageStatsFile = "partition-usage.json",
|
|
7347
|
+
...config
|
|
7348
|
+
}) {
|
|
7349
|
+
super(config);
|
|
7350
|
+
this.partitionStrategy = partitionStrategy;
|
|
7351
|
+
this.trackUsage = trackUsage;
|
|
7352
|
+
this.preloadRelated = preloadRelated;
|
|
7353
|
+
this.preloadThreshold = preloadThreshold;
|
|
7354
|
+
this.maxCacheSize = maxCacheSize;
|
|
7355
|
+
this.usageStatsFile = path.join(this.directory, usageStatsFile);
|
|
7356
|
+
this.partitionUsage = /* @__PURE__ */ new Map();
|
|
7357
|
+
this.loadUsageStats();
|
|
7358
|
+
}
|
|
7359
|
+
/**
|
|
7360
|
+
* Generate partition-aware cache key
|
|
7361
|
+
*/
|
|
7362
|
+
_getPartitionCacheKey(resource, action, partition, partitionValues = {}, params = {}) {
|
|
7363
|
+
const keyParts = [`resource=${resource}`, `action=${action}`];
|
|
7364
|
+
if (partition && Object.keys(partitionValues).length > 0) {
|
|
7365
|
+
keyParts.push(`partition=${partition}`);
|
|
7366
|
+
const sortedFields = Object.entries(partitionValues).sort(([a], [b]) => a.localeCompare(b));
|
|
7367
|
+
for (const [field, value] of sortedFields) {
|
|
7368
|
+
if (value !== null && value !== void 0) {
|
|
7369
|
+
keyParts.push(`${field}=${value}`);
|
|
7370
|
+
}
|
|
7371
|
+
}
|
|
7372
|
+
}
|
|
7373
|
+
if (Object.keys(params).length > 0) {
|
|
7374
|
+
const paramsStr = Object.entries(params).sort(([a], [b]) => a.localeCompare(b)).map(([k, v]) => `${k}=${v}`).join("|");
|
|
7375
|
+
keyParts.push(`params=${Buffer.from(paramsStr).toString("base64")}`);
|
|
7376
|
+
}
|
|
7377
|
+
return keyParts.join("/") + this.fileExtension;
|
|
7378
|
+
}
|
|
7379
|
+
/**
|
|
7380
|
+
* Get directory path for partition cache
|
|
7381
|
+
*/
|
|
7382
|
+
_getPartitionDirectory(resource, partition, partitionValues = {}) {
|
|
7383
|
+
const basePath = path.join(this.directory, `resource=${resource}`);
|
|
7384
|
+
if (!partition) {
|
|
7385
|
+
return basePath;
|
|
7386
|
+
}
|
|
7387
|
+
if (this.partitionStrategy === "flat") {
|
|
7388
|
+
return path.join(basePath, "partitions");
|
|
7389
|
+
}
|
|
7390
|
+
if (this.partitionStrategy === "temporal" && this._isTemporalPartition(partition, partitionValues)) {
|
|
7391
|
+
return this._getTemporalDirectory(basePath, partition, partitionValues);
|
|
7392
|
+
}
|
|
7393
|
+
const pathParts = [basePath, `partition=${partition}`];
|
|
7394
|
+
const sortedFields = Object.entries(partitionValues).sort(([a], [b]) => a.localeCompare(b));
|
|
7395
|
+
for (const [field, value] of sortedFields) {
|
|
7396
|
+
if (value !== null && value !== void 0) {
|
|
7397
|
+
pathParts.push(`${field}=${this._sanitizePathValue(value)}`);
|
|
7398
|
+
}
|
|
7399
|
+
}
|
|
7400
|
+
return path.join(...pathParts);
|
|
7401
|
+
}
|
|
7402
|
+
/**
|
|
7403
|
+
* Enhanced set method with partition awareness
|
|
7404
|
+
*/
|
|
7405
|
+
async _set(key, data, options = {}) {
|
|
7406
|
+
const { resource, action, partition, partitionValues, params } = options;
|
|
7407
|
+
if (resource && partition) {
|
|
7408
|
+
const partitionKey = this._getPartitionCacheKey(resource, action, partition, partitionValues, params);
|
|
7409
|
+
const partitionDir = this._getPartitionDirectory(resource, partition, partitionValues);
|
|
7410
|
+
await this._ensureDirectory(partitionDir);
|
|
7411
|
+
const filePath = path.join(partitionDir, this._sanitizeFileName(partitionKey));
|
|
7412
|
+
if (this.trackUsage) {
|
|
7413
|
+
await this._trackPartitionUsage(resource, partition, partitionValues);
|
|
7414
|
+
}
|
|
7415
|
+
const partitionData = {
|
|
7416
|
+
data,
|
|
7417
|
+
metadata: {
|
|
7418
|
+
resource,
|
|
7419
|
+
partition,
|
|
7420
|
+
partitionValues,
|
|
7421
|
+
timestamp: Date.now(),
|
|
7422
|
+
ttl: this.ttl
|
|
7423
|
+
}
|
|
7424
|
+
};
|
|
7425
|
+
return this._writeFileWithMetadata(filePath, partitionData);
|
|
7426
|
+
}
|
|
7427
|
+
return super._set(key, data);
|
|
7428
|
+
}
|
|
7429
|
+
/**
|
|
7430
|
+
* Enhanced get method with partition awareness
|
|
7431
|
+
*/
|
|
7432
|
+
async _get(key, options = {}) {
|
|
7433
|
+
const { resource, action, partition, partitionValues, params } = options;
|
|
7434
|
+
if (resource && partition) {
|
|
7435
|
+
const partitionKey = this._getPartitionCacheKey(resource, action, partition, partitionValues, params);
|
|
7436
|
+
const partitionDir = this._getPartitionDirectory(resource, partition, partitionValues);
|
|
7437
|
+
const filePath = path.join(partitionDir, this._sanitizeFileName(partitionKey));
|
|
7438
|
+
if (!await this._fileExists(filePath)) {
|
|
7439
|
+
if (this.preloadRelated) {
|
|
7440
|
+
await this._preloadRelatedPartitions(resource, partition, partitionValues);
|
|
7441
|
+
}
|
|
7442
|
+
return null;
|
|
7443
|
+
}
|
|
7444
|
+
const result = await this._readFileWithMetadata(filePath);
|
|
7445
|
+
if (result && this.trackUsage) {
|
|
7446
|
+
await this._trackPartitionUsage(resource, partition, partitionValues);
|
|
7447
|
+
}
|
|
7448
|
+
return result?.data || null;
|
|
7449
|
+
}
|
|
7450
|
+
return super._get(key);
|
|
7451
|
+
}
|
|
7452
|
+
/**
|
|
7453
|
+
* Clear cache for specific partition
|
|
7454
|
+
*/
|
|
7455
|
+
async clearPartition(resource, partition, partitionValues = {}) {
|
|
7456
|
+
const partitionDir = this._getPartitionDirectory(resource, partition, partitionValues);
|
|
7457
|
+
const [ok, err] = await try_fn_default(async () => {
|
|
7458
|
+
if (await this._fileExists(partitionDir)) {
|
|
7459
|
+
await rmdir(partitionDir, { recursive: true });
|
|
7460
|
+
}
|
|
7461
|
+
});
|
|
7462
|
+
if (!ok) {
|
|
7463
|
+
console.warn(`Failed to clear partition cache: ${err.message}`);
|
|
7464
|
+
}
|
|
7465
|
+
const usageKey = this._getUsageKey(resource, partition, partitionValues);
|
|
7466
|
+
this.partitionUsage.delete(usageKey);
|
|
7467
|
+
await this._saveUsageStats();
|
|
7468
|
+
return ok;
|
|
7469
|
+
}
|
|
7470
|
+
/**
|
|
7471
|
+
* Clear all partitions for a resource
|
|
7472
|
+
*/
|
|
7473
|
+
async clearResourcePartitions(resource) {
|
|
7474
|
+
const resourceDir = path.join(this.directory, `resource=${resource}`);
|
|
7475
|
+
const [ok, err] = await try_fn_default(async () => {
|
|
7476
|
+
if (await this._fileExists(resourceDir)) {
|
|
7477
|
+
await rmdir(resourceDir, { recursive: true });
|
|
7478
|
+
}
|
|
7479
|
+
});
|
|
7480
|
+
for (const [key] of this.partitionUsage.entries()) {
|
|
7481
|
+
if (key.startsWith(`${resource}/`)) {
|
|
7482
|
+
this.partitionUsage.delete(key);
|
|
7483
|
+
}
|
|
7484
|
+
}
|
|
7485
|
+
await this._saveUsageStats();
|
|
7486
|
+
return ok;
|
|
7487
|
+
}
|
|
7488
|
+
/**
|
|
7489
|
+
* Get partition cache statistics
|
|
7490
|
+
*/
|
|
7491
|
+
async getPartitionStats(resource, partition = null) {
|
|
7492
|
+
const stats = {
|
|
7493
|
+
totalFiles: 0,
|
|
7494
|
+
totalSize: 0,
|
|
7495
|
+
partitions: {},
|
|
7496
|
+
usage: {}
|
|
7497
|
+
};
|
|
7498
|
+
const resourceDir = path.join(this.directory, `resource=${resource}`);
|
|
7499
|
+
if (!await this._fileExists(resourceDir)) {
|
|
7500
|
+
return stats;
|
|
7501
|
+
}
|
|
7502
|
+
await this._calculateDirectoryStats(resourceDir, stats);
|
|
7503
|
+
for (const [key, usage] of this.partitionUsage.entries()) {
|
|
7504
|
+
if (key.startsWith(`${resource}/`)) {
|
|
7505
|
+
const partitionName = key.split("/")[1];
|
|
7506
|
+
if (!partition || partitionName === partition) {
|
|
7507
|
+
stats.usage[partitionName] = usage;
|
|
7508
|
+
}
|
|
7509
|
+
}
|
|
7510
|
+
}
|
|
7511
|
+
return stats;
|
|
7512
|
+
}
|
|
7513
|
+
/**
|
|
7514
|
+
* Get cache recommendations based on usage patterns
|
|
7515
|
+
*/
|
|
7516
|
+
async getCacheRecommendations(resource) {
|
|
7517
|
+
const recommendations = [];
|
|
7518
|
+
const now = Date.now();
|
|
7519
|
+
const dayMs = 24 * 60 * 60 * 1e3;
|
|
7520
|
+
for (const [key, usage] of this.partitionUsage.entries()) {
|
|
7521
|
+
if (key.startsWith(`${resource}/`)) {
|
|
7522
|
+
const [, partition] = key.split("/");
|
|
7523
|
+
const daysSinceLastAccess = (now - usage.lastAccess) / dayMs;
|
|
7524
|
+
const accessesPerDay = usage.count / Math.max(1, daysSinceLastAccess);
|
|
7525
|
+
let recommendation = "keep";
|
|
7526
|
+
let priority = usage.count;
|
|
7527
|
+
if (daysSinceLastAccess > 30) {
|
|
7528
|
+
recommendation = "archive";
|
|
7529
|
+
priority = 0;
|
|
7530
|
+
} else if (accessesPerDay < 0.1) {
|
|
7531
|
+
recommendation = "reduce_ttl";
|
|
7532
|
+
priority = 1;
|
|
7533
|
+
} else if (accessesPerDay > 10) {
|
|
7534
|
+
recommendation = "preload";
|
|
7535
|
+
priority = 100;
|
|
7536
|
+
}
|
|
7537
|
+
recommendations.push({
|
|
7538
|
+
partition,
|
|
7539
|
+
recommendation,
|
|
7540
|
+
priority,
|
|
7541
|
+
usage: accessesPerDay,
|
|
7542
|
+
lastAccess: new Date(usage.lastAccess).toISOString()
|
|
7543
|
+
});
|
|
7544
|
+
}
|
|
7545
|
+
}
|
|
7546
|
+
return recommendations.sort((a, b) => b.priority - a.priority);
|
|
7547
|
+
}
|
|
7548
|
+
/**
|
|
7549
|
+
* Preload frequently accessed partitions
|
|
7550
|
+
*/
|
|
7551
|
+
async warmPartitionCache(resource, options = {}) {
|
|
7552
|
+
const { partitions = [], maxFiles = 1e3 } = options;
|
|
7553
|
+
let warmedCount = 0;
|
|
7554
|
+
for (const partition of partitions) {
|
|
7555
|
+
const usageKey = `${resource}/${partition}`;
|
|
7556
|
+
const usage = this.partitionUsage.get(usageKey);
|
|
7557
|
+
if (usage && usage.count >= this.preloadThreshold) {
|
|
7558
|
+
console.log(`\u{1F525} Warming cache for ${resource}/${partition} (${usage.count} accesses)`);
|
|
7559
|
+
warmedCount++;
|
|
7560
|
+
}
|
|
7561
|
+
if (warmedCount >= maxFiles) break;
|
|
7562
|
+
}
|
|
7563
|
+
return warmedCount;
|
|
7564
|
+
}
|
|
7565
|
+
// Private helper methods
|
|
7566
|
+
async _trackPartitionUsage(resource, partition, partitionValues) {
|
|
7567
|
+
const usageKey = this._getUsageKey(resource, partition, partitionValues);
|
|
7568
|
+
const current = this.partitionUsage.get(usageKey) || {
|
|
7569
|
+
count: 0,
|
|
7570
|
+
firstAccess: Date.now(),
|
|
7571
|
+
lastAccess: Date.now()
|
|
7572
|
+
};
|
|
7573
|
+
current.count++;
|
|
7574
|
+
current.lastAccess = Date.now();
|
|
7575
|
+
this.partitionUsage.set(usageKey, current);
|
|
7576
|
+
if (current.count % 10 === 0) {
|
|
7577
|
+
await this._saveUsageStats();
|
|
7578
|
+
}
|
|
7579
|
+
}
|
|
7580
|
+
_getUsageKey(resource, partition, partitionValues) {
|
|
7581
|
+
const valuePart = Object.entries(partitionValues).sort(([a], [b]) => a.localeCompare(b)).map(([k, v]) => `${k}=${v}`).join("|");
|
|
7582
|
+
return `${resource}/${partition}/${valuePart}`;
|
|
7583
|
+
}
|
|
7584
|
+
async _preloadRelatedPartitions(resource, partition, partitionValues) {
|
|
7585
|
+
console.log(`\u{1F3AF} Preloading related partitions for ${resource}/${partition}`);
|
|
7586
|
+
if (partitionValues.timestamp || partitionValues.date) ;
|
|
7587
|
+
}
|
|
7588
|
+
_isTemporalPartition(partition, partitionValues) {
|
|
7589
|
+
const temporalFields = ["date", "timestamp", "createdAt", "updatedAt"];
|
|
7590
|
+
return Object.keys(partitionValues).some(
|
|
7591
|
+
(field) => temporalFields.some((tf) => field.toLowerCase().includes(tf))
|
|
7592
|
+
);
|
|
7593
|
+
}
|
|
7594
|
+
_getTemporalDirectory(basePath, partition, partitionValues) {
|
|
7595
|
+
const dateValue = Object.values(partitionValues)[0];
|
|
7596
|
+
if (typeof dateValue === "string" && dateValue.match(/^\d{4}-\d{2}-\d{2}/)) {
|
|
7597
|
+
const [year, month, day] = dateValue.split("-");
|
|
7598
|
+
return path.join(basePath, "temporal", year, month, day);
|
|
7599
|
+
}
|
|
7600
|
+
return path.join(basePath, `partition=${partition}`);
|
|
7601
|
+
}
|
|
7602
|
+
_sanitizePathValue(value) {
|
|
7603
|
+
return String(value).replace(/[<>:"/\\|?*]/g, "_");
|
|
7604
|
+
}
|
|
7605
|
+
_sanitizeFileName(filename) {
|
|
7606
|
+
return filename.replace(/[<>:"/\\|?*]/g, "_");
|
|
7607
|
+
}
|
|
7608
|
+
async _calculateDirectoryStats(dir, stats) {
|
|
7609
|
+
const [ok, err, files] = await try_fn_default(() => readdir(dir));
|
|
7610
|
+
if (!ok) return;
|
|
7611
|
+
for (const file of files) {
|
|
7612
|
+
const filePath = path.join(dir, file);
|
|
7613
|
+
const [statOk, statErr, fileStat] = await try_fn_default(() => stat(filePath));
|
|
7614
|
+
if (statOk) {
|
|
7615
|
+
if (fileStat.isDirectory()) {
|
|
7616
|
+
await this._calculateDirectoryStats(filePath, stats);
|
|
7617
|
+
} else {
|
|
7618
|
+
stats.totalFiles++;
|
|
7619
|
+
stats.totalSize += fileStat.size;
|
|
7620
|
+
}
|
|
7621
|
+
}
|
|
7622
|
+
}
|
|
7623
|
+
}
|
|
7624
|
+
async loadUsageStats() {
|
|
7625
|
+
const [ok, err, content] = await try_fn_default(async () => {
|
|
7626
|
+
const data = await readFile(this.usageStatsFile, "utf8");
|
|
7627
|
+
return JSON.parse(data);
|
|
7628
|
+
});
|
|
7629
|
+
if (ok && content) {
|
|
7630
|
+
this.partitionUsage = new Map(Object.entries(content));
|
|
7631
|
+
}
|
|
7632
|
+
}
|
|
7633
|
+
async _saveUsageStats() {
|
|
7634
|
+
const statsObject = Object.fromEntries(this.partitionUsage);
|
|
7635
|
+
await try_fn_default(async () => {
|
|
7636
|
+
await writeFile(
|
|
7637
|
+
this.usageStatsFile,
|
|
7638
|
+
JSON.stringify(statsObject, null, 2),
|
|
7639
|
+
"utf8"
|
|
7640
|
+
);
|
|
7641
|
+
});
|
|
7642
|
+
}
|
|
7643
|
+
async _writeFileWithMetadata(filePath, data) {
|
|
7644
|
+
const content = JSON.stringify(data);
|
|
7645
|
+
const [ok, err] = await try_fn_default(async () => {
|
|
7646
|
+
await writeFile(filePath, content, {
|
|
7647
|
+
encoding: this.encoding,
|
|
7648
|
+
mode: this.fileMode
|
|
7649
|
+
});
|
|
7650
|
+
});
|
|
7651
|
+
if (!ok) {
|
|
7652
|
+
throw new Error(`Failed to write cache file: ${err.message}`);
|
|
7653
|
+
}
|
|
7654
|
+
return true;
|
|
7655
|
+
}
|
|
7656
|
+
async _readFileWithMetadata(filePath) {
|
|
7657
|
+
const [ok, err, content] = await try_fn_default(async () => {
|
|
7658
|
+
return await readFile(filePath, this.encoding);
|
|
7659
|
+
});
|
|
7660
|
+
if (!ok || !content) return null;
|
|
7661
|
+
try {
|
|
7662
|
+
return JSON.parse(content);
|
|
7663
|
+
} catch (error) {
|
|
7664
|
+
return { data: content };
|
|
7665
|
+
}
|
|
7666
|
+
}
|
|
7667
|
+
}
|
|
7668
|
+
|
|
7669
|
+
class CachePlugin extends plugin_class_default {
|
|
7670
|
+
constructor(options = {}) {
|
|
7671
|
+
super(options);
|
|
7672
|
+
this.driver = options.driver;
|
|
7673
|
+
this.config = {
|
|
7674
|
+
includePartitions: options.includePartitions !== false,
|
|
7675
|
+
partitionStrategy: options.partitionStrategy || "hierarchical",
|
|
7676
|
+
partitionAware: options.partitionAware !== false,
|
|
7677
|
+
trackUsage: options.trackUsage !== false,
|
|
7678
|
+
preloadRelated: options.preloadRelated !== false,
|
|
7679
|
+
...options
|
|
7680
|
+
};
|
|
7681
|
+
}
|
|
7682
|
+
async setup(database) {
|
|
7683
|
+
await super.setup(database);
|
|
7684
|
+
}
|
|
7685
|
+
async onSetup() {
|
|
7686
|
+
if (this.config.driver) {
|
|
7687
|
+
this.driver = this.config.driver;
|
|
7688
|
+
} else if (this.config.driverType === "memory") {
|
|
7689
|
+
this.driver = new memory_cache_class_default(this.config.memoryOptions || {});
|
|
7690
|
+
} else if (this.config.driverType === "filesystem") {
|
|
7691
|
+
if (this.config.partitionAware) {
|
|
7692
|
+
this.driver = new PartitionAwareFilesystemCache({
|
|
7693
|
+
partitionStrategy: this.config.partitionStrategy,
|
|
7694
|
+
trackUsage: this.config.trackUsage,
|
|
7695
|
+
preloadRelated: this.config.preloadRelated,
|
|
7696
|
+
...this.config.filesystemOptions
|
|
7697
|
+
});
|
|
7698
|
+
} else {
|
|
7699
|
+
this.driver = new FilesystemCache(this.config.filesystemOptions || {});
|
|
7700
|
+
}
|
|
7701
|
+
} else {
|
|
7702
|
+
this.driver = new s3_cache_class_default({ client: this.database.client, ...this.config.s3Options || {} });
|
|
7703
|
+
}
|
|
7704
|
+
this.installDatabaseProxy();
|
|
7705
|
+
this.installResourceHooks();
|
|
7706
|
+
}
|
|
7707
|
+
async onStart() {
|
|
7708
|
+
}
|
|
7709
|
+
async onStop() {
|
|
7710
|
+
}
|
|
7711
|
+
installDatabaseProxy() {
|
|
7712
|
+
if (this.database._cacheProxyInstalled) {
|
|
7713
|
+
return;
|
|
7714
|
+
}
|
|
7715
|
+
const installResourceHooks = this.installResourceHooks.bind(this);
|
|
7716
|
+
this.database._originalCreateResourceForCache = this.database.createResource;
|
|
7717
|
+
this.database.createResource = async function(...args) {
|
|
7718
|
+
const resource = await this._originalCreateResourceForCache(...args);
|
|
7719
|
+
installResourceHooks(resource);
|
|
7720
|
+
return resource;
|
|
7721
|
+
};
|
|
7722
|
+
this.database._cacheProxyInstalled = true;
|
|
7723
|
+
}
|
|
7724
|
+
installResourceHooks() {
|
|
7725
|
+
for (const resource of Object.values(this.database.resources)) {
|
|
7726
|
+
this.installResourceHooksForResource(resource);
|
|
7727
|
+
}
|
|
7728
|
+
}
|
|
7729
|
+
installResourceHooksForResource(resource) {
|
|
7730
|
+
if (!this.driver) return;
|
|
7731
|
+
Object.defineProperty(resource, "cache", {
|
|
7732
|
+
value: this.driver,
|
|
7733
|
+
writable: true,
|
|
7734
|
+
configurable: true,
|
|
7735
|
+
enumerable: false
|
|
7736
|
+
});
|
|
7737
|
+
resource.cacheKeyFor = async (options = {}) => {
|
|
7738
|
+
const { action, params = {}, partition, partitionValues } = options;
|
|
7739
|
+
return this.generateCacheKey(resource, action, params, partition, partitionValues);
|
|
7740
|
+
};
|
|
7741
|
+
if (this.driver instanceof PartitionAwareFilesystemCache) {
|
|
7742
|
+
resource.clearPartitionCache = async (partition, partitionValues = {}) => {
|
|
7743
|
+
return await this.driver.clearPartition(resource.name, partition, partitionValues);
|
|
7744
|
+
};
|
|
7745
|
+
resource.getPartitionCacheStats = async (partition = null) => {
|
|
7746
|
+
return await this.driver.getPartitionStats(resource.name, partition);
|
|
7747
|
+
};
|
|
7748
|
+
resource.getCacheRecommendations = async () => {
|
|
7749
|
+
return await this.driver.getCacheRecommendations(resource.name);
|
|
7750
|
+
};
|
|
7751
|
+
resource.warmPartitionCache = async (partitions = [], options = {}) => {
|
|
7752
|
+
return await this.driver.warmPartitionCache(resource.name, { partitions, ...options });
|
|
7753
|
+
};
|
|
7754
|
+
}
|
|
7755
|
+
const cacheMethods = [
|
|
7756
|
+
"count",
|
|
7757
|
+
"listIds",
|
|
7758
|
+
"getMany",
|
|
7759
|
+
"getAll",
|
|
7760
|
+
"page",
|
|
7761
|
+
"list",
|
|
7762
|
+
"get"
|
|
7763
|
+
];
|
|
7764
|
+
for (const method of cacheMethods) {
|
|
7765
|
+
resource.useMiddleware(method, async (ctx, next) => {
|
|
7766
|
+
let key;
|
|
7767
|
+
if (method === "getMany") {
|
|
7768
|
+
key = await resource.cacheKeyFor({ action: method, params: { ids: ctx.args[0] } });
|
|
7769
|
+
} else if (method === "page") {
|
|
7770
|
+
const { offset, size, partition, partitionValues } = ctx.args[0] || {};
|
|
7771
|
+
key = await resource.cacheKeyFor({ action: method, params: { offset, size }, partition, partitionValues });
|
|
7772
|
+
} else if (method === "list" || method === "listIds" || method === "count") {
|
|
7773
|
+
const { partition, partitionValues } = ctx.args[0] || {};
|
|
7774
|
+
key = await resource.cacheKeyFor({ action: method, partition, partitionValues });
|
|
7775
|
+
} else if (method === "getAll") {
|
|
7776
|
+
key = await resource.cacheKeyFor({ action: method });
|
|
7777
|
+
} else if (method === "get") {
|
|
7778
|
+
key = await resource.cacheKeyFor({ action: method, params: { id: ctx.args[0] } });
|
|
7779
|
+
}
|
|
7780
|
+
if (this.driver instanceof PartitionAwareFilesystemCache) {
|
|
7781
|
+
let partition, partitionValues;
|
|
7782
|
+
if (method === "list" || method === "listIds" || method === "count" || method === "page") {
|
|
7783
|
+
const args = ctx.args[0] || {};
|
|
7784
|
+
partition = args.partition;
|
|
7785
|
+
partitionValues = args.partitionValues;
|
|
7786
|
+
}
|
|
7787
|
+
const [ok, err, result] = await try_fn_default(() => resource.cache._get(key, {
|
|
7788
|
+
resource: resource.name,
|
|
7789
|
+
action: method,
|
|
7790
|
+
partition,
|
|
7791
|
+
partitionValues
|
|
7792
|
+
}));
|
|
7793
|
+
if (ok && result !== null && result !== void 0) return result;
|
|
7794
|
+
if (!ok && err.name !== "NoSuchKey") throw err;
|
|
7795
|
+
const freshResult = await next();
|
|
7796
|
+
await resource.cache._set(key, freshResult, {
|
|
7797
|
+
resource: resource.name,
|
|
7798
|
+
action: method,
|
|
7799
|
+
partition,
|
|
7800
|
+
partitionValues
|
|
7801
|
+
});
|
|
7802
|
+
return freshResult;
|
|
7803
|
+
} else {
|
|
7804
|
+
const [ok, err, result] = await try_fn_default(() => resource.cache.get(key));
|
|
7805
|
+
if (ok && result !== null && result !== void 0) return result;
|
|
7806
|
+
if (!ok && err.name !== "NoSuchKey") throw err;
|
|
7807
|
+
const freshResult = await next();
|
|
7808
|
+
await resource.cache.set(key, freshResult);
|
|
7809
|
+
return freshResult;
|
|
7810
|
+
}
|
|
7811
|
+
});
|
|
7812
|
+
}
|
|
7813
|
+
const writeMethods = ["insert", "update", "delete", "deleteMany"];
|
|
7814
|
+
for (const method of writeMethods) {
|
|
7815
|
+
resource.useMiddleware(method, async (ctx, next) => {
|
|
7816
|
+
const result = await next();
|
|
7817
|
+
if (method === "insert") {
|
|
7818
|
+
await this.clearCacheForResource(resource, ctx.args[0]);
|
|
7819
|
+
} else if (method === "update") {
|
|
7820
|
+
await this.clearCacheForResource(resource, { id: ctx.args[0], ...ctx.args[1] });
|
|
7821
|
+
} else if (method === "delete") {
|
|
7822
|
+
let data = { id: ctx.args[0] };
|
|
7823
|
+
if (typeof resource.get === "function") {
|
|
7824
|
+
const [ok, err, full] = await try_fn_default(() => resource.get(ctx.args[0]));
|
|
7825
|
+
if (ok && full) data = full;
|
|
7826
|
+
}
|
|
7827
|
+
await this.clearCacheForResource(resource, data);
|
|
7828
|
+
} else if (method === "deleteMany") {
|
|
7829
|
+
await this.clearCacheForResource(resource);
|
|
7830
|
+
}
|
|
7831
|
+
return result;
|
|
7832
|
+
});
|
|
7833
|
+
}
|
|
7834
|
+
}
|
|
7835
|
+
async clearCacheForResource(resource, data) {
|
|
7836
|
+
if (!resource.cache) return;
|
|
7837
|
+
const keyPrefix = `resource=${resource.name}`;
|
|
7838
|
+
await resource.cache.clear(keyPrefix);
|
|
7839
|
+
if (this.config.includePartitions === true && resource.config?.partitions && Object.keys(resource.config.partitions).length > 0) {
|
|
7840
|
+
if (!data) {
|
|
7841
|
+
for (const partitionName of Object.keys(resource.config.partitions)) {
|
|
7842
|
+
const partitionKeyPrefix = join(keyPrefix, `partition=${partitionName}`);
|
|
7843
|
+
await resource.cache.clear(partitionKeyPrefix);
|
|
7844
|
+
}
|
|
7845
|
+
} else {
|
|
7846
|
+
const partitionValues = this.getPartitionValues(data, resource);
|
|
7847
|
+
for (const [partitionName, values] of Object.entries(partitionValues)) {
|
|
7848
|
+
if (values && Object.keys(values).length > 0 && Object.values(values).some((v) => v !== null && v !== void 0)) {
|
|
7849
|
+
const partitionKeyPrefix = join(keyPrefix, `partition=${partitionName}`);
|
|
7850
|
+
await resource.cache.clear(partitionKeyPrefix);
|
|
7851
|
+
}
|
|
7852
|
+
}
|
|
7853
|
+
}
|
|
7854
|
+
}
|
|
7855
|
+
}
|
|
7856
|
+
async generateCacheKey(resource, action, params = {}, partition = null, partitionValues = null) {
|
|
7857
|
+
const keyParts = [
|
|
7858
|
+
`resource=${resource.name}`,
|
|
7859
|
+
`action=${action}`
|
|
7860
|
+
];
|
|
7861
|
+
if (partition && partitionValues && Object.keys(partitionValues).length > 0) {
|
|
7862
|
+
keyParts.push(`partition:${partition}`);
|
|
7863
|
+
for (const [field, value] of Object.entries(partitionValues)) {
|
|
7864
|
+
if (value !== null && value !== void 0) {
|
|
7865
|
+
keyParts.push(`${field}:${value}`);
|
|
7866
|
+
}
|
|
7867
|
+
}
|
|
7868
|
+
}
|
|
7869
|
+
if (Object.keys(params).length > 0) {
|
|
7870
|
+
const paramsHash = await this.hashParams(params);
|
|
7871
|
+
keyParts.push(paramsHash);
|
|
7872
|
+
}
|
|
7873
|
+
return join(...keyParts) + ".json.gz";
|
|
7874
|
+
}
|
|
7875
|
+
async hashParams(params) {
|
|
7876
|
+
const sortedParams = Object.keys(params).sort().map((key) => `${key}:${params[key]}`).join("|") || "empty";
|
|
7877
|
+
return await sha256(sortedParams);
|
|
7878
|
+
}
|
|
7879
|
+
// Utility methods
|
|
7880
|
+
async getCacheStats() {
|
|
7881
|
+
if (!this.driver) return null;
|
|
7882
|
+
return {
|
|
7883
|
+
size: await this.driver.size(),
|
|
7884
|
+
keys: await this.driver.keys(),
|
|
7885
|
+
driver: this.driver.constructor.name
|
|
7886
|
+
};
|
|
7887
|
+
}
|
|
7888
|
+
async clearAllCache() {
|
|
7889
|
+
if (!this.driver) return;
|
|
7890
|
+
for (const resource of Object.values(this.database.resources)) {
|
|
7891
|
+
if (resource.cache) {
|
|
7892
|
+
const keyPrefix = `resource=${resource.name}`;
|
|
7893
|
+
await resource.cache.clear(keyPrefix);
|
|
7894
|
+
}
|
|
7895
|
+
}
|
|
7896
|
+
}
|
|
7897
|
+
async warmCache(resourceName, options = {}) {
|
|
7898
|
+
const resource = this.database.resources[resourceName];
|
|
7899
|
+
if (!resource) {
|
|
7900
|
+
throw new Error(`Resource '${resourceName}' not found`);
|
|
7901
|
+
}
|
|
7902
|
+
const { includePartitions = true } = options;
|
|
7903
|
+
if (this.driver instanceof PartitionAwareFilesystemCache && resource.warmPartitionCache) {
|
|
7904
|
+
const partitionNames = resource.config.partitions ? Object.keys(resource.config.partitions) : [];
|
|
7905
|
+
return await resource.warmPartitionCache(partitionNames, options);
|
|
7906
|
+
}
|
|
6995
7907
|
await resource.getAll();
|
|
6996
7908
|
if (includePartitions && resource.config.partitions) {
|
|
6997
7909
|
for (const [partitionName, partitionDef] of Object.entries(resource.config.partitions)) {
|
|
@@ -7013,6 +7925,57 @@ class CachePlugin extends plugin_class_default {
|
|
|
7013
7925
|
}
|
|
7014
7926
|
}
|
|
7015
7927
|
}
|
|
7928
|
+
// Partition-specific methods
|
|
7929
|
+
async getPartitionCacheStats(resourceName, partition = null) {
|
|
7930
|
+
if (!(this.driver instanceof PartitionAwareFilesystemCache)) {
|
|
7931
|
+
throw new Error("Partition cache statistics are only available with PartitionAwareFilesystemCache");
|
|
7932
|
+
}
|
|
7933
|
+
return await this.driver.getPartitionStats(resourceName, partition);
|
|
7934
|
+
}
|
|
7935
|
+
async getCacheRecommendations(resourceName) {
|
|
7936
|
+
if (!(this.driver instanceof PartitionAwareFilesystemCache)) {
|
|
7937
|
+
throw new Error("Cache recommendations are only available with PartitionAwareFilesystemCache");
|
|
7938
|
+
}
|
|
7939
|
+
return await this.driver.getCacheRecommendations(resourceName);
|
|
7940
|
+
}
|
|
7941
|
+
async clearPartitionCache(resourceName, partition, partitionValues = {}) {
|
|
7942
|
+
if (!(this.driver instanceof PartitionAwareFilesystemCache)) {
|
|
7943
|
+
throw new Error("Partition cache clearing is only available with PartitionAwareFilesystemCache");
|
|
7944
|
+
}
|
|
7945
|
+
return await this.driver.clearPartition(resourceName, partition, partitionValues);
|
|
7946
|
+
}
|
|
7947
|
+
async analyzeCacheUsage() {
|
|
7948
|
+
if (!(this.driver instanceof PartitionAwareFilesystemCache)) {
|
|
7949
|
+
return { message: "Cache usage analysis is only available with PartitionAwareFilesystemCache" };
|
|
7950
|
+
}
|
|
7951
|
+
const analysis = {
|
|
7952
|
+
totalResources: Object.keys(this.database.resources).length,
|
|
7953
|
+
resourceStats: {},
|
|
7954
|
+
recommendations: {},
|
|
7955
|
+
summary: {
|
|
7956
|
+
mostUsedPartitions: [],
|
|
7957
|
+
leastUsedPartitions: [],
|
|
7958
|
+
suggestedOptimizations: []
|
|
7959
|
+
}
|
|
7960
|
+
};
|
|
7961
|
+
for (const [resourceName, resource] of Object.entries(this.database.resources)) {
|
|
7962
|
+
try {
|
|
7963
|
+
analysis.resourceStats[resourceName] = await this.driver.getPartitionStats(resourceName);
|
|
7964
|
+
analysis.recommendations[resourceName] = await this.driver.getCacheRecommendations(resourceName);
|
|
7965
|
+
} catch (error) {
|
|
7966
|
+
analysis.resourceStats[resourceName] = { error: error.message };
|
|
7967
|
+
}
|
|
7968
|
+
}
|
|
7969
|
+
const allRecommendations = Object.values(analysis.recommendations).flat();
|
|
7970
|
+
analysis.summary.mostUsedPartitions = allRecommendations.filter((r) => r.recommendation === "preload").sort((a, b) => b.priority - a.priority).slice(0, 5);
|
|
7971
|
+
analysis.summary.leastUsedPartitions = allRecommendations.filter((r) => r.recommendation === "archive").slice(0, 5);
|
|
7972
|
+
analysis.summary.suggestedOptimizations = [
|
|
7973
|
+
`Consider preloading ${analysis.summary.mostUsedPartitions.length} high-usage partitions`,
|
|
7974
|
+
`Archive ${analysis.summary.leastUsedPartitions.length} unused partitions`,
|
|
7975
|
+
`Monitor cache hit rates for partition efficiency`
|
|
7976
|
+
];
|
|
7977
|
+
return analysis;
|
|
7978
|
+
}
|
|
7016
7979
|
}
|
|
7017
7980
|
|
|
7018
7981
|
const CostsPlugin = {
|
|
@@ -7189,24 +8152,29 @@ class FullTextPlugin extends plugin_class_default {
|
|
|
7189
8152
|
resource._deleteMany = resource.deleteMany;
|
|
7190
8153
|
this.wrapResourceMethod(resource, "insert", async (result, args, methodName) => {
|
|
7191
8154
|
const [data] = args;
|
|
7192
|
-
this.indexRecord(resource.name, result.id, data).catch(
|
|
8155
|
+
this.indexRecord(resource.name, result.id, data).catch(() => {
|
|
8156
|
+
});
|
|
7193
8157
|
return result;
|
|
7194
8158
|
});
|
|
7195
8159
|
this.wrapResourceMethod(resource, "update", async (result, args, methodName) => {
|
|
7196
8160
|
const [id, data] = args;
|
|
7197
|
-
this.removeRecordFromIndex(resource.name, id).catch(
|
|
7198
|
-
|
|
8161
|
+
this.removeRecordFromIndex(resource.name, id).catch(() => {
|
|
8162
|
+
});
|
|
8163
|
+
this.indexRecord(resource.name, id, result).catch(() => {
|
|
8164
|
+
});
|
|
7199
8165
|
return result;
|
|
7200
8166
|
});
|
|
7201
8167
|
this.wrapResourceMethod(resource, "delete", async (result, args, methodName) => {
|
|
7202
8168
|
const [id] = args;
|
|
7203
|
-
this.removeRecordFromIndex(resource.name, id).catch(
|
|
8169
|
+
this.removeRecordFromIndex(resource.name, id).catch(() => {
|
|
8170
|
+
});
|
|
7204
8171
|
return result;
|
|
7205
8172
|
});
|
|
7206
8173
|
this.wrapResourceMethod(resource, "deleteMany", async (result, args, methodName) => {
|
|
7207
8174
|
const [ids] = args;
|
|
7208
8175
|
for (const id of ids) {
|
|
7209
|
-
this.removeRecordFromIndex(resource.name, id).catch(
|
|
8176
|
+
this.removeRecordFromIndex(resource.name, id).catch(() => {
|
|
8177
|
+
});
|
|
7210
8178
|
}
|
|
7211
8179
|
return result;
|
|
7212
8180
|
});
|
|
@@ -7696,7 +8664,8 @@ class MetricsPlugin extends plugin_class_default {
|
|
|
7696
8664
|
}
|
|
7697
8665
|
if (this.config.flushInterval > 0) {
|
|
7698
8666
|
this.flushTimer = setInterval(() => {
|
|
7699
|
-
this.flushMetrics().catch(
|
|
8667
|
+
this.flushMetrics().catch(() => {
|
|
8668
|
+
});
|
|
7700
8669
|
}, this.config.flushInterval);
|
|
7701
8670
|
}
|
|
7702
8671
|
}
|
|
@@ -7768,9 +8737,6 @@ class MetricsPlugin extends plugin_class_default {
|
|
|
7768
8737
|
}
|
|
7769
8738
|
this.resetMetrics();
|
|
7770
8739
|
});
|
|
7771
|
-
if (!ok) {
|
|
7772
|
-
console.error("Failed to flush metrics:", err);
|
|
7773
|
-
}
|
|
7774
8740
|
}
|
|
7775
8741
|
resetMetrics() {
|
|
7776
8742
|
for (const operation of Object.keys(this.metrics.operations)) {
|
|
@@ -7902,20 +8868,628 @@ class MetricsPlugin extends plugin_class_default {
|
|
|
7902
8868
|
await this.metricsResource.delete(metric.id);
|
|
7903
8869
|
}
|
|
7904
8870
|
}
|
|
7905
|
-
if (this.errorsResource) {
|
|
7906
|
-
const oldErrors = await this.getErrorLogs({ endDate: cutoffDate.toISOString() });
|
|
7907
|
-
for (const error of oldErrors) {
|
|
7908
|
-
await this.errorsResource.delete(error.id);
|
|
7909
|
-
}
|
|
8871
|
+
if (this.errorsResource) {
|
|
8872
|
+
const oldErrors = await this.getErrorLogs({ endDate: cutoffDate.toISOString() });
|
|
8873
|
+
for (const error of oldErrors) {
|
|
8874
|
+
await this.errorsResource.delete(error.id);
|
|
8875
|
+
}
|
|
8876
|
+
}
|
|
8877
|
+
if (this.performanceResource) {
|
|
8878
|
+
const oldPerformance = await this.getPerformanceLogs({ endDate: cutoffDate.toISOString() });
|
|
8879
|
+
for (const perf of oldPerformance) {
|
|
8880
|
+
await this.performanceResource.delete(perf.id);
|
|
8881
|
+
}
|
|
8882
|
+
}
|
|
8883
|
+
}
|
|
8884
|
+
}
|
|
8885
|
+
|
|
8886
|
+
class BaseReplicator extends EventEmitter {
|
|
8887
|
+
constructor(config = {}) {
|
|
8888
|
+
super();
|
|
8889
|
+
this.config = config;
|
|
8890
|
+
this.name = this.constructor.name;
|
|
8891
|
+
this.enabled = config.enabled !== false;
|
|
8892
|
+
}
|
|
8893
|
+
/**
|
|
8894
|
+
* Initialize the replicator
|
|
8895
|
+
* @param {Object} database - The s3db database instance
|
|
8896
|
+
* @returns {Promise<void>}
|
|
8897
|
+
*/
|
|
8898
|
+
async initialize(database) {
|
|
8899
|
+
this.database = database;
|
|
8900
|
+
this.emit("initialized", { replicator: this.name });
|
|
8901
|
+
}
|
|
8902
|
+
/**
|
|
8903
|
+
* Replicate data to the target
|
|
8904
|
+
* @param {string} resourceName - Name of the resource being replicated
|
|
8905
|
+
* @param {string} operation - Operation type (insert, update, delete)
|
|
8906
|
+
* @param {Object} data - The data to replicate
|
|
8907
|
+
* @param {string} id - Record ID
|
|
8908
|
+
* @returns {Promise<Object>} replicator result
|
|
8909
|
+
*/
|
|
8910
|
+
async replicate(resourceName, operation, data, id) {
|
|
8911
|
+
throw new Error(`replicate() method must be implemented by ${this.name}`);
|
|
8912
|
+
}
|
|
8913
|
+
/**
|
|
8914
|
+
* Replicate multiple records in batch
|
|
8915
|
+
* @param {string} resourceName - Name of the resource being replicated
|
|
8916
|
+
* @param {Array} records - Array of records to replicate
|
|
8917
|
+
* @returns {Promise<Object>} Batch replicator result
|
|
8918
|
+
*/
|
|
8919
|
+
async replicateBatch(resourceName, records) {
|
|
8920
|
+
throw new Error(`replicateBatch() method must be implemented by ${this.name}`);
|
|
8921
|
+
}
|
|
8922
|
+
/**
|
|
8923
|
+
* Test the connection to the target
|
|
8924
|
+
* @returns {Promise<boolean>} True if connection is successful
|
|
8925
|
+
*/
|
|
8926
|
+
async testConnection() {
|
|
8927
|
+
throw new Error(`testConnection() method must be implemented by ${this.name}`);
|
|
8928
|
+
}
|
|
8929
|
+
/**
|
|
8930
|
+
* Get replicator status and statistics
|
|
8931
|
+
* @returns {Promise<Object>} Status information
|
|
8932
|
+
*/
|
|
8933
|
+
async getStatus() {
|
|
8934
|
+
return {
|
|
8935
|
+
name: this.name,
|
|
8936
|
+
// Removed: enabled: this.enabled,
|
|
8937
|
+
config: this.config,
|
|
8938
|
+
connected: false
|
|
8939
|
+
};
|
|
8940
|
+
}
|
|
8941
|
+
/**
|
|
8942
|
+
* Cleanup resources
|
|
8943
|
+
* @returns {Promise<void>}
|
|
8944
|
+
*/
|
|
8945
|
+
async cleanup() {
|
|
8946
|
+
this.emit("cleanup", { replicator: this.name });
|
|
8947
|
+
}
|
|
8948
|
+
/**
|
|
8949
|
+
* Validate replicator configuration
|
|
8950
|
+
* @returns {Object} Validation result
|
|
8951
|
+
*/
|
|
8952
|
+
validateConfig() {
|
|
8953
|
+
return { isValid: true, errors: [] };
|
|
8954
|
+
}
|
|
8955
|
+
}
|
|
8956
|
+
var base_replicator_class_default = BaseReplicator;
|
|
8957
|
+
|
|
8958
|
+
class BigqueryReplicator extends base_replicator_class_default {
|
|
8959
|
+
constructor(config = {}, resources = {}) {
|
|
8960
|
+
super(config);
|
|
8961
|
+
this.projectId = config.projectId;
|
|
8962
|
+
this.datasetId = config.datasetId;
|
|
8963
|
+
this.bigqueryClient = null;
|
|
8964
|
+
this.credentials = config.credentials;
|
|
8965
|
+
this.location = config.location || "US";
|
|
8966
|
+
this.logTable = config.logTable;
|
|
8967
|
+
this.resources = this.parseResourcesConfig(resources);
|
|
8968
|
+
}
|
|
8969
|
+
parseResourcesConfig(resources) {
|
|
8970
|
+
const parsed = {};
|
|
8971
|
+
for (const [resourceName, config] of Object.entries(resources)) {
|
|
8972
|
+
if (typeof config === "string") {
|
|
8973
|
+
parsed[resourceName] = [{
|
|
8974
|
+
table: config,
|
|
8975
|
+
actions: ["insert"],
|
|
8976
|
+
transform: null
|
|
8977
|
+
}];
|
|
8978
|
+
} else if (Array.isArray(config)) {
|
|
8979
|
+
parsed[resourceName] = config.map((item) => {
|
|
8980
|
+
if (typeof item === "string") {
|
|
8981
|
+
return { table: item, actions: ["insert"], transform: null };
|
|
8982
|
+
}
|
|
8983
|
+
return {
|
|
8984
|
+
table: item.table,
|
|
8985
|
+
actions: item.actions || ["insert"],
|
|
8986
|
+
transform: item.transform || null
|
|
8987
|
+
};
|
|
8988
|
+
});
|
|
8989
|
+
} else if (typeof config === "object") {
|
|
8990
|
+
parsed[resourceName] = [{
|
|
8991
|
+
table: config.table,
|
|
8992
|
+
actions: config.actions || ["insert"],
|
|
8993
|
+
transform: config.transform || null
|
|
8994
|
+
}];
|
|
8995
|
+
}
|
|
8996
|
+
}
|
|
8997
|
+
return parsed;
|
|
8998
|
+
}
|
|
8999
|
+
validateConfig() {
|
|
9000
|
+
const errors = [];
|
|
9001
|
+
if (!this.projectId) errors.push("projectId is required");
|
|
9002
|
+
if (!this.datasetId) errors.push("datasetId is required");
|
|
9003
|
+
if (Object.keys(this.resources).length === 0) errors.push("At least one resource must be configured");
|
|
9004
|
+
for (const [resourceName, tables] of Object.entries(this.resources)) {
|
|
9005
|
+
for (const tableConfig of tables) {
|
|
9006
|
+
if (!tableConfig.table) {
|
|
9007
|
+
errors.push(`Table name is required for resource '${resourceName}'`);
|
|
9008
|
+
}
|
|
9009
|
+
if (!Array.isArray(tableConfig.actions) || tableConfig.actions.length === 0) {
|
|
9010
|
+
errors.push(`Actions array is required for resource '${resourceName}'`);
|
|
9011
|
+
}
|
|
9012
|
+
const validActions = ["insert", "update", "delete"];
|
|
9013
|
+
const invalidActions = tableConfig.actions.filter((action) => !validActions.includes(action));
|
|
9014
|
+
if (invalidActions.length > 0) {
|
|
9015
|
+
errors.push(`Invalid actions for resource '${resourceName}': ${invalidActions.join(", ")}. Valid actions: ${validActions.join(", ")}`);
|
|
9016
|
+
}
|
|
9017
|
+
if (tableConfig.transform && typeof tableConfig.transform !== "function") {
|
|
9018
|
+
errors.push(`Transform must be a function for resource '${resourceName}'`);
|
|
9019
|
+
}
|
|
9020
|
+
}
|
|
9021
|
+
}
|
|
9022
|
+
return { isValid: errors.length === 0, errors };
|
|
9023
|
+
}
|
|
9024
|
+
async initialize(database) {
|
|
9025
|
+
await super.initialize(database);
|
|
9026
|
+
const [ok, err, sdk] = await try_fn_default(() => import('@google-cloud/bigquery'));
|
|
9027
|
+
if (!ok) {
|
|
9028
|
+
this.emit("initialization_error", { replicator: this.name, error: err.message });
|
|
9029
|
+
throw err;
|
|
9030
|
+
}
|
|
9031
|
+
const { BigQuery } = sdk;
|
|
9032
|
+
this.bigqueryClient = new BigQuery({
|
|
9033
|
+
projectId: this.projectId,
|
|
9034
|
+
credentials: this.credentials,
|
|
9035
|
+
location: this.location
|
|
9036
|
+
});
|
|
9037
|
+
this.emit("initialized", {
|
|
9038
|
+
replicator: this.name,
|
|
9039
|
+
projectId: this.projectId,
|
|
9040
|
+
datasetId: this.datasetId,
|
|
9041
|
+
resources: Object.keys(this.resources)
|
|
9042
|
+
});
|
|
9043
|
+
}
|
|
9044
|
+
shouldReplicateResource(resourceName) {
|
|
9045
|
+
return this.resources.hasOwnProperty(resourceName);
|
|
9046
|
+
}
|
|
9047
|
+
shouldReplicateAction(resourceName, operation) {
|
|
9048
|
+
if (!this.resources[resourceName]) return false;
|
|
9049
|
+
return this.resources[resourceName].some(
|
|
9050
|
+
(tableConfig) => tableConfig.actions.includes(operation)
|
|
9051
|
+
);
|
|
9052
|
+
}
|
|
9053
|
+
getTablesForResource(resourceName, operation) {
|
|
9054
|
+
if (!this.resources[resourceName]) return [];
|
|
9055
|
+
return this.resources[resourceName].filter((tableConfig) => tableConfig.actions.includes(operation)).map((tableConfig) => ({
|
|
9056
|
+
table: tableConfig.table,
|
|
9057
|
+
transform: tableConfig.transform
|
|
9058
|
+
}));
|
|
9059
|
+
}
|
|
9060
|
+
applyTransform(data, transformFn) {
|
|
9061
|
+
if (!transformFn) return data;
|
|
9062
|
+
let transformedData = JSON.parse(JSON.stringify(data));
|
|
9063
|
+
if (transformedData._length) delete transformedData._length;
|
|
9064
|
+
return transformFn(transformedData);
|
|
9065
|
+
}
|
|
9066
|
+
async replicate(resourceName, operation, data, id, beforeData = null) {
|
|
9067
|
+
if (!this.enabled || !this.shouldReplicateResource(resourceName)) {
|
|
9068
|
+
return { skipped: true, reason: "resource_not_included" };
|
|
9069
|
+
}
|
|
9070
|
+
if (!this.shouldReplicateAction(resourceName, operation)) {
|
|
9071
|
+
return { skipped: true, reason: "action_not_included" };
|
|
9072
|
+
}
|
|
9073
|
+
const tableConfigs = this.getTablesForResource(resourceName, operation);
|
|
9074
|
+
if (tableConfigs.length === 0) {
|
|
9075
|
+
return { skipped: true, reason: "no_tables_for_action" };
|
|
9076
|
+
}
|
|
9077
|
+
const results = [];
|
|
9078
|
+
const errors = [];
|
|
9079
|
+
const [ok, err, result] = await try_fn_default(async () => {
|
|
9080
|
+
const dataset = this.bigqueryClient.dataset(this.datasetId);
|
|
9081
|
+
for (const tableConfig of tableConfigs) {
|
|
9082
|
+
const [okTable, errTable] = await try_fn_default(async () => {
|
|
9083
|
+
const table = dataset.table(tableConfig.table);
|
|
9084
|
+
let job;
|
|
9085
|
+
if (operation === "insert") {
|
|
9086
|
+
const transformedData = this.applyTransform(data, tableConfig.transform);
|
|
9087
|
+
job = await table.insert([transformedData]);
|
|
9088
|
+
} else if (operation === "update") {
|
|
9089
|
+
const transformedData = this.applyTransform(data, tableConfig.transform);
|
|
9090
|
+
const keys = Object.keys(transformedData).filter((k) => k !== "id");
|
|
9091
|
+
const setClause = keys.map((k) => `${k} = @${k}`).join(", ");
|
|
9092
|
+
const params = { id, ...transformedData };
|
|
9093
|
+
const query = `UPDATE \`${this.projectId}.${this.datasetId}.${tableConfig.table}\` SET ${setClause} WHERE id = @id`;
|
|
9094
|
+
const maxRetries = 2;
|
|
9095
|
+
let lastError = null;
|
|
9096
|
+
for (let attempt = 1; attempt <= maxRetries; attempt++) {
|
|
9097
|
+
try {
|
|
9098
|
+
const [updateJob] = await this.bigqueryClient.createQueryJob({
|
|
9099
|
+
query,
|
|
9100
|
+
params,
|
|
9101
|
+
location: this.location
|
|
9102
|
+
});
|
|
9103
|
+
await updateJob.getQueryResults();
|
|
9104
|
+
job = [updateJob];
|
|
9105
|
+
break;
|
|
9106
|
+
} catch (error) {
|
|
9107
|
+
lastError = error;
|
|
9108
|
+
if (error?.message?.includes("streaming buffer") && attempt < maxRetries) {
|
|
9109
|
+
const delaySeconds = 30;
|
|
9110
|
+
await new Promise((resolve) => setTimeout(resolve, delaySeconds * 1e3));
|
|
9111
|
+
continue;
|
|
9112
|
+
}
|
|
9113
|
+
throw error;
|
|
9114
|
+
}
|
|
9115
|
+
}
|
|
9116
|
+
if (!job) throw lastError;
|
|
9117
|
+
} else if (operation === "delete") {
|
|
9118
|
+
const query = `DELETE FROM \`${this.projectId}.${this.datasetId}.${tableConfig.table}\` WHERE id = @id`;
|
|
9119
|
+
const [deleteJob] = await this.bigqueryClient.createQueryJob({
|
|
9120
|
+
query,
|
|
9121
|
+
params: { id },
|
|
9122
|
+
location: this.location
|
|
9123
|
+
});
|
|
9124
|
+
await deleteJob.getQueryResults();
|
|
9125
|
+
job = [deleteJob];
|
|
9126
|
+
} else {
|
|
9127
|
+
throw new Error(`Unsupported operation: ${operation}`);
|
|
9128
|
+
}
|
|
9129
|
+
results.push({
|
|
9130
|
+
table: tableConfig.table,
|
|
9131
|
+
success: true,
|
|
9132
|
+
jobId: job[0]?.id
|
|
9133
|
+
});
|
|
9134
|
+
});
|
|
9135
|
+
if (!okTable) {
|
|
9136
|
+
errors.push({
|
|
9137
|
+
table: tableConfig.table,
|
|
9138
|
+
error: errTable.message
|
|
9139
|
+
});
|
|
9140
|
+
}
|
|
9141
|
+
}
|
|
9142
|
+
if (this.logTable) {
|
|
9143
|
+
const [okLog, errLog] = await try_fn_default(async () => {
|
|
9144
|
+
const logTable = dataset.table(this.logTable);
|
|
9145
|
+
await logTable.insert([{
|
|
9146
|
+
resource_name: resourceName,
|
|
9147
|
+
operation,
|
|
9148
|
+
record_id: id,
|
|
9149
|
+
data: JSON.stringify(data),
|
|
9150
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
9151
|
+
source: "s3db-replicator"
|
|
9152
|
+
}]);
|
|
9153
|
+
});
|
|
9154
|
+
if (!okLog) {
|
|
9155
|
+
}
|
|
9156
|
+
}
|
|
9157
|
+
const success = errors.length === 0;
|
|
9158
|
+
this.emit("replicated", {
|
|
9159
|
+
replicator: this.name,
|
|
9160
|
+
resourceName,
|
|
9161
|
+
operation,
|
|
9162
|
+
id,
|
|
9163
|
+
tables: tableConfigs.map((t) => t.table),
|
|
9164
|
+
results,
|
|
9165
|
+
errors,
|
|
9166
|
+
success
|
|
9167
|
+
});
|
|
9168
|
+
return {
|
|
9169
|
+
success,
|
|
9170
|
+
results,
|
|
9171
|
+
errors,
|
|
9172
|
+
tables: tableConfigs.map((t) => t.table)
|
|
9173
|
+
};
|
|
9174
|
+
});
|
|
9175
|
+
if (ok) return result;
|
|
9176
|
+
this.emit("replicator_error", {
|
|
9177
|
+
replicator: this.name,
|
|
9178
|
+
resourceName,
|
|
9179
|
+
operation,
|
|
9180
|
+
id,
|
|
9181
|
+
error: err.message
|
|
9182
|
+
});
|
|
9183
|
+
return { success: false, error: err.message };
|
|
9184
|
+
}
|
|
9185
|
+
async replicateBatch(resourceName, records) {
|
|
9186
|
+
const results = [];
|
|
9187
|
+
const errors = [];
|
|
9188
|
+
for (const record of records) {
|
|
9189
|
+
const [ok, err, res] = await try_fn_default(() => this.replicate(
|
|
9190
|
+
resourceName,
|
|
9191
|
+
record.operation,
|
|
9192
|
+
record.data,
|
|
9193
|
+
record.id,
|
|
9194
|
+
record.beforeData
|
|
9195
|
+
));
|
|
9196
|
+
if (ok) results.push(res);
|
|
9197
|
+
else errors.push({ id: record.id, error: err.message });
|
|
9198
|
+
}
|
|
9199
|
+
return {
|
|
9200
|
+
success: errors.length === 0,
|
|
9201
|
+
results,
|
|
9202
|
+
errors
|
|
9203
|
+
};
|
|
9204
|
+
}
|
|
9205
|
+
async testConnection() {
|
|
9206
|
+
const [ok, err] = await try_fn_default(async () => {
|
|
9207
|
+
if (!this.bigqueryClient) await this.initialize();
|
|
9208
|
+
const dataset = this.bigqueryClient.dataset(this.datasetId);
|
|
9209
|
+
await dataset.getMetadata();
|
|
9210
|
+
return true;
|
|
9211
|
+
});
|
|
9212
|
+
if (ok) return true;
|
|
9213
|
+
this.emit("connection_error", { replicator: this.name, error: err.message });
|
|
9214
|
+
return false;
|
|
9215
|
+
}
|
|
9216
|
+
async cleanup() {
|
|
9217
|
+
}
|
|
9218
|
+
getStatus() {
|
|
9219
|
+
return {
|
|
9220
|
+
...super.getStatus(),
|
|
9221
|
+
projectId: this.projectId,
|
|
9222
|
+
datasetId: this.datasetId,
|
|
9223
|
+
resources: this.resources,
|
|
9224
|
+
logTable: this.logTable
|
|
9225
|
+
};
|
|
9226
|
+
}
|
|
9227
|
+
}
|
|
9228
|
+
var bigquery_replicator_class_default = BigqueryReplicator;
|
|
9229
|
+
|
|
9230
|
+
class PostgresReplicator extends base_replicator_class_default {
|
|
9231
|
+
constructor(config = {}, resources = {}) {
|
|
9232
|
+
super(config);
|
|
9233
|
+
this.connectionString = config.connectionString;
|
|
9234
|
+
this.host = config.host;
|
|
9235
|
+
this.port = config.port || 5432;
|
|
9236
|
+
this.database = config.database;
|
|
9237
|
+
this.user = config.user;
|
|
9238
|
+
this.password = config.password;
|
|
9239
|
+
this.client = null;
|
|
9240
|
+
this.ssl = config.ssl;
|
|
9241
|
+
this.logTable = config.logTable;
|
|
9242
|
+
this.resources = this.parseResourcesConfig(resources);
|
|
9243
|
+
}
|
|
9244
|
+
parseResourcesConfig(resources) {
|
|
9245
|
+
const parsed = {};
|
|
9246
|
+
for (const [resourceName, config] of Object.entries(resources)) {
|
|
9247
|
+
if (typeof config === "string") {
|
|
9248
|
+
parsed[resourceName] = [{
|
|
9249
|
+
table: config,
|
|
9250
|
+
actions: ["insert"]
|
|
9251
|
+
}];
|
|
9252
|
+
} else if (Array.isArray(config)) {
|
|
9253
|
+
parsed[resourceName] = config.map((item) => {
|
|
9254
|
+
if (typeof item === "string") {
|
|
9255
|
+
return { table: item, actions: ["insert"] };
|
|
9256
|
+
}
|
|
9257
|
+
return {
|
|
9258
|
+
table: item.table,
|
|
9259
|
+
actions: item.actions || ["insert"]
|
|
9260
|
+
};
|
|
9261
|
+
});
|
|
9262
|
+
} else if (typeof config === "object") {
|
|
9263
|
+
parsed[resourceName] = [{
|
|
9264
|
+
table: config.table,
|
|
9265
|
+
actions: config.actions || ["insert"]
|
|
9266
|
+
}];
|
|
9267
|
+
}
|
|
9268
|
+
}
|
|
9269
|
+
return parsed;
|
|
9270
|
+
}
|
|
9271
|
+
validateConfig() {
|
|
9272
|
+
const errors = [];
|
|
9273
|
+
if (!this.connectionString && (!this.host || !this.database)) {
|
|
9274
|
+
errors.push("Either connectionString or host+database must be provided");
|
|
9275
|
+
}
|
|
9276
|
+
if (Object.keys(this.resources).length === 0) {
|
|
9277
|
+
errors.push("At least one resource must be configured");
|
|
9278
|
+
}
|
|
9279
|
+
for (const [resourceName, tables] of Object.entries(this.resources)) {
|
|
9280
|
+
for (const tableConfig of tables) {
|
|
9281
|
+
if (!tableConfig.table) {
|
|
9282
|
+
errors.push(`Table name is required for resource '${resourceName}'`);
|
|
9283
|
+
}
|
|
9284
|
+
if (!Array.isArray(tableConfig.actions) || tableConfig.actions.length === 0) {
|
|
9285
|
+
errors.push(`Actions array is required for resource '${resourceName}'`);
|
|
9286
|
+
}
|
|
9287
|
+
const validActions = ["insert", "update", "delete"];
|
|
9288
|
+
const invalidActions = tableConfig.actions.filter((action) => !validActions.includes(action));
|
|
9289
|
+
if (invalidActions.length > 0) {
|
|
9290
|
+
errors.push(`Invalid actions for resource '${resourceName}': ${invalidActions.join(", ")}. Valid actions: ${validActions.join(", ")}`);
|
|
9291
|
+
}
|
|
9292
|
+
}
|
|
9293
|
+
}
|
|
9294
|
+
return { isValid: errors.length === 0, errors };
|
|
9295
|
+
}
|
|
9296
|
+
async initialize(database) {
|
|
9297
|
+
await super.initialize(database);
|
|
9298
|
+
const [ok, err, sdk] = await try_fn_default(() => import('pg'));
|
|
9299
|
+
if (!ok) {
|
|
9300
|
+
this.emit("initialization_error", {
|
|
9301
|
+
replicator: this.name,
|
|
9302
|
+
error: err.message
|
|
9303
|
+
});
|
|
9304
|
+
throw err;
|
|
9305
|
+
}
|
|
9306
|
+
const { Client } = sdk;
|
|
9307
|
+
const config = this.connectionString ? {
|
|
9308
|
+
connectionString: this.connectionString,
|
|
9309
|
+
ssl: this.ssl
|
|
9310
|
+
} : {
|
|
9311
|
+
host: this.host,
|
|
9312
|
+
port: this.port,
|
|
9313
|
+
database: this.database,
|
|
9314
|
+
user: this.user,
|
|
9315
|
+
password: this.password,
|
|
9316
|
+
ssl: this.ssl
|
|
9317
|
+
};
|
|
9318
|
+
this.client = new Client(config);
|
|
9319
|
+
await this.client.connect();
|
|
9320
|
+
if (this.logTable) {
|
|
9321
|
+
await this.createLogTableIfNotExists();
|
|
9322
|
+
}
|
|
9323
|
+
this.emit("initialized", {
|
|
9324
|
+
replicator: this.name,
|
|
9325
|
+
database: this.database || "postgres",
|
|
9326
|
+
resources: Object.keys(this.resources)
|
|
9327
|
+
});
|
|
9328
|
+
}
|
|
9329
|
+
async createLogTableIfNotExists() {
|
|
9330
|
+
const createTableQuery = `
|
|
9331
|
+
CREATE TABLE IF NOT EXISTS ${this.logTable} (
|
|
9332
|
+
id SERIAL PRIMARY KEY,
|
|
9333
|
+
resource_name VARCHAR(255) NOT NULL,
|
|
9334
|
+
operation VARCHAR(50) NOT NULL,
|
|
9335
|
+
record_id VARCHAR(255) NOT NULL,
|
|
9336
|
+
data JSONB,
|
|
9337
|
+
timestamp TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
|
|
9338
|
+
source VARCHAR(100) DEFAULT 's3db-replicator',
|
|
9339
|
+
created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
|
|
9340
|
+
);
|
|
9341
|
+
CREATE INDEX IF NOT EXISTS idx_${this.logTable}_resource_name ON ${this.logTable}(resource_name);
|
|
9342
|
+
CREATE INDEX IF NOT EXISTS idx_${this.logTable}_operation ON ${this.logTable}(operation);
|
|
9343
|
+
CREATE INDEX IF NOT EXISTS idx_${this.logTable}_record_id ON ${this.logTable}(record_id);
|
|
9344
|
+
CREATE INDEX IF NOT EXISTS idx_${this.logTable}_timestamp ON ${this.logTable}(timestamp);
|
|
9345
|
+
`;
|
|
9346
|
+
await this.client.query(createTableQuery);
|
|
9347
|
+
}
|
|
9348
|
+
shouldReplicateResource(resourceName) {
|
|
9349
|
+
return this.resources.hasOwnProperty(resourceName);
|
|
9350
|
+
}
|
|
9351
|
+
shouldReplicateAction(resourceName, operation) {
|
|
9352
|
+
if (!this.resources[resourceName]) return false;
|
|
9353
|
+
return this.resources[resourceName].some(
|
|
9354
|
+
(tableConfig) => tableConfig.actions.includes(operation)
|
|
9355
|
+
);
|
|
9356
|
+
}
|
|
9357
|
+
getTablesForResource(resourceName, operation) {
|
|
9358
|
+
if (!this.resources[resourceName]) return [];
|
|
9359
|
+
return this.resources[resourceName].filter((tableConfig) => tableConfig.actions.includes(operation)).map((tableConfig) => tableConfig.table);
|
|
9360
|
+
}
|
|
9361
|
+
async replicate(resourceName, operation, data, id, beforeData = null) {
|
|
9362
|
+
if (!this.enabled || !this.shouldReplicateResource(resourceName)) {
|
|
9363
|
+
return { skipped: true, reason: "resource_not_included" };
|
|
9364
|
+
}
|
|
9365
|
+
if (!this.shouldReplicateAction(resourceName, operation)) {
|
|
9366
|
+
return { skipped: true, reason: "action_not_included" };
|
|
7910
9367
|
}
|
|
7911
|
-
|
|
7912
|
-
|
|
7913
|
-
|
|
7914
|
-
|
|
9368
|
+
const tables = this.getTablesForResource(resourceName, operation);
|
|
9369
|
+
if (tables.length === 0) {
|
|
9370
|
+
return { skipped: true, reason: "no_tables_for_action" };
|
|
9371
|
+
}
|
|
9372
|
+
const results = [];
|
|
9373
|
+
const errors = [];
|
|
9374
|
+
const [ok, err, result] = await try_fn_default(async () => {
|
|
9375
|
+
for (const table of tables) {
|
|
9376
|
+
const [okTable, errTable] = await try_fn_default(async () => {
|
|
9377
|
+
let result2;
|
|
9378
|
+
if (operation === "insert") {
|
|
9379
|
+
const keys = Object.keys(data);
|
|
9380
|
+
const values = keys.map((k) => data[k]);
|
|
9381
|
+
const columns = keys.map((k) => `"${k}"`).join(", ");
|
|
9382
|
+
const params = keys.map((_, i) => `$${i + 1}`).join(", ");
|
|
9383
|
+
const sql = `INSERT INTO ${table} (${columns}) VALUES (${params}) ON CONFLICT (id) DO NOTHING RETURNING *`;
|
|
9384
|
+
result2 = await this.client.query(sql, values);
|
|
9385
|
+
} else if (operation === "update") {
|
|
9386
|
+
const keys = Object.keys(data).filter((k) => k !== "id");
|
|
9387
|
+
const setClause = keys.map((k, i) => `"${k}"=$${i + 1}`).join(", ");
|
|
9388
|
+
const values = keys.map((k) => data[k]);
|
|
9389
|
+
values.push(id);
|
|
9390
|
+
const sql = `UPDATE ${table} SET ${setClause} WHERE id=$${keys.length + 1} RETURNING *`;
|
|
9391
|
+
result2 = await this.client.query(sql, values);
|
|
9392
|
+
} else if (operation === "delete") {
|
|
9393
|
+
const sql = `DELETE FROM ${table} WHERE id=$1 RETURNING *`;
|
|
9394
|
+
result2 = await this.client.query(sql, [id]);
|
|
9395
|
+
} else {
|
|
9396
|
+
throw new Error(`Unsupported operation: ${operation}`);
|
|
9397
|
+
}
|
|
9398
|
+
results.push({
|
|
9399
|
+
table,
|
|
9400
|
+
success: true,
|
|
9401
|
+
rows: result2.rows,
|
|
9402
|
+
rowCount: result2.rowCount
|
|
9403
|
+
});
|
|
9404
|
+
});
|
|
9405
|
+
if (!okTable) {
|
|
9406
|
+
errors.push({
|
|
9407
|
+
table,
|
|
9408
|
+
error: errTable.message
|
|
9409
|
+
});
|
|
9410
|
+
}
|
|
9411
|
+
}
|
|
9412
|
+
if (this.logTable) {
|
|
9413
|
+
const [okLog, errLog] = await try_fn_default(async () => {
|
|
9414
|
+
await this.client.query(
|
|
9415
|
+
`INSERT INTO ${this.logTable} (resource_name, operation, record_id, data, timestamp, source) VALUES ($1, $2, $3, $4, $5, $6)`,
|
|
9416
|
+
[resourceName, operation, id, JSON.stringify(data), (/* @__PURE__ */ new Date()).toISOString(), "s3db-replicator"]
|
|
9417
|
+
);
|
|
9418
|
+
});
|
|
9419
|
+
if (!okLog) {
|
|
9420
|
+
}
|
|
7915
9421
|
}
|
|
9422
|
+
const success = errors.length === 0;
|
|
9423
|
+
this.emit("replicated", {
|
|
9424
|
+
replicator: this.name,
|
|
9425
|
+
resourceName,
|
|
9426
|
+
operation,
|
|
9427
|
+
id,
|
|
9428
|
+
tables,
|
|
9429
|
+
results,
|
|
9430
|
+
errors,
|
|
9431
|
+
success
|
|
9432
|
+
});
|
|
9433
|
+
return {
|
|
9434
|
+
success,
|
|
9435
|
+
results,
|
|
9436
|
+
errors,
|
|
9437
|
+
tables
|
|
9438
|
+
};
|
|
9439
|
+
});
|
|
9440
|
+
if (ok) return result;
|
|
9441
|
+
this.emit("replicator_error", {
|
|
9442
|
+
replicator: this.name,
|
|
9443
|
+
resourceName,
|
|
9444
|
+
operation,
|
|
9445
|
+
id,
|
|
9446
|
+
error: err.message
|
|
9447
|
+
});
|
|
9448
|
+
return { success: false, error: err.message };
|
|
9449
|
+
}
|
|
9450
|
+
async replicateBatch(resourceName, records) {
|
|
9451
|
+
const results = [];
|
|
9452
|
+
const errors = [];
|
|
9453
|
+
for (const record of records) {
|
|
9454
|
+
const [ok, err, res] = await try_fn_default(() => this.replicate(
|
|
9455
|
+
resourceName,
|
|
9456
|
+
record.operation,
|
|
9457
|
+
record.data,
|
|
9458
|
+
record.id,
|
|
9459
|
+
record.beforeData
|
|
9460
|
+
));
|
|
9461
|
+
if (ok) results.push(res);
|
|
9462
|
+
else errors.push({ id: record.id, error: err.message });
|
|
7916
9463
|
}
|
|
9464
|
+
return {
|
|
9465
|
+
success: errors.length === 0,
|
|
9466
|
+
results,
|
|
9467
|
+
errors
|
|
9468
|
+
};
|
|
9469
|
+
}
|
|
9470
|
+
async testConnection() {
|
|
9471
|
+
const [ok, err] = await try_fn_default(async () => {
|
|
9472
|
+
if (!this.client) await this.initialize();
|
|
9473
|
+
await this.client.query("SELECT 1");
|
|
9474
|
+
return true;
|
|
9475
|
+
});
|
|
9476
|
+
if (ok) return true;
|
|
9477
|
+
this.emit("connection_error", { replicator: this.name, error: err.message });
|
|
9478
|
+
return false;
|
|
9479
|
+
}
|
|
9480
|
+
async cleanup() {
|
|
9481
|
+
if (this.client) await this.client.end();
|
|
9482
|
+
}
|
|
9483
|
+
getStatus() {
|
|
9484
|
+
return {
|
|
9485
|
+
...super.getStatus(),
|
|
9486
|
+
database: this.database || "postgres",
|
|
9487
|
+
resources: this.resources,
|
|
9488
|
+
logTable: this.logTable
|
|
9489
|
+
};
|
|
7917
9490
|
}
|
|
7918
9491
|
}
|
|
9492
|
+
var postgres_replicator_class_default = PostgresReplicator;
|
|
7919
9493
|
|
|
7920
9494
|
const S3_DEFAULT_REGION = "us-east-1";
|
|
7921
9495
|
const S3_DEFAULT_ENDPOINT = "https://s3.us-east-1.amazonaws.com";
|
|
@@ -9999,7 +11573,7 @@ class Resource extends EventEmitter {
|
|
|
9999
11573
|
if (okParse) contentType = "application/json";
|
|
10000
11574
|
}
|
|
10001
11575
|
if (this.behavior === "body-only" && (!body || body === "")) {
|
|
10002
|
-
throw new Error(`[Resource.insert]
|
|
11576
|
+
throw new Error(`[Resource.insert] Attempt to save object without body! Data: id=${finalId}, resource=${this.name}`);
|
|
10003
11577
|
}
|
|
10004
11578
|
const [okPut, errPut, putResult] = await try_fn_default(() => this.client.putObject({
|
|
10005
11579
|
key,
|
|
@@ -11346,8 +12920,8 @@ class Resource extends EventEmitter {
|
|
|
11346
12920
|
return mappedData;
|
|
11347
12921
|
}
|
|
11348
12922
|
/**
|
|
11349
|
-
* Compose the full object (metadata + body) as
|
|
11350
|
-
*
|
|
12923
|
+
* Compose the full object (metadata + body) as returned by .get(),
|
|
12924
|
+
* using in-memory data after insert/update, according to behavior
|
|
11351
12925
|
*/
|
|
11352
12926
|
async composeFullObjectFromWrite({ id, metadata, body, behavior }) {
|
|
11353
12927
|
const behaviorFlags = {};
|
|
@@ -11502,7 +13076,7 @@ class Resource extends EventEmitter {
|
|
|
11502
13076
|
if (!this._middlewares.has(method)) throw new ResourceError(`No such method for middleware: ${method}`, { operation: "useMiddleware", method });
|
|
11503
13077
|
this._middlewares.get(method).push(fn);
|
|
11504
13078
|
}
|
|
11505
|
-
//
|
|
13079
|
+
// Utility to apply schema default values
|
|
11506
13080
|
applyDefaults(data) {
|
|
11507
13081
|
const out = { ...data };
|
|
11508
13082
|
for (const [key, def] of Object.entries(this.attributes)) {
|
|
@@ -11634,7 +13208,7 @@ class Database extends EventEmitter {
|
|
|
11634
13208
|
super();
|
|
11635
13209
|
this.version = "1";
|
|
11636
13210
|
this.s3dbVersion = (() => {
|
|
11637
|
-
const [ok, err, version] = try_fn_default(() => true ? "7.
|
|
13211
|
+
const [ok, err, version] = try_fn_default(() => true ? "7.3.3" : "latest");
|
|
11638
13212
|
return ok ? version : "latest";
|
|
11639
13213
|
})();
|
|
11640
13214
|
this.resources = {};
|
|
@@ -11707,7 +13281,7 @@ class Database extends EventEmitter {
|
|
|
11707
13281
|
name,
|
|
11708
13282
|
client: this.client,
|
|
11709
13283
|
database: this,
|
|
11710
|
-
//
|
|
13284
|
+
// ensure reference
|
|
11711
13285
|
version: currentVersion,
|
|
11712
13286
|
attributes: versionData.attributes,
|
|
11713
13287
|
behavior: versionData.behavior || "user-managed",
|
|
@@ -12109,6 +13683,570 @@ class Database extends EventEmitter {
|
|
|
12109
13683
|
class S3db extends Database {
|
|
12110
13684
|
}
|
|
12111
13685
|
|
|
13686
|
+
function normalizeResourceName$1(name) {
|
|
13687
|
+
return typeof name === "string" ? name.trim().toLowerCase() : name;
|
|
13688
|
+
}
|
|
13689
|
+
class S3dbReplicator extends base_replicator_class_default {
|
|
13690
|
+
constructor(config = {}, resources = [], client = null) {
|
|
13691
|
+
super(config);
|
|
13692
|
+
this.instanceId = Math.random().toString(36).slice(2, 10);
|
|
13693
|
+
this.client = client;
|
|
13694
|
+
this.connectionString = config.connectionString;
|
|
13695
|
+
let normalizedResources = resources;
|
|
13696
|
+
if (!resources) normalizedResources = {};
|
|
13697
|
+
else if (Array.isArray(resources)) {
|
|
13698
|
+
normalizedResources = {};
|
|
13699
|
+
for (const res of resources) {
|
|
13700
|
+
if (typeof res === "string") normalizedResources[normalizeResourceName$1(res)] = res;
|
|
13701
|
+
}
|
|
13702
|
+
} else if (typeof resources === "string") {
|
|
13703
|
+
normalizedResources[normalizeResourceName$1(resources)] = resources;
|
|
13704
|
+
}
|
|
13705
|
+
this.resourcesMap = this._normalizeResources(normalizedResources);
|
|
13706
|
+
}
|
|
13707
|
+
_normalizeResources(resources) {
|
|
13708
|
+
if (!resources) return {};
|
|
13709
|
+
if (Array.isArray(resources)) {
|
|
13710
|
+
const map = {};
|
|
13711
|
+
for (const res of resources) {
|
|
13712
|
+
if (typeof res === "string") map[normalizeResourceName$1(res)] = res;
|
|
13713
|
+
else if (Array.isArray(res) && typeof res[0] === "string") map[normalizeResourceName$1(res[0])] = res;
|
|
13714
|
+
else if (typeof res === "object" && res.resource) {
|
|
13715
|
+
map[normalizeResourceName$1(res.resource)] = { ...res };
|
|
13716
|
+
}
|
|
13717
|
+
}
|
|
13718
|
+
return map;
|
|
13719
|
+
}
|
|
13720
|
+
if (typeof resources === "object") {
|
|
13721
|
+
const map = {};
|
|
13722
|
+
for (const [src, dest] of Object.entries(resources)) {
|
|
13723
|
+
const normSrc = normalizeResourceName$1(src);
|
|
13724
|
+
if (typeof dest === "string") map[normSrc] = dest;
|
|
13725
|
+
else if (Array.isArray(dest)) {
|
|
13726
|
+
map[normSrc] = dest.map((item) => {
|
|
13727
|
+
if (typeof item === "string") return item;
|
|
13728
|
+
if (typeof item === "function") return item;
|
|
13729
|
+
if (typeof item === "object" && item.resource) {
|
|
13730
|
+
return { ...item };
|
|
13731
|
+
}
|
|
13732
|
+
return item;
|
|
13733
|
+
});
|
|
13734
|
+
} else if (typeof dest === "function") map[normSrc] = dest;
|
|
13735
|
+
else if (typeof dest === "object" && dest.resource) {
|
|
13736
|
+
map[normSrc] = { ...dest };
|
|
13737
|
+
}
|
|
13738
|
+
}
|
|
13739
|
+
return map;
|
|
13740
|
+
}
|
|
13741
|
+
if (typeof resources === "function") {
|
|
13742
|
+
return resources;
|
|
13743
|
+
}
|
|
13744
|
+
if (typeof resources === "string") {
|
|
13745
|
+
const map = { [normalizeResourceName$1(resources)]: resources };
|
|
13746
|
+
return map;
|
|
13747
|
+
}
|
|
13748
|
+
return {};
|
|
13749
|
+
}
|
|
13750
|
+
validateConfig() {
|
|
13751
|
+
const errors = [];
|
|
13752
|
+
if (!this.client && !this.connectionString) {
|
|
13753
|
+
errors.push("You must provide a client or a connectionString");
|
|
13754
|
+
}
|
|
13755
|
+
if (!this.resourcesMap || typeof this.resourcesMap === "object" && Object.keys(this.resourcesMap).length === 0) {
|
|
13756
|
+
errors.push("You must provide a resources map or array");
|
|
13757
|
+
}
|
|
13758
|
+
return { isValid: errors.length === 0, errors };
|
|
13759
|
+
}
|
|
13760
|
+
async initialize(database) {
|
|
13761
|
+
try {
|
|
13762
|
+
await super.initialize(database);
|
|
13763
|
+
if (this.client) {
|
|
13764
|
+
this.targetDatabase = this.client;
|
|
13765
|
+
} else if (this.connectionString) {
|
|
13766
|
+
const targetConfig = {
|
|
13767
|
+
connectionString: this.connectionString,
|
|
13768
|
+
region: this.region,
|
|
13769
|
+
keyPrefix: this.keyPrefix,
|
|
13770
|
+
verbose: this.config.verbose || false
|
|
13771
|
+
};
|
|
13772
|
+
this.targetDatabase = new S3db(targetConfig);
|
|
13773
|
+
await this.targetDatabase.connect();
|
|
13774
|
+
} else {
|
|
13775
|
+
throw new Error("S3dbReplicator: No client or connectionString provided");
|
|
13776
|
+
}
|
|
13777
|
+
this.emit("connected", {
|
|
13778
|
+
replicator: this.name,
|
|
13779
|
+
target: this.connectionString || "client-provided"
|
|
13780
|
+
});
|
|
13781
|
+
} catch (err) {
|
|
13782
|
+
throw err;
|
|
13783
|
+
}
|
|
13784
|
+
}
|
|
13785
|
+
// Support both object and parameter signatures for flexibility
|
|
13786
|
+
async replicate(resourceOrObj, operation, data, recordId, beforeData) {
|
|
13787
|
+
let resource, op, payload, id;
|
|
13788
|
+
if (typeof resourceOrObj === "object" && resourceOrObj.resource) {
|
|
13789
|
+
resource = resourceOrObj.resource;
|
|
13790
|
+
op = resourceOrObj.operation;
|
|
13791
|
+
payload = resourceOrObj.data;
|
|
13792
|
+
id = resourceOrObj.id;
|
|
13793
|
+
} else {
|
|
13794
|
+
resource = resourceOrObj;
|
|
13795
|
+
op = operation;
|
|
13796
|
+
payload = data;
|
|
13797
|
+
id = recordId;
|
|
13798
|
+
}
|
|
13799
|
+
const normResource = normalizeResourceName$1(resource);
|
|
13800
|
+
const destResource = this._resolveDestResource(normResource, payload);
|
|
13801
|
+
const destResourceObj = this._getDestResourceObj(destResource);
|
|
13802
|
+
const transformedData = this._applyTransformer(normResource, payload);
|
|
13803
|
+
let result;
|
|
13804
|
+
if (op === "insert") {
|
|
13805
|
+
result = await destResourceObj.insert(transformedData);
|
|
13806
|
+
} else if (op === "update") {
|
|
13807
|
+
result = await destResourceObj.update(id, transformedData);
|
|
13808
|
+
} else if (op === "delete") {
|
|
13809
|
+
result = await destResourceObj.delete(id);
|
|
13810
|
+
} else {
|
|
13811
|
+
throw new Error(`Invalid operation: ${op}. Supported operations are: insert, update, delete`);
|
|
13812
|
+
}
|
|
13813
|
+
return result;
|
|
13814
|
+
}
|
|
13815
|
+
_applyTransformer(resource, data) {
|
|
13816
|
+
const normResource = normalizeResourceName$1(resource);
|
|
13817
|
+
const entry = this.resourcesMap[normResource];
|
|
13818
|
+
let result;
|
|
13819
|
+
if (!entry) return data;
|
|
13820
|
+
if (Array.isArray(entry) && typeof entry[1] === "function") {
|
|
13821
|
+
result = entry[1](data);
|
|
13822
|
+
} else if (typeof entry === "function") {
|
|
13823
|
+
result = entry(data);
|
|
13824
|
+
} else if (typeof entry === "object") {
|
|
13825
|
+
if (typeof entry.transform === "function") result = entry.transform(data);
|
|
13826
|
+
else if (typeof entry.transformer === "function") result = entry.transformer(data);
|
|
13827
|
+
} else {
|
|
13828
|
+
result = data;
|
|
13829
|
+
}
|
|
13830
|
+
if (result && data && data.id && !result.id) result.id = data.id;
|
|
13831
|
+
if (!result && data) result = data;
|
|
13832
|
+
return result;
|
|
13833
|
+
}
|
|
13834
|
+
_resolveDestResource(resource, data) {
|
|
13835
|
+
const normResource = normalizeResourceName$1(resource);
|
|
13836
|
+
const entry = this.resourcesMap[normResource];
|
|
13837
|
+
if (!entry) return resource;
|
|
13838
|
+
if (Array.isArray(entry)) {
|
|
13839
|
+
if (typeof entry[0] === "string") return entry[0];
|
|
13840
|
+
if (typeof entry[0] === "object" && entry[0].resource) return entry[0].resource;
|
|
13841
|
+
if (typeof entry[0] === "function") return resource;
|
|
13842
|
+
}
|
|
13843
|
+
if (typeof entry === "string") return entry;
|
|
13844
|
+
if (resource && !targetResourceName) targetResourceName = resource;
|
|
13845
|
+
if (typeof entry === "object" && entry.resource) return entry.resource;
|
|
13846
|
+
return resource;
|
|
13847
|
+
}
|
|
13848
|
+
_getDestResourceObj(resource) {
|
|
13849
|
+
if (!this.client || !this.client.resources) return null;
|
|
13850
|
+
const available = Object.keys(this.client.resources);
|
|
13851
|
+
const norm = normalizeResourceName$1(resource);
|
|
13852
|
+
const found = available.find((r) => normalizeResourceName$1(r) === norm);
|
|
13853
|
+
if (!found) {
|
|
13854
|
+
throw new Error(`[S3dbReplicator] Destination resource not found: ${resource}. Available: ${available.join(", ")}`);
|
|
13855
|
+
}
|
|
13856
|
+
return this.client.resources[found];
|
|
13857
|
+
}
|
|
13858
|
+
async replicateBatch(resourceName, records) {
|
|
13859
|
+
if (!this.enabled || !this.shouldReplicateResource(resourceName)) {
|
|
13860
|
+
return { skipped: true, reason: "resource_not_included" };
|
|
13861
|
+
}
|
|
13862
|
+
const results = [];
|
|
13863
|
+
const errors = [];
|
|
13864
|
+
for (const record of records) {
|
|
13865
|
+
const [ok, err, result] = await try_fn_default(() => this.replicate({
|
|
13866
|
+
resource: resourceName,
|
|
13867
|
+
operation: record.operation,
|
|
13868
|
+
id: record.id,
|
|
13869
|
+
data: record.data,
|
|
13870
|
+
beforeData: record.beforeData
|
|
13871
|
+
}));
|
|
13872
|
+
if (ok) results.push(result);
|
|
13873
|
+
else errors.push({ id: record.id, error: err.message });
|
|
13874
|
+
}
|
|
13875
|
+
this.emit("batch_replicated", {
|
|
13876
|
+
replicator: this.name,
|
|
13877
|
+
resourceName,
|
|
13878
|
+
total: records.length,
|
|
13879
|
+
successful: results.length,
|
|
13880
|
+
errors: errors.length
|
|
13881
|
+
});
|
|
13882
|
+
return {
|
|
13883
|
+
success: errors.length === 0,
|
|
13884
|
+
results,
|
|
13885
|
+
errors,
|
|
13886
|
+
total: records.length
|
|
13887
|
+
};
|
|
13888
|
+
}
|
|
13889
|
+
async testConnection() {
|
|
13890
|
+
const [ok, err] = await try_fn_default(async () => {
|
|
13891
|
+
if (!this.targetDatabase) {
|
|
13892
|
+
await this.initialize(this.database);
|
|
13893
|
+
}
|
|
13894
|
+
await this.targetDatabase.listResources();
|
|
13895
|
+
return true;
|
|
13896
|
+
});
|
|
13897
|
+
if (ok) return true;
|
|
13898
|
+
this.emit("connection_error", {
|
|
13899
|
+
replicator: this.name,
|
|
13900
|
+
error: err.message
|
|
13901
|
+
});
|
|
13902
|
+
return false;
|
|
13903
|
+
}
|
|
13904
|
+
async getStatus() {
|
|
13905
|
+
const baseStatus = await super.getStatus();
|
|
13906
|
+
return {
|
|
13907
|
+
...baseStatus,
|
|
13908
|
+
connected: !!this.targetDatabase,
|
|
13909
|
+
targetDatabase: this.connectionString || "client-provided",
|
|
13910
|
+
resources: Object.keys(this.resourcesMap || {}),
|
|
13911
|
+
totalreplicators: this.listenerCount("replicated"),
|
|
13912
|
+
totalErrors: this.listenerCount("replicator_error")
|
|
13913
|
+
};
|
|
13914
|
+
}
|
|
13915
|
+
async cleanup() {
|
|
13916
|
+
if (this.targetDatabase) {
|
|
13917
|
+
this.targetDatabase.removeAllListeners();
|
|
13918
|
+
}
|
|
13919
|
+
await super.cleanup();
|
|
13920
|
+
}
|
|
13921
|
+
shouldReplicateResource(resource, action) {
|
|
13922
|
+
const normResource = normalizeResourceName$1(resource);
|
|
13923
|
+
const entry = this.resourcesMap[normResource];
|
|
13924
|
+
if (!entry) return false;
|
|
13925
|
+
if (!action) return true;
|
|
13926
|
+
if (Array.isArray(entry)) {
|
|
13927
|
+
for (const item of entry) {
|
|
13928
|
+
if (typeof item === "object" && item.resource) {
|
|
13929
|
+
if (item.actions && Array.isArray(item.actions)) {
|
|
13930
|
+
if (item.actions.includes(action)) return true;
|
|
13931
|
+
} else {
|
|
13932
|
+
return true;
|
|
13933
|
+
}
|
|
13934
|
+
} else if (typeof item === "string" || typeof item === "function") {
|
|
13935
|
+
return true;
|
|
13936
|
+
}
|
|
13937
|
+
}
|
|
13938
|
+
return false;
|
|
13939
|
+
}
|
|
13940
|
+
if (typeof entry === "object" && entry.resource) {
|
|
13941
|
+
if (entry.actions && Array.isArray(entry.actions)) {
|
|
13942
|
+
return entry.actions.includes(action);
|
|
13943
|
+
}
|
|
13944
|
+
return true;
|
|
13945
|
+
}
|
|
13946
|
+
if (typeof entry === "string" || typeof entry === "function") {
|
|
13947
|
+
return true;
|
|
13948
|
+
}
|
|
13949
|
+
return false;
|
|
13950
|
+
}
|
|
13951
|
+
}
|
|
13952
|
+
var s3db_replicator_class_default = S3dbReplicator;
|
|
13953
|
+
|
|
13954
|
+
class SqsReplicator extends base_replicator_class_default {
|
|
13955
|
+
constructor(config = {}, resources = [], client = null) {
|
|
13956
|
+
super(config);
|
|
13957
|
+
this.client = client;
|
|
13958
|
+
this.queueUrl = config.queueUrl;
|
|
13959
|
+
this.queues = config.queues || {};
|
|
13960
|
+
this.defaultQueue = config.defaultQueue || config.defaultQueueUrl || config.queueUrlDefault;
|
|
13961
|
+
this.region = config.region || "us-east-1";
|
|
13962
|
+
this.sqsClient = client || null;
|
|
13963
|
+
this.messageGroupId = config.messageGroupId;
|
|
13964
|
+
this.deduplicationId = config.deduplicationId;
|
|
13965
|
+
if (Array.isArray(resources)) {
|
|
13966
|
+
this.resources = {};
|
|
13967
|
+
for (const resource of resources) {
|
|
13968
|
+
if (typeof resource === "string") {
|
|
13969
|
+
this.resources[resource] = true;
|
|
13970
|
+
} else if (typeof resource === "object" && resource.name) {
|
|
13971
|
+
this.resources[resource.name] = resource;
|
|
13972
|
+
}
|
|
13973
|
+
}
|
|
13974
|
+
} else if (typeof resources === "object") {
|
|
13975
|
+
this.resources = resources;
|
|
13976
|
+
for (const [resourceName, resourceConfig] of Object.entries(resources)) {
|
|
13977
|
+
if (resourceConfig && resourceConfig.queueUrl) {
|
|
13978
|
+
this.queues[resourceName] = resourceConfig.queueUrl;
|
|
13979
|
+
}
|
|
13980
|
+
}
|
|
13981
|
+
} else {
|
|
13982
|
+
this.resources = {};
|
|
13983
|
+
}
|
|
13984
|
+
}
|
|
13985
|
+
validateConfig() {
|
|
13986
|
+
const errors = [];
|
|
13987
|
+
if (!this.queueUrl && Object.keys(this.queues).length === 0 && !this.defaultQueue && !this.resourceQueueMap) {
|
|
13988
|
+
errors.push("Either queueUrl, queues object, defaultQueue, or resourceQueueMap must be provided");
|
|
13989
|
+
}
|
|
13990
|
+
return {
|
|
13991
|
+
isValid: errors.length === 0,
|
|
13992
|
+
errors
|
|
13993
|
+
};
|
|
13994
|
+
}
|
|
13995
|
+
getQueueUrlsForResource(resource) {
|
|
13996
|
+
if (this.resourceQueueMap && this.resourceQueueMap[resource]) {
|
|
13997
|
+
return this.resourceQueueMap[resource];
|
|
13998
|
+
}
|
|
13999
|
+
if (this.queues[resource]) {
|
|
14000
|
+
return [this.queues[resource]];
|
|
14001
|
+
}
|
|
14002
|
+
if (this.queueUrl) {
|
|
14003
|
+
return [this.queueUrl];
|
|
14004
|
+
}
|
|
14005
|
+
if (this.defaultQueue) {
|
|
14006
|
+
return [this.defaultQueue];
|
|
14007
|
+
}
|
|
14008
|
+
throw new Error(`No queue URL found for resource '${resource}'`);
|
|
14009
|
+
}
|
|
14010
|
+
_applyTransformer(resource, data) {
|
|
14011
|
+
const entry = this.resources[resource];
|
|
14012
|
+
let result = data;
|
|
14013
|
+
if (!entry) return data;
|
|
14014
|
+
if (typeof entry.transform === "function") {
|
|
14015
|
+
result = entry.transform(data);
|
|
14016
|
+
} else if (typeof entry.transformer === "function") {
|
|
14017
|
+
result = entry.transformer(data);
|
|
14018
|
+
}
|
|
14019
|
+
return result || data;
|
|
14020
|
+
}
|
|
14021
|
+
/**
|
|
14022
|
+
* Create standardized message structure
|
|
14023
|
+
*/
|
|
14024
|
+
createMessage(resource, operation, data, id, beforeData = null) {
|
|
14025
|
+
const baseMessage = {
|
|
14026
|
+
resource,
|
|
14027
|
+
// padronizado para 'resource'
|
|
14028
|
+
action: operation,
|
|
14029
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
14030
|
+
source: "s3db-replicator"
|
|
14031
|
+
};
|
|
14032
|
+
switch (operation) {
|
|
14033
|
+
case "insert":
|
|
14034
|
+
return {
|
|
14035
|
+
...baseMessage,
|
|
14036
|
+
data
|
|
14037
|
+
};
|
|
14038
|
+
case "update":
|
|
14039
|
+
return {
|
|
14040
|
+
...baseMessage,
|
|
14041
|
+
before: beforeData,
|
|
14042
|
+
data
|
|
14043
|
+
};
|
|
14044
|
+
case "delete":
|
|
14045
|
+
return {
|
|
14046
|
+
...baseMessage,
|
|
14047
|
+
data
|
|
14048
|
+
};
|
|
14049
|
+
default:
|
|
14050
|
+
return {
|
|
14051
|
+
...baseMessage,
|
|
14052
|
+
data
|
|
14053
|
+
};
|
|
14054
|
+
}
|
|
14055
|
+
}
|
|
14056
|
+
async initialize(database, client) {
|
|
14057
|
+
await super.initialize(database);
|
|
14058
|
+
if (!this.sqsClient) {
|
|
14059
|
+
const [ok, err, sdk] = await try_fn_default(() => import('@aws-sdk/client-sqs'));
|
|
14060
|
+
if (!ok) {
|
|
14061
|
+
this.emit("initialization_error", {
|
|
14062
|
+
replicator: this.name,
|
|
14063
|
+
error: err.message
|
|
14064
|
+
});
|
|
14065
|
+
throw err;
|
|
14066
|
+
}
|
|
14067
|
+
const { SQSClient } = sdk;
|
|
14068
|
+
this.sqsClient = client || new SQSClient({
|
|
14069
|
+
region: this.region,
|
|
14070
|
+
credentials: this.config.credentials
|
|
14071
|
+
});
|
|
14072
|
+
this.emit("initialized", {
|
|
14073
|
+
replicator: this.name,
|
|
14074
|
+
queueUrl: this.queueUrl,
|
|
14075
|
+
queues: this.queues,
|
|
14076
|
+
defaultQueue: this.defaultQueue
|
|
14077
|
+
});
|
|
14078
|
+
}
|
|
14079
|
+
}
|
|
14080
|
+
async replicate(resource, operation, data, id, beforeData = null) {
|
|
14081
|
+
if (!this.enabled || !this.shouldReplicateResource(resource)) {
|
|
14082
|
+
return { skipped: true, reason: "resource_not_included" };
|
|
14083
|
+
}
|
|
14084
|
+
const [ok, err, result] = await try_fn_default(async () => {
|
|
14085
|
+
const { SendMessageCommand } = await import('@aws-sdk/client-sqs');
|
|
14086
|
+
const queueUrls = this.getQueueUrlsForResource(resource);
|
|
14087
|
+
const transformedData = this._applyTransformer(resource, data);
|
|
14088
|
+
const message = this.createMessage(resource, operation, transformedData, id, beforeData);
|
|
14089
|
+
const results = [];
|
|
14090
|
+
for (const queueUrl of queueUrls) {
|
|
14091
|
+
const command = new SendMessageCommand({
|
|
14092
|
+
QueueUrl: queueUrl,
|
|
14093
|
+
MessageBody: JSON.stringify(message),
|
|
14094
|
+
MessageGroupId: this.messageGroupId,
|
|
14095
|
+
MessageDeduplicationId: this.deduplicationId ? `${resource}:${operation}:${id}` : void 0
|
|
14096
|
+
});
|
|
14097
|
+
const result2 = await this.sqsClient.send(command);
|
|
14098
|
+
results.push({ queueUrl, messageId: result2.MessageId });
|
|
14099
|
+
this.emit("replicated", {
|
|
14100
|
+
replicator: this.name,
|
|
14101
|
+
resource,
|
|
14102
|
+
operation,
|
|
14103
|
+
id,
|
|
14104
|
+
queueUrl,
|
|
14105
|
+
messageId: result2.MessageId,
|
|
14106
|
+
success: true
|
|
14107
|
+
});
|
|
14108
|
+
}
|
|
14109
|
+
return { success: true, results };
|
|
14110
|
+
});
|
|
14111
|
+
if (ok) return result;
|
|
14112
|
+
this.emit("replicator_error", {
|
|
14113
|
+
replicator: this.name,
|
|
14114
|
+
resource,
|
|
14115
|
+
operation,
|
|
14116
|
+
id,
|
|
14117
|
+
error: err.message
|
|
14118
|
+
});
|
|
14119
|
+
return { success: false, error: err.message };
|
|
14120
|
+
}
|
|
14121
|
+
async replicateBatch(resource, records) {
|
|
14122
|
+
if (!this.enabled || !this.shouldReplicateResource(resource)) {
|
|
14123
|
+
return { skipped: true, reason: "resource_not_included" };
|
|
14124
|
+
}
|
|
14125
|
+
const [ok, err, result] = await try_fn_default(async () => {
|
|
14126
|
+
const { SendMessageBatchCommand } = await import('@aws-sdk/client-sqs');
|
|
14127
|
+
const queueUrls = this.getQueueUrlsForResource(resource);
|
|
14128
|
+
const batchSize = 10;
|
|
14129
|
+
const batches = [];
|
|
14130
|
+
for (let i = 0; i < records.length; i += batchSize) {
|
|
14131
|
+
batches.push(records.slice(i, i + batchSize));
|
|
14132
|
+
}
|
|
14133
|
+
const results = [];
|
|
14134
|
+
const errors = [];
|
|
14135
|
+
for (const batch of batches) {
|
|
14136
|
+
const [okBatch, errBatch] = await try_fn_default(async () => {
|
|
14137
|
+
const entries = batch.map((record, index) => ({
|
|
14138
|
+
Id: `${record.id}-${index}`,
|
|
14139
|
+
MessageBody: JSON.stringify(this.createMessage(
|
|
14140
|
+
resource,
|
|
14141
|
+
record.operation,
|
|
14142
|
+
record.data,
|
|
14143
|
+
record.id,
|
|
14144
|
+
record.beforeData
|
|
14145
|
+
)),
|
|
14146
|
+
MessageGroupId: this.messageGroupId,
|
|
14147
|
+
MessageDeduplicationId: this.deduplicationId ? `${resource}:${record.operation}:${record.id}` : void 0
|
|
14148
|
+
}));
|
|
14149
|
+
const command = new SendMessageBatchCommand({
|
|
14150
|
+
QueueUrl: queueUrls[0],
|
|
14151
|
+
// Assuming all queueUrls in a batch are the same for batching
|
|
14152
|
+
Entries: entries
|
|
14153
|
+
});
|
|
14154
|
+
const result2 = await this.sqsClient.send(command);
|
|
14155
|
+
results.push(result2);
|
|
14156
|
+
});
|
|
14157
|
+
if (!okBatch) {
|
|
14158
|
+
errors.push({ batch: batch.length, error: errBatch.message });
|
|
14159
|
+
if (errBatch.message && (errBatch.message.includes("Batch error") || errBatch.message.includes("Connection") || errBatch.message.includes("Network"))) {
|
|
14160
|
+
throw errBatch;
|
|
14161
|
+
}
|
|
14162
|
+
}
|
|
14163
|
+
}
|
|
14164
|
+
this.emit("batch_replicated", {
|
|
14165
|
+
replicator: this.name,
|
|
14166
|
+
resource,
|
|
14167
|
+
queueUrl: queueUrls[0],
|
|
14168
|
+
// Assuming all queueUrls in a batch are the same for batching
|
|
14169
|
+
total: records.length,
|
|
14170
|
+
successful: results.length,
|
|
14171
|
+
errors: errors.length
|
|
14172
|
+
});
|
|
14173
|
+
return {
|
|
14174
|
+
success: errors.length === 0,
|
|
14175
|
+
results,
|
|
14176
|
+
errors,
|
|
14177
|
+
total: records.length,
|
|
14178
|
+
queueUrl: queueUrls[0]
|
|
14179
|
+
// Assuming all queueUrls in a batch are the same for batching
|
|
14180
|
+
};
|
|
14181
|
+
});
|
|
14182
|
+
if (ok) return result;
|
|
14183
|
+
const errorMessage = err?.message || err || "Unknown error";
|
|
14184
|
+
this.emit("batch_replicator_error", {
|
|
14185
|
+
replicator: this.name,
|
|
14186
|
+
resource,
|
|
14187
|
+
error: errorMessage
|
|
14188
|
+
});
|
|
14189
|
+
return { success: false, error: errorMessage };
|
|
14190
|
+
}
|
|
14191
|
+
async testConnection() {
|
|
14192
|
+
const [ok, err] = await try_fn_default(async () => {
|
|
14193
|
+
if (!this.sqsClient) {
|
|
14194
|
+
await this.initialize(this.database);
|
|
14195
|
+
}
|
|
14196
|
+
const { GetQueueAttributesCommand } = await import('@aws-sdk/client-sqs');
|
|
14197
|
+
const command = new GetQueueAttributesCommand({
|
|
14198
|
+
QueueUrl: this.queueUrl,
|
|
14199
|
+
AttributeNames: ["QueueArn"]
|
|
14200
|
+
});
|
|
14201
|
+
await this.sqsClient.send(command);
|
|
14202
|
+
return true;
|
|
14203
|
+
});
|
|
14204
|
+
if (ok) return true;
|
|
14205
|
+
this.emit("connection_error", {
|
|
14206
|
+
replicator: this.name,
|
|
14207
|
+
error: err.message
|
|
14208
|
+
});
|
|
14209
|
+
return false;
|
|
14210
|
+
}
|
|
14211
|
+
async getStatus() {
|
|
14212
|
+
const baseStatus = await super.getStatus();
|
|
14213
|
+
return {
|
|
14214
|
+
...baseStatus,
|
|
14215
|
+
connected: !!this.sqsClient,
|
|
14216
|
+
queueUrl: this.queueUrl,
|
|
14217
|
+
region: this.region,
|
|
14218
|
+
resources: Object.keys(this.resources || {}),
|
|
14219
|
+
totalreplicators: this.listenerCount("replicated"),
|
|
14220
|
+
totalErrors: this.listenerCount("replicator_error")
|
|
14221
|
+
};
|
|
14222
|
+
}
|
|
14223
|
+
async cleanup() {
|
|
14224
|
+
if (this.sqsClient) {
|
|
14225
|
+
this.sqsClient.destroy();
|
|
14226
|
+
}
|
|
14227
|
+
await super.cleanup();
|
|
14228
|
+
}
|
|
14229
|
+
shouldReplicateResource(resource) {
|
|
14230
|
+
const result = this.resourceQueueMap && Object.keys(this.resourceQueueMap).includes(resource) || this.queues && Object.keys(this.queues).includes(resource) || !!(this.defaultQueue || this.queueUrl) || this.resources && Object.keys(this.resources).includes(resource) || false;
|
|
14231
|
+
return result;
|
|
14232
|
+
}
|
|
14233
|
+
}
|
|
14234
|
+
var sqs_replicator_class_default = SqsReplicator;
|
|
14235
|
+
|
|
14236
|
+
const REPLICATOR_DRIVERS = {
|
|
14237
|
+
s3db: s3db_replicator_class_default,
|
|
14238
|
+
sqs: sqs_replicator_class_default,
|
|
14239
|
+
bigquery: bigquery_replicator_class_default,
|
|
14240
|
+
postgres: postgres_replicator_class_default
|
|
14241
|
+
};
|
|
14242
|
+
function createReplicator(driver, config = {}, resources = [], client = null) {
|
|
14243
|
+
const ReplicatorClass = REPLICATOR_DRIVERS[driver];
|
|
14244
|
+
if (!ReplicatorClass) {
|
|
14245
|
+
throw new Error(`Unknown replicator driver: ${driver}. Available drivers: ${Object.keys(REPLICATOR_DRIVERS).join(", ")}`);
|
|
14246
|
+
}
|
|
14247
|
+
return new ReplicatorClass(config, resources, client);
|
|
14248
|
+
}
|
|
14249
|
+
|
|
12112
14250
|
function normalizeResourceName(name) {
|
|
12113
14251
|
return typeof name === "string" ? name.trim().toLowerCase() : name;
|
|
12114
14252
|
}
|
|
@@ -12156,7 +14294,7 @@ class ReplicatorPlugin extends plugin_class_default {
|
|
|
12156
14294
|
return filtered;
|
|
12157
14295
|
}
|
|
12158
14296
|
installEventListeners(resource, database, plugin) {
|
|
12159
|
-
if (!resource || this.eventListenersInstalled.has(resource.name)) {
|
|
14297
|
+
if (!resource || this.eventListenersInstalled.has(resource.name) || resource.name === this.config.replicatorLogResource) {
|
|
12160
14298
|
return;
|
|
12161
14299
|
}
|
|
12162
14300
|
resource.on("insert", async (data) => {
|
|
@@ -12233,10 +14371,15 @@ class ReplicatorPlugin extends plugin_class_default {
|
|
|
12233
14371
|
this.installEventListeners(resource, database, this);
|
|
12234
14372
|
}
|
|
12235
14373
|
}
|
|
14374
|
+
createReplicator(driver, config, resources, client) {
|
|
14375
|
+
return createReplicator(driver, config, resources, client);
|
|
14376
|
+
}
|
|
12236
14377
|
async initializeReplicators(database) {
|
|
12237
14378
|
for (const replicatorConfig of this.config.replicators) {
|
|
12238
|
-
const { driver, config, resources } = replicatorConfig;
|
|
12239
|
-
const
|
|
14379
|
+
const { driver, config = {}, resources, client, ...otherConfig } = replicatorConfig;
|
|
14380
|
+
const replicatorResources = resources || config.resources || {};
|
|
14381
|
+
const mergedConfig = { ...config, ...otherConfig };
|
|
14382
|
+
const replicator = this.createReplicator(driver, mergedConfig, replicatorResources, client);
|
|
12240
14383
|
if (replicator) {
|
|
12241
14384
|
await replicator.initialize(database);
|
|
12242
14385
|
this.replicators.push(replicator);
|
|
@@ -12247,6 +14390,66 @@ class ReplicatorPlugin extends plugin_class_default {
|
|
|
12247
14390
|
}
|
|
12248
14391
|
async stop() {
|
|
12249
14392
|
}
|
|
14393
|
+
filterInternalFields(data) {
|
|
14394
|
+
if (!data || typeof data !== "object") return data;
|
|
14395
|
+
const filtered = {};
|
|
14396
|
+
for (const [key, value] of Object.entries(data)) {
|
|
14397
|
+
if (!key.startsWith("_") && !key.startsWith("$")) {
|
|
14398
|
+
filtered[key] = value;
|
|
14399
|
+
}
|
|
14400
|
+
}
|
|
14401
|
+
return filtered;
|
|
14402
|
+
}
|
|
14403
|
+
async uploadMetadataFile(database) {
|
|
14404
|
+
if (typeof database.uploadMetadataFile === "function") {
|
|
14405
|
+
await database.uploadMetadataFile();
|
|
14406
|
+
}
|
|
14407
|
+
}
|
|
14408
|
+
async getCompleteData(resource, data) {
|
|
14409
|
+
try {
|
|
14410
|
+
const [ok, err, record] = await try_fn_default(() => resource.get(data.id));
|
|
14411
|
+
if (ok && record) {
|
|
14412
|
+
return record;
|
|
14413
|
+
}
|
|
14414
|
+
} catch (error) {
|
|
14415
|
+
}
|
|
14416
|
+
return data;
|
|
14417
|
+
}
|
|
14418
|
+
async retryWithBackoff(operation, maxRetries = 3) {
|
|
14419
|
+
let lastError;
|
|
14420
|
+
for (let attempt = 1; attempt <= maxRetries; attempt++) {
|
|
14421
|
+
try {
|
|
14422
|
+
return await operation();
|
|
14423
|
+
} catch (error) {
|
|
14424
|
+
lastError = error;
|
|
14425
|
+
if (attempt === maxRetries) {
|
|
14426
|
+
throw error;
|
|
14427
|
+
}
|
|
14428
|
+
const delay = Math.pow(2, attempt - 1) * 1e3;
|
|
14429
|
+
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
14430
|
+
}
|
|
14431
|
+
}
|
|
14432
|
+
throw lastError;
|
|
14433
|
+
}
|
|
14434
|
+
async logError(replicator, resourceName, operation, recordId, data, error) {
|
|
14435
|
+
try {
|
|
14436
|
+
const logResourceName = this.config.replicatorLogResource;
|
|
14437
|
+
if (this.database && this.database.resources && this.database.resources[logResourceName]) {
|
|
14438
|
+
const logResource = this.database.resources[logResourceName];
|
|
14439
|
+
await logResource.insert({
|
|
14440
|
+
replicator: replicator.name || replicator.id,
|
|
14441
|
+
resourceName,
|
|
14442
|
+
operation,
|
|
14443
|
+
recordId,
|
|
14444
|
+
data: JSON.stringify(data),
|
|
14445
|
+
error: error.message,
|
|
14446
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
14447
|
+
status: "error"
|
|
14448
|
+
});
|
|
14449
|
+
}
|
|
14450
|
+
} catch (logError) {
|
|
14451
|
+
}
|
|
14452
|
+
}
|
|
12250
14453
|
async processReplicatorEvent(operation, resourceName, recordId, data, beforeData = null) {
|
|
12251
14454
|
if (!this.config.enabled) return;
|
|
12252
14455
|
const applicableReplicators = this.replicators.filter((replicator) => {
|
|
@@ -12377,7 +14580,7 @@ class ReplicatorPlugin extends plugin_class_default {
|
|
|
12377
14580
|
async getreplicatorStats() {
|
|
12378
14581
|
const replicatorStats = await Promise.all(
|
|
12379
14582
|
this.replicators.map(async (replicator) => {
|
|
12380
|
-
const status = await replicator.
|
|
14583
|
+
const status = await replicator.getStatus();
|
|
12381
14584
|
return {
|
|
12382
14585
|
id: replicator.id,
|
|
12383
14586
|
driver: replicator.driver,
|
|
@@ -12451,12 +14654,12 @@ class ReplicatorPlugin extends plugin_class_default {
|
|
|
12451
14654
|
this.stats.lastSync = (/* @__PURE__ */ new Date()).toISOString();
|
|
12452
14655
|
for (const resourceName in this.database.resources) {
|
|
12453
14656
|
if (normalizeResourceName(resourceName) === normalizeResourceName("replicator_logs")) continue;
|
|
12454
|
-
if (replicator.
|
|
14657
|
+
if (replicator.shouldReplicateResource(resourceName)) {
|
|
12455
14658
|
this.emit("replicator.sync.resource", { resourceName, replicatorId });
|
|
12456
14659
|
const resource = this.database.resources[resourceName];
|
|
12457
14660
|
const allRecords = await resource.getAll();
|
|
12458
14661
|
for (const record of allRecords) {
|
|
12459
|
-
await replicator.
|
|
14662
|
+
await replicator.replicate(resourceName, "insert", record, record.id);
|
|
12460
14663
|
}
|
|
12461
14664
|
}
|
|
12462
14665
|
}
|