s3db.js 7.3.1 → 7.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/s3db.cjs.js +2394 -191
- package/dist/s3db.cjs.min.js +1 -1
- package/dist/s3db.es.js +2394 -191
- package/dist/s3db.es.min.js +1 -1
- package/dist/s3db.iife.js +2393 -190
- package/dist/s3db.iife.min.js +1 -1
- package/package.json +25 -26
- package/src/plugins/cache/filesystem-cache.class.js +1 -1
- package/src/plugins/cache/partition-aware-filesystem-cache.class.js +1 -1
- package/src/plugins/cache/s3-cache.class.js +1 -1
package/dist/s3db.iife.js
CHANGED
|
@@ -1333,7 +1333,8 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
1333
1333
|
version: "2.0"
|
|
1334
1334
|
})
|
|
1335
1335
|
};
|
|
1336
|
-
this.logAudit(auditRecord).catch(
|
|
1336
|
+
this.logAudit(auditRecord).catch(() => {
|
|
1337
|
+
});
|
|
1337
1338
|
});
|
|
1338
1339
|
resource.on("update", async (data) => {
|
|
1339
1340
|
const recordId = data.id;
|
|
@@ -1359,7 +1360,8 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
1359
1360
|
version: "2.0"
|
|
1360
1361
|
})
|
|
1361
1362
|
};
|
|
1362
|
-
this.logAudit(auditRecord).catch(
|
|
1363
|
+
this.logAudit(auditRecord).catch(() => {
|
|
1364
|
+
});
|
|
1363
1365
|
});
|
|
1364
1366
|
resource.on("delete", async (data) => {
|
|
1365
1367
|
const recordId = data.id;
|
|
@@ -1385,7 +1387,8 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
1385
1387
|
version: "2.0"
|
|
1386
1388
|
})
|
|
1387
1389
|
};
|
|
1388
|
-
this.logAudit(auditRecord).catch(
|
|
1390
|
+
this.logAudit(auditRecord).catch(() => {
|
|
1391
|
+
});
|
|
1389
1392
|
});
|
|
1390
1393
|
resource.useMiddleware("deleteMany", async (ctx, next) => {
|
|
1391
1394
|
const ids = ctx.args[0];
|
|
@@ -1418,7 +1421,8 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
1418
1421
|
batchOperation: true
|
|
1419
1422
|
})
|
|
1420
1423
|
};
|
|
1421
|
-
this.logAudit(auditRecord).catch(
|
|
1424
|
+
this.logAudit(auditRecord).catch(() => {
|
|
1425
|
+
});
|
|
1422
1426
|
}
|
|
1423
1427
|
}
|
|
1424
1428
|
return result;
|
|
@@ -2065,6 +2069,16 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
2065
2069
|
// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
|
|
2066
2070
|
// USE OR OTHER DEALINGS IN THE SOFTWARE.
|
|
2067
2071
|
|
|
2072
|
+
var getOwnPropertyDescriptors = Object.getOwnPropertyDescriptors ||
|
|
2073
|
+
function getOwnPropertyDescriptors(obj) {
|
|
2074
|
+
var keys = Object.keys(obj);
|
|
2075
|
+
var descriptors = {};
|
|
2076
|
+
for (var i = 0; i < keys.length; i++) {
|
|
2077
|
+
descriptors[keys[i]] = Object.getOwnPropertyDescriptor(obj, keys[i]);
|
|
2078
|
+
}
|
|
2079
|
+
return descriptors;
|
|
2080
|
+
};
|
|
2081
|
+
|
|
2068
2082
|
var formatRegExp = /%[sdj%]/g;
|
|
2069
2083
|
function format(f) {
|
|
2070
2084
|
if (!isString(f)) {
|
|
@@ -2550,6 +2564,64 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
2550
2564
|
return Object.prototype.hasOwnProperty.call(obj, prop);
|
|
2551
2565
|
}
|
|
2552
2566
|
|
|
2567
|
+
var kCustomPromisifiedSymbol = typeof Symbol !== 'undefined' ? Symbol('util.promisify.custom') : undefined;
|
|
2568
|
+
|
|
2569
|
+
function promisify(original) {
|
|
2570
|
+
if (typeof original !== 'function')
|
|
2571
|
+
throw new TypeError('The "original" argument must be of type Function');
|
|
2572
|
+
|
|
2573
|
+
if (kCustomPromisifiedSymbol && original[kCustomPromisifiedSymbol]) {
|
|
2574
|
+
var fn = original[kCustomPromisifiedSymbol];
|
|
2575
|
+
if (typeof fn !== 'function') {
|
|
2576
|
+
throw new TypeError('The "util.promisify.custom" argument must be of type Function');
|
|
2577
|
+
}
|
|
2578
|
+
Object.defineProperty(fn, kCustomPromisifiedSymbol, {
|
|
2579
|
+
value: fn, enumerable: false, writable: false, configurable: true
|
|
2580
|
+
});
|
|
2581
|
+
return fn;
|
|
2582
|
+
}
|
|
2583
|
+
|
|
2584
|
+
function fn() {
|
|
2585
|
+
var promiseResolve, promiseReject;
|
|
2586
|
+
var promise = new Promise(function (resolve, reject) {
|
|
2587
|
+
promiseResolve = resolve;
|
|
2588
|
+
promiseReject = reject;
|
|
2589
|
+
});
|
|
2590
|
+
|
|
2591
|
+
var args = [];
|
|
2592
|
+
for (var i = 0; i < arguments.length; i++) {
|
|
2593
|
+
args.push(arguments[i]);
|
|
2594
|
+
}
|
|
2595
|
+
args.push(function (err, value) {
|
|
2596
|
+
if (err) {
|
|
2597
|
+
promiseReject(err);
|
|
2598
|
+
} else {
|
|
2599
|
+
promiseResolve(value);
|
|
2600
|
+
}
|
|
2601
|
+
});
|
|
2602
|
+
|
|
2603
|
+
try {
|
|
2604
|
+
original.apply(this, args);
|
|
2605
|
+
} catch (err) {
|
|
2606
|
+
promiseReject(err);
|
|
2607
|
+
}
|
|
2608
|
+
|
|
2609
|
+
return promise;
|
|
2610
|
+
}
|
|
2611
|
+
|
|
2612
|
+
Object.setPrototypeOf(fn, Object.getPrototypeOf(original));
|
|
2613
|
+
|
|
2614
|
+
if (kCustomPromisifiedSymbol) Object.defineProperty(fn, kCustomPromisifiedSymbol, {
|
|
2615
|
+
value: fn, enumerable: false, writable: false, configurable: true
|
|
2616
|
+
});
|
|
2617
|
+
return Object.defineProperties(
|
|
2618
|
+
fn,
|
|
2619
|
+
getOwnPropertyDescriptors(original)
|
|
2620
|
+
);
|
|
2621
|
+
}
|
|
2622
|
+
|
|
2623
|
+
promisify.custom = kCustomPromisifiedSymbol;
|
|
2624
|
+
|
|
2553
2625
|
var lookup = [];
|
|
2554
2626
|
var revLookup = [];
|
|
2555
2627
|
var Arr = typeof Uint8Array !== 'undefined' ? Uint8Array : Array;
|
|
@@ -6800,186 +6872,1026 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
6800
6872
|
}
|
|
6801
6873
|
var memory_cache_class_default = MemoryCache;
|
|
6802
6874
|
|
|
6803
|
-
|
|
6804
|
-
|
|
6805
|
-
|
|
6806
|
-
|
|
6807
|
-
|
|
6808
|
-
|
|
6809
|
-
|
|
6875
|
+
var fs = {};
|
|
6876
|
+
|
|
6877
|
+
const readFile$1 = promisify(fs.readFile);
|
|
6878
|
+
const writeFile$1 = promisify(fs.writeFile);
|
|
6879
|
+
const unlink = promisify(fs.unlink);
|
|
6880
|
+
const readdir$1 = promisify(fs.readdir);
|
|
6881
|
+
const stat$1 = promisify(fs.stat);
|
|
6882
|
+
const mkdir = promisify(fs.mkdir);
|
|
6883
|
+
class FilesystemCache extends Cache {
|
|
6884
|
+
constructor({
|
|
6885
|
+
directory,
|
|
6886
|
+
prefix = "cache",
|
|
6887
|
+
ttl = 36e5,
|
|
6888
|
+
enableCompression = true,
|
|
6889
|
+
compressionThreshold = 1024,
|
|
6890
|
+
createDirectory = true,
|
|
6891
|
+
fileExtension = ".cache",
|
|
6892
|
+
enableMetadata = true,
|
|
6893
|
+
maxFileSize = 10485760,
|
|
6894
|
+
// 10MB
|
|
6895
|
+
enableStats = false,
|
|
6896
|
+
enableCleanup = true,
|
|
6897
|
+
cleanupInterval = 3e5,
|
|
6898
|
+
// 5 minutes
|
|
6899
|
+
encoding = "utf8",
|
|
6900
|
+
fileMode = 420,
|
|
6901
|
+
enableBackup = false,
|
|
6902
|
+
backupSuffix = ".bak",
|
|
6903
|
+
enableLocking = false,
|
|
6904
|
+
lockTimeout = 5e3,
|
|
6905
|
+
enableJournal = false,
|
|
6906
|
+
journalFile = "cache.journal",
|
|
6907
|
+
...config
|
|
6908
|
+
}) {
|
|
6909
|
+
super(config);
|
|
6910
|
+
if (!directory) {
|
|
6911
|
+
throw new Error("FilesystemCache: directory parameter is required");
|
|
6912
|
+
}
|
|
6913
|
+
this.directory = path.resolve(directory);
|
|
6914
|
+
this.prefix = prefix;
|
|
6915
|
+
this.ttl = ttl;
|
|
6916
|
+
this.enableCompression = enableCompression;
|
|
6917
|
+
this.compressionThreshold = compressionThreshold;
|
|
6918
|
+
this.createDirectory = createDirectory;
|
|
6919
|
+
this.fileExtension = fileExtension;
|
|
6920
|
+
this.enableMetadata = enableMetadata;
|
|
6921
|
+
this.maxFileSize = maxFileSize;
|
|
6922
|
+
this.enableStats = enableStats;
|
|
6923
|
+
this.enableCleanup = enableCleanup;
|
|
6924
|
+
this.cleanupInterval = cleanupInterval;
|
|
6925
|
+
this.encoding = encoding;
|
|
6926
|
+
this.fileMode = fileMode;
|
|
6927
|
+
this.enableBackup = enableBackup;
|
|
6928
|
+
this.backupSuffix = backupSuffix;
|
|
6929
|
+
this.enableLocking = enableLocking;
|
|
6930
|
+
this.lockTimeout = lockTimeout;
|
|
6931
|
+
this.enableJournal = enableJournal;
|
|
6932
|
+
this.journalFile = path.join(this.directory, journalFile);
|
|
6933
|
+
this.stats = {
|
|
6934
|
+
hits: 0,
|
|
6935
|
+
misses: 0,
|
|
6936
|
+
sets: 0,
|
|
6937
|
+
deletes: 0,
|
|
6938
|
+
clears: 0,
|
|
6939
|
+
errors: 0
|
|
6810
6940
|
};
|
|
6811
|
-
|
|
6812
|
-
|
|
6813
|
-
|
|
6814
|
-
}
|
|
6815
|
-
async
|
|
6816
|
-
if (this.
|
|
6817
|
-
this.
|
|
6818
|
-
}
|
|
6819
|
-
|
|
6820
|
-
|
|
6821
|
-
|
|
6941
|
+
this.locks = /* @__PURE__ */ new Map();
|
|
6942
|
+
this.cleanupTimer = null;
|
|
6943
|
+
this._init();
|
|
6944
|
+
}
|
|
6945
|
+
async _init() {
|
|
6946
|
+
if (this.createDirectory) {
|
|
6947
|
+
await this._ensureDirectory(this.directory);
|
|
6948
|
+
}
|
|
6949
|
+
if (this.enableCleanup && this.cleanupInterval > 0) {
|
|
6950
|
+
this.cleanupTimer = setInterval(() => {
|
|
6951
|
+
this._cleanup().catch((err) => {
|
|
6952
|
+
console.warn("FilesystemCache cleanup error:", err.message);
|
|
6953
|
+
});
|
|
6954
|
+
}, this.cleanupInterval);
|
|
6822
6955
|
}
|
|
6823
|
-
this.installDatabaseProxy();
|
|
6824
|
-
this.installResourceHooks();
|
|
6825
6956
|
}
|
|
6826
|
-
async
|
|
6957
|
+
async _ensureDirectory(dir) {
|
|
6958
|
+
const [ok, err] = await try_fn_default(async () => {
|
|
6959
|
+
await mkdir(dir, { recursive: true });
|
|
6960
|
+
});
|
|
6961
|
+
if (!ok && err.code !== "EEXIST") {
|
|
6962
|
+
throw new Error(`Failed to create cache directory: ${err.message}`);
|
|
6963
|
+
}
|
|
6827
6964
|
}
|
|
6828
|
-
|
|
6965
|
+
_getFilePath(key) {
|
|
6966
|
+
const sanitizedKey = key.replace(/[<>:"/\\|?*]/g, "_");
|
|
6967
|
+
const filename = `${this.prefix}_${sanitizedKey}${this.fileExtension}`;
|
|
6968
|
+
return path.join(this.directory, filename);
|
|
6829
6969
|
}
|
|
6830
|
-
|
|
6831
|
-
|
|
6832
|
-
return;
|
|
6833
|
-
}
|
|
6834
|
-
const installResourceHooks = this.installResourceHooks.bind(this);
|
|
6835
|
-
this.database._originalCreateResourceForCache = this.database.createResource;
|
|
6836
|
-
this.database.createResource = async function(...args) {
|
|
6837
|
-
const resource = await this._originalCreateResourceForCache(...args);
|
|
6838
|
-
installResourceHooks(resource);
|
|
6839
|
-
return resource;
|
|
6840
|
-
};
|
|
6841
|
-
this.database._cacheProxyInstalled = true;
|
|
6970
|
+
_getMetadataPath(filePath) {
|
|
6971
|
+
return filePath + ".meta";
|
|
6842
6972
|
}
|
|
6843
|
-
|
|
6844
|
-
|
|
6845
|
-
|
|
6973
|
+
async _set(key, data) {
|
|
6974
|
+
const filePath = this._getFilePath(key);
|
|
6975
|
+
try {
|
|
6976
|
+
let serialized = JSON.stringify(data);
|
|
6977
|
+
const originalSize = Buffer.byteLength(serialized, this.encoding);
|
|
6978
|
+
if (originalSize > this.maxFileSize) {
|
|
6979
|
+
throw new Error(`Cache data exceeds maximum file size: ${originalSize} > ${this.maxFileSize}`);
|
|
6980
|
+
}
|
|
6981
|
+
let compressed = false;
|
|
6982
|
+
let finalData = serialized;
|
|
6983
|
+
if (this.enableCompression && originalSize >= this.compressionThreshold) {
|
|
6984
|
+
const compressedBuffer = zlib.gzipSync(Buffer.from(serialized, this.encoding));
|
|
6985
|
+
finalData = compressedBuffer.toString("base64");
|
|
6986
|
+
compressed = true;
|
|
6987
|
+
}
|
|
6988
|
+
if (this.enableBackup && await this._fileExists(filePath)) {
|
|
6989
|
+
const backupPath = filePath + this.backupSuffix;
|
|
6990
|
+
await this._copyFile(filePath, backupPath);
|
|
6991
|
+
}
|
|
6992
|
+
if (this.enableLocking) {
|
|
6993
|
+
await this._acquireLock(filePath);
|
|
6994
|
+
}
|
|
6995
|
+
try {
|
|
6996
|
+
await writeFile$1(filePath, finalData, {
|
|
6997
|
+
encoding: compressed ? "utf8" : this.encoding,
|
|
6998
|
+
mode: this.fileMode
|
|
6999
|
+
});
|
|
7000
|
+
if (this.enableMetadata) {
|
|
7001
|
+
const metadata = {
|
|
7002
|
+
key,
|
|
7003
|
+
timestamp: Date.now(),
|
|
7004
|
+
ttl: this.ttl,
|
|
7005
|
+
compressed,
|
|
7006
|
+
originalSize,
|
|
7007
|
+
compressedSize: compressed ? Buffer.byteLength(finalData, "utf8") : originalSize,
|
|
7008
|
+
compressionRatio: compressed ? (Buffer.byteLength(finalData, "utf8") / originalSize).toFixed(2) : 1
|
|
7009
|
+
};
|
|
7010
|
+
await writeFile$1(this._getMetadataPath(filePath), JSON.stringify(metadata), {
|
|
7011
|
+
encoding: this.encoding,
|
|
7012
|
+
mode: this.fileMode
|
|
7013
|
+
});
|
|
7014
|
+
}
|
|
7015
|
+
if (this.enableStats) {
|
|
7016
|
+
this.stats.sets++;
|
|
7017
|
+
}
|
|
7018
|
+
if (this.enableJournal) {
|
|
7019
|
+
await this._journalOperation("set", key, { size: originalSize, compressed });
|
|
7020
|
+
}
|
|
7021
|
+
} finally {
|
|
7022
|
+
if (this.enableLocking) {
|
|
7023
|
+
this._releaseLock(filePath);
|
|
7024
|
+
}
|
|
7025
|
+
}
|
|
7026
|
+
return data;
|
|
7027
|
+
} catch (error) {
|
|
7028
|
+
if (this.enableStats) {
|
|
7029
|
+
this.stats.errors++;
|
|
7030
|
+
}
|
|
7031
|
+
throw new Error(`Failed to set cache key '${key}': ${error.message}`);
|
|
6846
7032
|
}
|
|
6847
7033
|
}
|
|
6848
|
-
|
|
6849
|
-
|
|
6850
|
-
|
|
6851
|
-
|
|
6852
|
-
|
|
6853
|
-
|
|
6854
|
-
enumerable: false
|
|
6855
|
-
});
|
|
6856
|
-
resource.cacheKeyFor = async (options = {}) => {
|
|
6857
|
-
const { action, params = {}, partition, partitionValues } = options;
|
|
6858
|
-
return this.generateCacheKey(resource, action, params, partition, partitionValues);
|
|
6859
|
-
};
|
|
6860
|
-
const cacheMethods = [
|
|
6861
|
-
"count",
|
|
6862
|
-
"listIds",
|
|
6863
|
-
"getMany",
|
|
6864
|
-
"getAll",
|
|
6865
|
-
"page",
|
|
6866
|
-
"list",
|
|
6867
|
-
"get"
|
|
6868
|
-
];
|
|
6869
|
-
for (const method of cacheMethods) {
|
|
6870
|
-
resource.useMiddleware(method, async (ctx, next) => {
|
|
6871
|
-
let key;
|
|
6872
|
-
if (method === "getMany") {
|
|
6873
|
-
key = await resource.cacheKeyFor({ action: method, params: { ids: ctx.args[0] } });
|
|
6874
|
-
} else if (method === "page") {
|
|
6875
|
-
const { offset, size, partition, partitionValues } = ctx.args[0] || {};
|
|
6876
|
-
key = await resource.cacheKeyFor({ action: method, params: { offset, size }, partition, partitionValues });
|
|
6877
|
-
} else if (method === "list" || method === "listIds" || method === "count") {
|
|
6878
|
-
const { partition, partitionValues } = ctx.args[0] || {};
|
|
6879
|
-
key = await resource.cacheKeyFor({ action: method, partition, partitionValues });
|
|
6880
|
-
} else if (method === "getAll") {
|
|
6881
|
-
key = await resource.cacheKeyFor({ action: method });
|
|
6882
|
-
} else if (method === "get") {
|
|
6883
|
-
key = await resource.cacheKeyFor({ action: method, params: { id: ctx.args[0] } });
|
|
7034
|
+
async _get(key) {
|
|
7035
|
+
const filePath = this._getFilePath(key);
|
|
7036
|
+
try {
|
|
7037
|
+
if (!await this._fileExists(filePath)) {
|
|
7038
|
+
if (this.enableStats) {
|
|
7039
|
+
this.stats.misses++;
|
|
6884
7040
|
}
|
|
6885
|
-
|
|
6886
|
-
|
|
6887
|
-
|
|
6888
|
-
|
|
6889
|
-
|
|
6890
|
-
|
|
6891
|
-
|
|
6892
|
-
|
|
6893
|
-
|
|
6894
|
-
|
|
6895
|
-
|
|
6896
|
-
|
|
6897
|
-
|
|
6898
|
-
await this.clearCacheForResource(resource, ctx.args[0]);
|
|
6899
|
-
} else if (method === "update") {
|
|
6900
|
-
await this.clearCacheForResource(resource, { id: ctx.args[0], ...ctx.args[1] });
|
|
6901
|
-
} else if (method === "delete") {
|
|
6902
|
-
let data = { id: ctx.args[0] };
|
|
6903
|
-
if (typeof resource.get === "function") {
|
|
6904
|
-
const [ok, err, full] = await try_fn_default(() => resource.get(ctx.args[0]));
|
|
6905
|
-
if (ok && full) data = full;
|
|
7041
|
+
return null;
|
|
7042
|
+
}
|
|
7043
|
+
let isExpired = false;
|
|
7044
|
+
if (this.enableMetadata) {
|
|
7045
|
+
const metadataPath = this._getMetadataPath(filePath);
|
|
7046
|
+
if (await this._fileExists(metadataPath)) {
|
|
7047
|
+
const [ok, err, metadata] = await try_fn_default(async () => {
|
|
7048
|
+
const metaContent = await readFile$1(metadataPath, this.encoding);
|
|
7049
|
+
return JSON.parse(metaContent);
|
|
7050
|
+
});
|
|
7051
|
+
if (ok && metadata.ttl > 0) {
|
|
7052
|
+
const age = Date.now() - metadata.timestamp;
|
|
7053
|
+
isExpired = age > metadata.ttl;
|
|
6906
7054
|
}
|
|
6907
|
-
await this.clearCacheForResource(resource, data);
|
|
6908
|
-
} else if (method === "deleteMany") {
|
|
6909
|
-
await this.clearCacheForResource(resource);
|
|
6910
7055
|
}
|
|
6911
|
-
|
|
6912
|
-
|
|
6913
|
-
|
|
6914
|
-
|
|
6915
|
-
|
|
6916
|
-
|
|
6917
|
-
|
|
6918
|
-
|
|
6919
|
-
|
|
6920
|
-
if (!data) {
|
|
6921
|
-
for (const partitionName of Object.keys(resource.config.partitions)) {
|
|
6922
|
-
const partitionKeyPrefix = join(keyPrefix, `partition=${partitionName}`);
|
|
6923
|
-
await resource.cache.clear(partitionKeyPrefix);
|
|
7056
|
+
} else if (this.ttl > 0) {
|
|
7057
|
+
const stats = await stat$1(filePath);
|
|
7058
|
+
const age = Date.now() - stats.mtime.getTime();
|
|
7059
|
+
isExpired = age > this.ttl;
|
|
7060
|
+
}
|
|
7061
|
+
if (isExpired) {
|
|
7062
|
+
await this._del(key);
|
|
7063
|
+
if (this.enableStats) {
|
|
7064
|
+
this.stats.misses++;
|
|
6924
7065
|
}
|
|
6925
|
-
|
|
6926
|
-
|
|
6927
|
-
|
|
6928
|
-
|
|
6929
|
-
|
|
6930
|
-
|
|
7066
|
+
return null;
|
|
7067
|
+
}
|
|
7068
|
+
if (this.enableLocking) {
|
|
7069
|
+
await this._acquireLock(filePath);
|
|
7070
|
+
}
|
|
7071
|
+
try {
|
|
7072
|
+
const content = await readFile$1(filePath, this.encoding);
|
|
7073
|
+
let isCompressed = false;
|
|
7074
|
+
if (this.enableMetadata) {
|
|
7075
|
+
const metadataPath = this._getMetadataPath(filePath);
|
|
7076
|
+
if (await this._fileExists(metadataPath)) {
|
|
7077
|
+
const [ok, err, metadata] = await try_fn_default(async () => {
|
|
7078
|
+
const metaContent = await readFile$1(metadataPath, this.encoding);
|
|
7079
|
+
return JSON.parse(metaContent);
|
|
7080
|
+
});
|
|
7081
|
+
if (ok) {
|
|
7082
|
+
isCompressed = metadata.compressed;
|
|
7083
|
+
}
|
|
7084
|
+
}
|
|
7085
|
+
}
|
|
7086
|
+
let finalContent = content;
|
|
7087
|
+
if (isCompressed || this.enableCompression && content.match(/^[A-Za-z0-9+/=]+$/)) {
|
|
7088
|
+
try {
|
|
7089
|
+
const compressedBuffer = Buffer.from(content, "base64");
|
|
7090
|
+
finalContent = zlib.gunzipSync(compressedBuffer).toString(this.encoding);
|
|
7091
|
+
} catch (decompressError) {
|
|
7092
|
+
finalContent = content;
|
|
6931
7093
|
}
|
|
6932
7094
|
}
|
|
7095
|
+
const data = JSON.parse(finalContent);
|
|
7096
|
+
if (this.enableStats) {
|
|
7097
|
+
this.stats.hits++;
|
|
7098
|
+
}
|
|
7099
|
+
return data;
|
|
7100
|
+
} finally {
|
|
7101
|
+
if (this.enableLocking) {
|
|
7102
|
+
this._releaseLock(filePath);
|
|
7103
|
+
}
|
|
6933
7104
|
}
|
|
7105
|
+
} catch (error) {
|
|
7106
|
+
if (this.enableStats) {
|
|
7107
|
+
this.stats.errors++;
|
|
7108
|
+
}
|
|
7109
|
+
await this._del(key);
|
|
7110
|
+
return null;
|
|
6934
7111
|
}
|
|
6935
7112
|
}
|
|
6936
|
-
async
|
|
6937
|
-
const
|
|
6938
|
-
|
|
6939
|
-
|
|
6940
|
-
|
|
6941
|
-
|
|
6942
|
-
|
|
6943
|
-
|
|
6944
|
-
if (
|
|
6945
|
-
|
|
7113
|
+
async _del(key) {
|
|
7114
|
+
const filePath = this._getFilePath(key);
|
|
7115
|
+
try {
|
|
7116
|
+
if (await this._fileExists(filePath)) {
|
|
7117
|
+
await unlink(filePath);
|
|
7118
|
+
}
|
|
7119
|
+
if (this.enableMetadata) {
|
|
7120
|
+
const metadataPath = this._getMetadataPath(filePath);
|
|
7121
|
+
if (await this._fileExists(metadataPath)) {
|
|
7122
|
+
await unlink(metadataPath);
|
|
6946
7123
|
}
|
|
6947
7124
|
}
|
|
7125
|
+
if (this.enableBackup) {
|
|
7126
|
+
const backupPath = filePath + this.backupSuffix;
|
|
7127
|
+
if (await this._fileExists(backupPath)) {
|
|
7128
|
+
await unlink(backupPath);
|
|
7129
|
+
}
|
|
7130
|
+
}
|
|
7131
|
+
if (this.enableStats) {
|
|
7132
|
+
this.stats.deletes++;
|
|
7133
|
+
}
|
|
7134
|
+
if (this.enableJournal) {
|
|
7135
|
+
await this._journalOperation("delete", key);
|
|
7136
|
+
}
|
|
7137
|
+
return true;
|
|
7138
|
+
} catch (error) {
|
|
7139
|
+
if (this.enableStats) {
|
|
7140
|
+
this.stats.errors++;
|
|
7141
|
+
}
|
|
7142
|
+
throw new Error(`Failed to delete cache key '${key}': ${error.message}`);
|
|
6948
7143
|
}
|
|
6949
|
-
if (Object.keys(params).length > 0) {
|
|
6950
|
-
const paramsHash = await this.hashParams(params);
|
|
6951
|
-
keyParts.push(paramsHash);
|
|
6952
|
-
}
|
|
6953
|
-
return join(...keyParts) + ".json.gz";
|
|
6954
|
-
}
|
|
6955
|
-
async hashParams(params) {
|
|
6956
|
-
const sortedParams = Object.keys(params).sort().map((key) => `${key}:${params[key]}`).join("|") || "empty";
|
|
6957
|
-
return await sha256(sortedParams);
|
|
6958
|
-
}
|
|
6959
|
-
// Utility methods
|
|
6960
|
-
async getCacheStats() {
|
|
6961
|
-
if (!this.driver) return null;
|
|
6962
|
-
return {
|
|
6963
|
-
size: await this.driver.size(),
|
|
6964
|
-
keys: await this.driver.keys(),
|
|
6965
|
-
driver: this.driver.constructor.name
|
|
6966
|
-
};
|
|
6967
7144
|
}
|
|
6968
|
-
async
|
|
6969
|
-
|
|
6970
|
-
|
|
6971
|
-
|
|
6972
|
-
|
|
6973
|
-
|
|
7145
|
+
async _clear(prefix) {
|
|
7146
|
+
try {
|
|
7147
|
+
const files = await readdir$1(this.directory);
|
|
7148
|
+
const cacheFiles = files.filter((file) => {
|
|
7149
|
+
if (!file.startsWith(this.prefix)) return false;
|
|
7150
|
+
if (!file.endsWith(this.fileExtension)) return false;
|
|
7151
|
+
if (prefix) {
|
|
7152
|
+
const keyPart = file.slice(this.prefix.length + 1, -this.fileExtension.length);
|
|
7153
|
+
return keyPart.startsWith(prefix);
|
|
7154
|
+
}
|
|
7155
|
+
return true;
|
|
7156
|
+
});
|
|
7157
|
+
for (const file of cacheFiles) {
|
|
7158
|
+
const filePath = path.join(this.directory, file);
|
|
7159
|
+
if (await this._fileExists(filePath)) {
|
|
7160
|
+
await unlink(filePath);
|
|
7161
|
+
}
|
|
7162
|
+
if (this.enableMetadata) {
|
|
7163
|
+
const metadataPath = this._getMetadataPath(filePath);
|
|
7164
|
+
if (await this._fileExists(metadataPath)) {
|
|
7165
|
+
await unlink(metadataPath);
|
|
7166
|
+
}
|
|
7167
|
+
}
|
|
7168
|
+
if (this.enableBackup) {
|
|
7169
|
+
const backupPath = filePath + this.backupSuffix;
|
|
7170
|
+
if (await this._fileExists(backupPath)) {
|
|
7171
|
+
await unlink(backupPath);
|
|
7172
|
+
}
|
|
7173
|
+
}
|
|
7174
|
+
}
|
|
7175
|
+
if (this.enableStats) {
|
|
7176
|
+
this.stats.clears++;
|
|
7177
|
+
}
|
|
7178
|
+
if (this.enableJournal) {
|
|
7179
|
+
await this._journalOperation("clear", prefix || "all", { count: cacheFiles.length });
|
|
7180
|
+
}
|
|
7181
|
+
return true;
|
|
7182
|
+
} catch (error) {
|
|
7183
|
+
if (this.enableStats) {
|
|
7184
|
+
this.stats.errors++;
|
|
6974
7185
|
}
|
|
7186
|
+
throw new Error(`Failed to clear cache: ${error.message}`);
|
|
6975
7187
|
}
|
|
6976
7188
|
}
|
|
6977
|
-
async
|
|
6978
|
-
const
|
|
6979
|
-
|
|
6980
|
-
|
|
6981
|
-
|
|
6982
|
-
|
|
7189
|
+
async size() {
|
|
7190
|
+
const keys = await this.keys();
|
|
7191
|
+
return keys.length;
|
|
7192
|
+
}
|
|
7193
|
+
async keys() {
|
|
7194
|
+
try {
|
|
7195
|
+
const files = await readdir$1(this.directory);
|
|
7196
|
+
const cacheFiles = files.filter(
|
|
7197
|
+
(file) => file.startsWith(this.prefix) && file.endsWith(this.fileExtension)
|
|
7198
|
+
);
|
|
7199
|
+
const keys = cacheFiles.map((file) => {
|
|
7200
|
+
const keyPart = file.slice(this.prefix.length + 1, -this.fileExtension.length);
|
|
7201
|
+
return keyPart;
|
|
7202
|
+
});
|
|
7203
|
+
return keys;
|
|
7204
|
+
} catch (error) {
|
|
7205
|
+
console.warn("FilesystemCache: Failed to list keys:", error.message);
|
|
7206
|
+
return [];
|
|
7207
|
+
}
|
|
7208
|
+
}
|
|
7209
|
+
// Helper methods
|
|
7210
|
+
async _fileExists(filePath) {
|
|
7211
|
+
const [ok] = await try_fn_default(async () => {
|
|
7212
|
+
await stat$1(filePath);
|
|
7213
|
+
});
|
|
7214
|
+
return ok;
|
|
7215
|
+
}
|
|
7216
|
+
async _copyFile(src, dest) {
|
|
7217
|
+
const [ok, err] = await try_fn_default(async () => {
|
|
7218
|
+
const content = await readFile$1(src);
|
|
7219
|
+
await writeFile$1(dest, content);
|
|
7220
|
+
});
|
|
7221
|
+
if (!ok) {
|
|
7222
|
+
console.warn("FilesystemCache: Failed to create backup:", err.message);
|
|
7223
|
+
}
|
|
7224
|
+
}
|
|
7225
|
+
async _cleanup() {
|
|
7226
|
+
if (!this.ttl || this.ttl <= 0) return;
|
|
7227
|
+
try {
|
|
7228
|
+
const files = await readdir$1(this.directory);
|
|
7229
|
+
const now = Date.now();
|
|
7230
|
+
for (const file of files) {
|
|
7231
|
+
if (!file.startsWith(this.prefix) || !file.endsWith(this.fileExtension)) {
|
|
7232
|
+
continue;
|
|
7233
|
+
}
|
|
7234
|
+
const filePath = path.join(this.directory, file);
|
|
7235
|
+
let shouldDelete = false;
|
|
7236
|
+
if (this.enableMetadata) {
|
|
7237
|
+
const metadataPath = this._getMetadataPath(filePath);
|
|
7238
|
+
if (await this._fileExists(metadataPath)) {
|
|
7239
|
+
const [ok, err, metadata] = await try_fn_default(async () => {
|
|
7240
|
+
const metaContent = await readFile$1(metadataPath, this.encoding);
|
|
7241
|
+
return JSON.parse(metaContent);
|
|
7242
|
+
});
|
|
7243
|
+
if (ok && metadata.ttl > 0) {
|
|
7244
|
+
const age = now - metadata.timestamp;
|
|
7245
|
+
shouldDelete = age > metadata.ttl;
|
|
7246
|
+
}
|
|
7247
|
+
}
|
|
7248
|
+
} else {
|
|
7249
|
+
const [ok, err, stats] = await try_fn_default(async () => {
|
|
7250
|
+
return await stat$1(filePath);
|
|
7251
|
+
});
|
|
7252
|
+
if (ok) {
|
|
7253
|
+
const age = now - stats.mtime.getTime();
|
|
7254
|
+
shouldDelete = age > this.ttl;
|
|
7255
|
+
}
|
|
7256
|
+
}
|
|
7257
|
+
if (shouldDelete) {
|
|
7258
|
+
const keyPart = file.slice(this.prefix.length + 1, -this.fileExtension.length);
|
|
7259
|
+
await this._del(keyPart);
|
|
7260
|
+
}
|
|
7261
|
+
}
|
|
7262
|
+
} catch (error) {
|
|
7263
|
+
console.warn("FilesystemCache cleanup error:", error.message);
|
|
7264
|
+
}
|
|
7265
|
+
}
|
|
7266
|
+
async _acquireLock(filePath) {
|
|
7267
|
+
if (!this.enableLocking) return;
|
|
7268
|
+
const lockKey = filePath;
|
|
7269
|
+
const startTime = Date.now();
|
|
7270
|
+
while (this.locks.has(lockKey)) {
|
|
7271
|
+
if (Date.now() - startTime > this.lockTimeout) {
|
|
7272
|
+
throw new Error(`Lock timeout for file: ${filePath}`);
|
|
7273
|
+
}
|
|
7274
|
+
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
7275
|
+
}
|
|
7276
|
+
this.locks.set(lockKey, Date.now());
|
|
7277
|
+
}
|
|
7278
|
+
_releaseLock(filePath) {
|
|
7279
|
+
if (!this.enableLocking) return;
|
|
7280
|
+
this.locks.delete(filePath);
|
|
7281
|
+
}
|
|
7282
|
+
async _journalOperation(operation, key, metadata = {}) {
|
|
7283
|
+
if (!this.enableJournal) return;
|
|
7284
|
+
const entry = {
|
|
7285
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
7286
|
+
operation,
|
|
7287
|
+
key,
|
|
7288
|
+
metadata
|
|
7289
|
+
};
|
|
7290
|
+
const [ok, err] = await try_fn_default(async () => {
|
|
7291
|
+
const line = JSON.stringify(entry) + "\n";
|
|
7292
|
+
await fs.promises.appendFile(this.journalFile, line, this.encoding);
|
|
7293
|
+
});
|
|
7294
|
+
if (!ok) {
|
|
7295
|
+
console.warn("FilesystemCache journal error:", err.message);
|
|
7296
|
+
}
|
|
7297
|
+
}
|
|
7298
|
+
// Cleanup on process exit
|
|
7299
|
+
destroy() {
|
|
7300
|
+
if (this.cleanupTimer) {
|
|
7301
|
+
clearInterval(this.cleanupTimer);
|
|
7302
|
+
this.cleanupTimer = null;
|
|
7303
|
+
}
|
|
7304
|
+
}
|
|
7305
|
+
// Get cache statistics
|
|
7306
|
+
getStats() {
|
|
7307
|
+
return {
|
|
7308
|
+
...this.stats,
|
|
7309
|
+
directory: this.directory,
|
|
7310
|
+
ttl: this.ttl,
|
|
7311
|
+
compression: this.enableCompression,
|
|
7312
|
+
metadata: this.enableMetadata,
|
|
7313
|
+
cleanup: this.enableCleanup,
|
|
7314
|
+
locking: this.enableLocking,
|
|
7315
|
+
journal: this.enableJournal
|
|
7316
|
+
};
|
|
7317
|
+
}
|
|
7318
|
+
}
|
|
7319
|
+
|
|
7320
|
+
promisify(fs.mkdir);
|
|
7321
|
+
const rmdir = promisify(fs.rm);
|
|
7322
|
+
const readdir = promisify(fs.readdir);
|
|
7323
|
+
const stat = promisify(fs.stat);
|
|
7324
|
+
const writeFile = promisify(fs.writeFile);
|
|
7325
|
+
const readFile = promisify(fs.readFile);
|
|
7326
|
+
class PartitionAwareFilesystemCache extends FilesystemCache {
|
|
7327
|
+
constructor({
|
|
7328
|
+
partitionStrategy = "hierarchical",
|
|
7329
|
+
// 'hierarchical', 'flat', 'temporal'
|
|
7330
|
+
trackUsage = true,
|
|
7331
|
+
preloadRelated = false,
|
|
7332
|
+
preloadThreshold = 10,
|
|
7333
|
+
maxCacheSize = null,
|
|
7334
|
+
usageStatsFile = "partition-usage.json",
|
|
7335
|
+
...config
|
|
7336
|
+
}) {
|
|
7337
|
+
super(config);
|
|
7338
|
+
this.partitionStrategy = partitionStrategy;
|
|
7339
|
+
this.trackUsage = trackUsage;
|
|
7340
|
+
this.preloadRelated = preloadRelated;
|
|
7341
|
+
this.preloadThreshold = preloadThreshold;
|
|
7342
|
+
this.maxCacheSize = maxCacheSize;
|
|
7343
|
+
this.usageStatsFile = path.join(this.directory, usageStatsFile);
|
|
7344
|
+
this.partitionUsage = /* @__PURE__ */ new Map();
|
|
7345
|
+
this.loadUsageStats();
|
|
7346
|
+
}
|
|
7347
|
+
/**
|
|
7348
|
+
* Generate partition-aware cache key
|
|
7349
|
+
*/
|
|
7350
|
+
_getPartitionCacheKey(resource, action, partition, partitionValues = {}, params = {}) {
|
|
7351
|
+
const keyParts = [`resource=${resource}`, `action=${action}`];
|
|
7352
|
+
if (partition && Object.keys(partitionValues).length > 0) {
|
|
7353
|
+
keyParts.push(`partition=${partition}`);
|
|
7354
|
+
const sortedFields = Object.entries(partitionValues).sort(([a], [b]) => a.localeCompare(b));
|
|
7355
|
+
for (const [field, value] of sortedFields) {
|
|
7356
|
+
if (value !== null && value !== void 0) {
|
|
7357
|
+
keyParts.push(`${field}=${value}`);
|
|
7358
|
+
}
|
|
7359
|
+
}
|
|
7360
|
+
}
|
|
7361
|
+
if (Object.keys(params).length > 0) {
|
|
7362
|
+
const paramsStr = Object.entries(params).sort(([a], [b]) => a.localeCompare(b)).map(([k, v]) => `${k}=${v}`).join("|");
|
|
7363
|
+
keyParts.push(`params=${Buffer.from(paramsStr).toString("base64")}`);
|
|
7364
|
+
}
|
|
7365
|
+
return keyParts.join("/") + this.fileExtension;
|
|
7366
|
+
}
|
|
7367
|
+
/**
|
|
7368
|
+
* Get directory path for partition cache
|
|
7369
|
+
*/
|
|
7370
|
+
_getPartitionDirectory(resource, partition, partitionValues = {}) {
|
|
7371
|
+
const basePath = path.join(this.directory, `resource=${resource}`);
|
|
7372
|
+
if (!partition) {
|
|
7373
|
+
return basePath;
|
|
7374
|
+
}
|
|
7375
|
+
if (this.partitionStrategy === "flat") {
|
|
7376
|
+
return path.join(basePath, "partitions");
|
|
7377
|
+
}
|
|
7378
|
+
if (this.partitionStrategy === "temporal" && this._isTemporalPartition(partition, partitionValues)) {
|
|
7379
|
+
return this._getTemporalDirectory(basePath, partition, partitionValues);
|
|
7380
|
+
}
|
|
7381
|
+
const pathParts = [basePath, `partition=${partition}`];
|
|
7382
|
+
const sortedFields = Object.entries(partitionValues).sort(([a], [b]) => a.localeCompare(b));
|
|
7383
|
+
for (const [field, value] of sortedFields) {
|
|
7384
|
+
if (value !== null && value !== void 0) {
|
|
7385
|
+
pathParts.push(`${field}=${this._sanitizePathValue(value)}`);
|
|
7386
|
+
}
|
|
7387
|
+
}
|
|
7388
|
+
return path.join(...pathParts);
|
|
7389
|
+
}
|
|
7390
|
+
/**
|
|
7391
|
+
* Enhanced set method with partition awareness
|
|
7392
|
+
*/
|
|
7393
|
+
async _set(key, data, options = {}) {
|
|
7394
|
+
const { resource, action, partition, partitionValues, params } = options;
|
|
7395
|
+
if (resource && partition) {
|
|
7396
|
+
const partitionKey = this._getPartitionCacheKey(resource, action, partition, partitionValues, params);
|
|
7397
|
+
const partitionDir = this._getPartitionDirectory(resource, partition, partitionValues);
|
|
7398
|
+
await this._ensureDirectory(partitionDir);
|
|
7399
|
+
const filePath = path.join(partitionDir, this._sanitizeFileName(partitionKey));
|
|
7400
|
+
if (this.trackUsage) {
|
|
7401
|
+
await this._trackPartitionUsage(resource, partition, partitionValues);
|
|
7402
|
+
}
|
|
7403
|
+
const partitionData = {
|
|
7404
|
+
data,
|
|
7405
|
+
metadata: {
|
|
7406
|
+
resource,
|
|
7407
|
+
partition,
|
|
7408
|
+
partitionValues,
|
|
7409
|
+
timestamp: Date.now(),
|
|
7410
|
+
ttl: this.ttl
|
|
7411
|
+
}
|
|
7412
|
+
};
|
|
7413
|
+
return this._writeFileWithMetadata(filePath, partitionData);
|
|
7414
|
+
}
|
|
7415
|
+
return super._set(key, data);
|
|
7416
|
+
}
|
|
7417
|
+
/**
|
|
7418
|
+
* Enhanced get method with partition awareness
|
|
7419
|
+
*/
|
|
7420
|
+
async _get(key, options = {}) {
|
|
7421
|
+
const { resource, action, partition, partitionValues, params } = options;
|
|
7422
|
+
if (resource && partition) {
|
|
7423
|
+
const partitionKey = this._getPartitionCacheKey(resource, action, partition, partitionValues, params);
|
|
7424
|
+
const partitionDir = this._getPartitionDirectory(resource, partition, partitionValues);
|
|
7425
|
+
const filePath = path.join(partitionDir, this._sanitizeFileName(partitionKey));
|
|
7426
|
+
if (!await this._fileExists(filePath)) {
|
|
7427
|
+
if (this.preloadRelated) {
|
|
7428
|
+
await this._preloadRelatedPartitions(resource, partition, partitionValues);
|
|
7429
|
+
}
|
|
7430
|
+
return null;
|
|
7431
|
+
}
|
|
7432
|
+
const result = await this._readFileWithMetadata(filePath);
|
|
7433
|
+
if (result && this.trackUsage) {
|
|
7434
|
+
await this._trackPartitionUsage(resource, partition, partitionValues);
|
|
7435
|
+
}
|
|
7436
|
+
return result?.data || null;
|
|
7437
|
+
}
|
|
7438
|
+
return super._get(key);
|
|
7439
|
+
}
|
|
7440
|
+
/**
|
|
7441
|
+
* Clear cache for specific partition
|
|
7442
|
+
*/
|
|
7443
|
+
async clearPartition(resource, partition, partitionValues = {}) {
|
|
7444
|
+
const partitionDir = this._getPartitionDirectory(resource, partition, partitionValues);
|
|
7445
|
+
const [ok, err] = await try_fn_default(async () => {
|
|
7446
|
+
if (await this._fileExists(partitionDir)) {
|
|
7447
|
+
await rmdir(partitionDir, { recursive: true });
|
|
7448
|
+
}
|
|
7449
|
+
});
|
|
7450
|
+
if (!ok) {
|
|
7451
|
+
console.warn(`Failed to clear partition cache: ${err.message}`);
|
|
7452
|
+
}
|
|
7453
|
+
const usageKey = this._getUsageKey(resource, partition, partitionValues);
|
|
7454
|
+
this.partitionUsage.delete(usageKey);
|
|
7455
|
+
await this._saveUsageStats();
|
|
7456
|
+
return ok;
|
|
7457
|
+
}
|
|
7458
|
+
/**
|
|
7459
|
+
* Clear all partitions for a resource
|
|
7460
|
+
*/
|
|
7461
|
+
async clearResourcePartitions(resource) {
|
|
7462
|
+
const resourceDir = path.join(this.directory, `resource=${resource}`);
|
|
7463
|
+
const [ok, err] = await try_fn_default(async () => {
|
|
7464
|
+
if (await this._fileExists(resourceDir)) {
|
|
7465
|
+
await rmdir(resourceDir, { recursive: true });
|
|
7466
|
+
}
|
|
7467
|
+
});
|
|
7468
|
+
for (const [key] of this.partitionUsage.entries()) {
|
|
7469
|
+
if (key.startsWith(`${resource}/`)) {
|
|
7470
|
+
this.partitionUsage.delete(key);
|
|
7471
|
+
}
|
|
7472
|
+
}
|
|
7473
|
+
await this._saveUsageStats();
|
|
7474
|
+
return ok;
|
|
7475
|
+
}
|
|
7476
|
+
/**
|
|
7477
|
+
* Get partition cache statistics
|
|
7478
|
+
*/
|
|
7479
|
+
async getPartitionStats(resource, partition = null) {
|
|
7480
|
+
const stats = {
|
|
7481
|
+
totalFiles: 0,
|
|
7482
|
+
totalSize: 0,
|
|
7483
|
+
partitions: {},
|
|
7484
|
+
usage: {}
|
|
7485
|
+
};
|
|
7486
|
+
const resourceDir = path.join(this.directory, `resource=${resource}`);
|
|
7487
|
+
if (!await this._fileExists(resourceDir)) {
|
|
7488
|
+
return stats;
|
|
7489
|
+
}
|
|
7490
|
+
await this._calculateDirectoryStats(resourceDir, stats);
|
|
7491
|
+
for (const [key, usage] of this.partitionUsage.entries()) {
|
|
7492
|
+
if (key.startsWith(`${resource}/`)) {
|
|
7493
|
+
const partitionName = key.split("/")[1];
|
|
7494
|
+
if (!partition || partitionName === partition) {
|
|
7495
|
+
stats.usage[partitionName] = usage;
|
|
7496
|
+
}
|
|
7497
|
+
}
|
|
7498
|
+
}
|
|
7499
|
+
return stats;
|
|
7500
|
+
}
|
|
7501
|
+
/**
|
|
7502
|
+
* Get cache recommendations based on usage patterns
|
|
7503
|
+
*/
|
|
7504
|
+
async getCacheRecommendations(resource) {
|
|
7505
|
+
const recommendations = [];
|
|
7506
|
+
const now = Date.now();
|
|
7507
|
+
const dayMs = 24 * 60 * 60 * 1e3;
|
|
7508
|
+
for (const [key, usage] of this.partitionUsage.entries()) {
|
|
7509
|
+
if (key.startsWith(`${resource}/`)) {
|
|
7510
|
+
const [, partition] = key.split("/");
|
|
7511
|
+
const daysSinceLastAccess = (now - usage.lastAccess) / dayMs;
|
|
7512
|
+
const accessesPerDay = usage.count / Math.max(1, daysSinceLastAccess);
|
|
7513
|
+
let recommendation = "keep";
|
|
7514
|
+
let priority = usage.count;
|
|
7515
|
+
if (daysSinceLastAccess > 30) {
|
|
7516
|
+
recommendation = "archive";
|
|
7517
|
+
priority = 0;
|
|
7518
|
+
} else if (accessesPerDay < 0.1) {
|
|
7519
|
+
recommendation = "reduce_ttl";
|
|
7520
|
+
priority = 1;
|
|
7521
|
+
} else if (accessesPerDay > 10) {
|
|
7522
|
+
recommendation = "preload";
|
|
7523
|
+
priority = 100;
|
|
7524
|
+
}
|
|
7525
|
+
recommendations.push({
|
|
7526
|
+
partition,
|
|
7527
|
+
recommendation,
|
|
7528
|
+
priority,
|
|
7529
|
+
usage: accessesPerDay,
|
|
7530
|
+
lastAccess: new Date(usage.lastAccess).toISOString()
|
|
7531
|
+
});
|
|
7532
|
+
}
|
|
7533
|
+
}
|
|
7534
|
+
return recommendations.sort((a, b) => b.priority - a.priority);
|
|
7535
|
+
}
|
|
7536
|
+
/**
|
|
7537
|
+
* Preload frequently accessed partitions
|
|
7538
|
+
*/
|
|
7539
|
+
async warmPartitionCache(resource, options = {}) {
|
|
7540
|
+
const { partitions = [], maxFiles = 1e3 } = options;
|
|
7541
|
+
let warmedCount = 0;
|
|
7542
|
+
for (const partition of partitions) {
|
|
7543
|
+
const usageKey = `${resource}/${partition}`;
|
|
7544
|
+
const usage = this.partitionUsage.get(usageKey);
|
|
7545
|
+
if (usage && usage.count >= this.preloadThreshold) {
|
|
7546
|
+
console.log(`\u{1F525} Warming cache for ${resource}/${partition} (${usage.count} accesses)`);
|
|
7547
|
+
warmedCount++;
|
|
7548
|
+
}
|
|
7549
|
+
if (warmedCount >= maxFiles) break;
|
|
7550
|
+
}
|
|
7551
|
+
return warmedCount;
|
|
7552
|
+
}
|
|
7553
|
+
// Private helper methods
|
|
7554
|
+
async _trackPartitionUsage(resource, partition, partitionValues) {
|
|
7555
|
+
const usageKey = this._getUsageKey(resource, partition, partitionValues);
|
|
7556
|
+
const current = this.partitionUsage.get(usageKey) || {
|
|
7557
|
+
count: 0,
|
|
7558
|
+
firstAccess: Date.now(),
|
|
7559
|
+
lastAccess: Date.now()
|
|
7560
|
+
};
|
|
7561
|
+
current.count++;
|
|
7562
|
+
current.lastAccess = Date.now();
|
|
7563
|
+
this.partitionUsage.set(usageKey, current);
|
|
7564
|
+
if (current.count % 10 === 0) {
|
|
7565
|
+
await this._saveUsageStats();
|
|
7566
|
+
}
|
|
7567
|
+
}
|
|
7568
|
+
_getUsageKey(resource, partition, partitionValues) {
|
|
7569
|
+
const valuePart = Object.entries(partitionValues).sort(([a], [b]) => a.localeCompare(b)).map(([k, v]) => `${k}=${v}`).join("|");
|
|
7570
|
+
return `${resource}/${partition}/${valuePart}`;
|
|
7571
|
+
}
|
|
7572
|
+
async _preloadRelatedPartitions(resource, partition, partitionValues) {
|
|
7573
|
+
console.log(`\u{1F3AF} Preloading related partitions for ${resource}/${partition}`);
|
|
7574
|
+
if (partitionValues.timestamp || partitionValues.date) ;
|
|
7575
|
+
}
|
|
7576
|
+
_isTemporalPartition(partition, partitionValues) {
|
|
7577
|
+
const temporalFields = ["date", "timestamp", "createdAt", "updatedAt"];
|
|
7578
|
+
return Object.keys(partitionValues).some(
|
|
7579
|
+
(field) => temporalFields.some((tf) => field.toLowerCase().includes(tf))
|
|
7580
|
+
);
|
|
7581
|
+
}
|
|
7582
|
+
_getTemporalDirectory(basePath, partition, partitionValues) {
|
|
7583
|
+
const dateValue = Object.values(partitionValues)[0];
|
|
7584
|
+
if (typeof dateValue === "string" && dateValue.match(/^\d{4}-\d{2}-\d{2}/)) {
|
|
7585
|
+
const [year, month, day] = dateValue.split("-");
|
|
7586
|
+
return path.join(basePath, "temporal", year, month, day);
|
|
7587
|
+
}
|
|
7588
|
+
return path.join(basePath, `partition=${partition}`);
|
|
7589
|
+
}
|
|
7590
|
+
_sanitizePathValue(value) {
|
|
7591
|
+
return String(value).replace(/[<>:"/\\|?*]/g, "_");
|
|
7592
|
+
}
|
|
7593
|
+
_sanitizeFileName(filename) {
|
|
7594
|
+
return filename.replace(/[<>:"/\\|?*]/g, "_");
|
|
7595
|
+
}
|
|
7596
|
+
async _calculateDirectoryStats(dir, stats) {
|
|
7597
|
+
const [ok, err, files] = await try_fn_default(() => readdir(dir));
|
|
7598
|
+
if (!ok) return;
|
|
7599
|
+
for (const file of files) {
|
|
7600
|
+
const filePath = path.join(dir, file);
|
|
7601
|
+
const [statOk, statErr, fileStat] = await try_fn_default(() => stat(filePath));
|
|
7602
|
+
if (statOk) {
|
|
7603
|
+
if (fileStat.isDirectory()) {
|
|
7604
|
+
await this._calculateDirectoryStats(filePath, stats);
|
|
7605
|
+
} else {
|
|
7606
|
+
stats.totalFiles++;
|
|
7607
|
+
stats.totalSize += fileStat.size;
|
|
7608
|
+
}
|
|
7609
|
+
}
|
|
7610
|
+
}
|
|
7611
|
+
}
|
|
7612
|
+
async loadUsageStats() {
|
|
7613
|
+
const [ok, err, content] = await try_fn_default(async () => {
|
|
7614
|
+
const data = await readFile(this.usageStatsFile, "utf8");
|
|
7615
|
+
return JSON.parse(data);
|
|
7616
|
+
});
|
|
7617
|
+
if (ok && content) {
|
|
7618
|
+
this.partitionUsage = new Map(Object.entries(content));
|
|
7619
|
+
}
|
|
7620
|
+
}
|
|
7621
|
+
async _saveUsageStats() {
|
|
7622
|
+
const statsObject = Object.fromEntries(this.partitionUsage);
|
|
7623
|
+
await try_fn_default(async () => {
|
|
7624
|
+
await writeFile(
|
|
7625
|
+
this.usageStatsFile,
|
|
7626
|
+
JSON.stringify(statsObject, null, 2),
|
|
7627
|
+
"utf8"
|
|
7628
|
+
);
|
|
7629
|
+
});
|
|
7630
|
+
}
|
|
7631
|
+
async _writeFileWithMetadata(filePath, data) {
|
|
7632
|
+
const content = JSON.stringify(data);
|
|
7633
|
+
const [ok, err] = await try_fn_default(async () => {
|
|
7634
|
+
await writeFile(filePath, content, {
|
|
7635
|
+
encoding: this.encoding,
|
|
7636
|
+
mode: this.fileMode
|
|
7637
|
+
});
|
|
7638
|
+
});
|
|
7639
|
+
if (!ok) {
|
|
7640
|
+
throw new Error(`Failed to write cache file: ${err.message}`);
|
|
7641
|
+
}
|
|
7642
|
+
return true;
|
|
7643
|
+
}
|
|
7644
|
+
async _readFileWithMetadata(filePath) {
|
|
7645
|
+
const [ok, err, content] = await try_fn_default(async () => {
|
|
7646
|
+
return await readFile(filePath, this.encoding);
|
|
7647
|
+
});
|
|
7648
|
+
if (!ok || !content) return null;
|
|
7649
|
+
try {
|
|
7650
|
+
return JSON.parse(content);
|
|
7651
|
+
} catch (error) {
|
|
7652
|
+
return { data: content };
|
|
7653
|
+
}
|
|
7654
|
+
}
|
|
7655
|
+
}
|
|
7656
|
+
|
|
7657
|
+
class CachePlugin extends plugin_class_default {
|
|
7658
|
+
constructor(options = {}) {
|
|
7659
|
+
super(options);
|
|
7660
|
+
this.driver = options.driver;
|
|
7661
|
+
this.config = {
|
|
7662
|
+
includePartitions: options.includePartitions !== false,
|
|
7663
|
+
partitionStrategy: options.partitionStrategy || "hierarchical",
|
|
7664
|
+
partitionAware: options.partitionAware !== false,
|
|
7665
|
+
trackUsage: options.trackUsage !== false,
|
|
7666
|
+
preloadRelated: options.preloadRelated !== false,
|
|
7667
|
+
...options
|
|
7668
|
+
};
|
|
7669
|
+
}
|
|
7670
|
+
async setup(database) {
|
|
7671
|
+
await super.setup(database);
|
|
7672
|
+
}
|
|
7673
|
+
async onSetup() {
|
|
7674
|
+
if (this.config.driver) {
|
|
7675
|
+
this.driver = this.config.driver;
|
|
7676
|
+
} else if (this.config.driverType === "memory") {
|
|
7677
|
+
this.driver = new memory_cache_class_default(this.config.memoryOptions || {});
|
|
7678
|
+
} else if (this.config.driverType === "filesystem") {
|
|
7679
|
+
if (this.config.partitionAware) {
|
|
7680
|
+
this.driver = new PartitionAwareFilesystemCache({
|
|
7681
|
+
partitionStrategy: this.config.partitionStrategy,
|
|
7682
|
+
trackUsage: this.config.trackUsage,
|
|
7683
|
+
preloadRelated: this.config.preloadRelated,
|
|
7684
|
+
...this.config.filesystemOptions
|
|
7685
|
+
});
|
|
7686
|
+
} else {
|
|
7687
|
+
this.driver = new FilesystemCache(this.config.filesystemOptions || {});
|
|
7688
|
+
}
|
|
7689
|
+
} else {
|
|
7690
|
+
this.driver = new s3_cache_class_default({ client: this.database.client, ...this.config.s3Options || {} });
|
|
7691
|
+
}
|
|
7692
|
+
this.installDatabaseProxy();
|
|
7693
|
+
this.installResourceHooks();
|
|
7694
|
+
}
|
|
7695
|
+
async onStart() {
|
|
7696
|
+
}
|
|
7697
|
+
async onStop() {
|
|
7698
|
+
}
|
|
7699
|
+
installDatabaseProxy() {
|
|
7700
|
+
if (this.database._cacheProxyInstalled) {
|
|
7701
|
+
return;
|
|
7702
|
+
}
|
|
7703
|
+
const installResourceHooks = this.installResourceHooks.bind(this);
|
|
7704
|
+
this.database._originalCreateResourceForCache = this.database.createResource;
|
|
7705
|
+
this.database.createResource = async function(...args) {
|
|
7706
|
+
const resource = await this._originalCreateResourceForCache(...args);
|
|
7707
|
+
installResourceHooks(resource);
|
|
7708
|
+
return resource;
|
|
7709
|
+
};
|
|
7710
|
+
this.database._cacheProxyInstalled = true;
|
|
7711
|
+
}
|
|
7712
|
+
installResourceHooks() {
|
|
7713
|
+
for (const resource of Object.values(this.database.resources)) {
|
|
7714
|
+
this.installResourceHooksForResource(resource);
|
|
7715
|
+
}
|
|
7716
|
+
}
|
|
7717
|
+
installResourceHooksForResource(resource) {
|
|
7718
|
+
if (!this.driver) return;
|
|
7719
|
+
Object.defineProperty(resource, "cache", {
|
|
7720
|
+
value: this.driver,
|
|
7721
|
+
writable: true,
|
|
7722
|
+
configurable: true,
|
|
7723
|
+
enumerable: false
|
|
7724
|
+
});
|
|
7725
|
+
resource.cacheKeyFor = async (options = {}) => {
|
|
7726
|
+
const { action, params = {}, partition, partitionValues } = options;
|
|
7727
|
+
return this.generateCacheKey(resource, action, params, partition, partitionValues);
|
|
7728
|
+
};
|
|
7729
|
+
if (this.driver instanceof PartitionAwareFilesystemCache) {
|
|
7730
|
+
resource.clearPartitionCache = async (partition, partitionValues = {}) => {
|
|
7731
|
+
return await this.driver.clearPartition(resource.name, partition, partitionValues);
|
|
7732
|
+
};
|
|
7733
|
+
resource.getPartitionCacheStats = async (partition = null) => {
|
|
7734
|
+
return await this.driver.getPartitionStats(resource.name, partition);
|
|
7735
|
+
};
|
|
7736
|
+
resource.getCacheRecommendations = async () => {
|
|
7737
|
+
return await this.driver.getCacheRecommendations(resource.name);
|
|
7738
|
+
};
|
|
7739
|
+
resource.warmPartitionCache = async (partitions = [], options = {}) => {
|
|
7740
|
+
return await this.driver.warmPartitionCache(resource.name, { partitions, ...options });
|
|
7741
|
+
};
|
|
7742
|
+
}
|
|
7743
|
+
const cacheMethods = [
|
|
7744
|
+
"count",
|
|
7745
|
+
"listIds",
|
|
7746
|
+
"getMany",
|
|
7747
|
+
"getAll",
|
|
7748
|
+
"page",
|
|
7749
|
+
"list",
|
|
7750
|
+
"get"
|
|
7751
|
+
];
|
|
7752
|
+
for (const method of cacheMethods) {
|
|
7753
|
+
resource.useMiddleware(method, async (ctx, next) => {
|
|
7754
|
+
let key;
|
|
7755
|
+
if (method === "getMany") {
|
|
7756
|
+
key = await resource.cacheKeyFor({ action: method, params: { ids: ctx.args[0] } });
|
|
7757
|
+
} else if (method === "page") {
|
|
7758
|
+
const { offset, size, partition, partitionValues } = ctx.args[0] || {};
|
|
7759
|
+
key = await resource.cacheKeyFor({ action: method, params: { offset, size }, partition, partitionValues });
|
|
7760
|
+
} else if (method === "list" || method === "listIds" || method === "count") {
|
|
7761
|
+
const { partition, partitionValues } = ctx.args[0] || {};
|
|
7762
|
+
key = await resource.cacheKeyFor({ action: method, partition, partitionValues });
|
|
7763
|
+
} else if (method === "getAll") {
|
|
7764
|
+
key = await resource.cacheKeyFor({ action: method });
|
|
7765
|
+
} else if (method === "get") {
|
|
7766
|
+
key = await resource.cacheKeyFor({ action: method, params: { id: ctx.args[0] } });
|
|
7767
|
+
}
|
|
7768
|
+
if (this.driver instanceof PartitionAwareFilesystemCache) {
|
|
7769
|
+
let partition, partitionValues;
|
|
7770
|
+
if (method === "list" || method === "listIds" || method === "count" || method === "page") {
|
|
7771
|
+
const args = ctx.args[0] || {};
|
|
7772
|
+
partition = args.partition;
|
|
7773
|
+
partitionValues = args.partitionValues;
|
|
7774
|
+
}
|
|
7775
|
+
const [ok, err, result] = await try_fn_default(() => resource.cache._get(key, {
|
|
7776
|
+
resource: resource.name,
|
|
7777
|
+
action: method,
|
|
7778
|
+
partition,
|
|
7779
|
+
partitionValues
|
|
7780
|
+
}));
|
|
7781
|
+
if (ok && result !== null && result !== void 0) return result;
|
|
7782
|
+
if (!ok && err.name !== "NoSuchKey") throw err;
|
|
7783
|
+
const freshResult = await next();
|
|
7784
|
+
await resource.cache._set(key, freshResult, {
|
|
7785
|
+
resource: resource.name,
|
|
7786
|
+
action: method,
|
|
7787
|
+
partition,
|
|
7788
|
+
partitionValues
|
|
7789
|
+
});
|
|
7790
|
+
return freshResult;
|
|
7791
|
+
} else {
|
|
7792
|
+
const [ok, err, result] = await try_fn_default(() => resource.cache.get(key));
|
|
7793
|
+
if (ok && result !== null && result !== void 0) return result;
|
|
7794
|
+
if (!ok && err.name !== "NoSuchKey") throw err;
|
|
7795
|
+
const freshResult = await next();
|
|
7796
|
+
await resource.cache.set(key, freshResult);
|
|
7797
|
+
return freshResult;
|
|
7798
|
+
}
|
|
7799
|
+
});
|
|
7800
|
+
}
|
|
7801
|
+
const writeMethods = ["insert", "update", "delete", "deleteMany"];
|
|
7802
|
+
for (const method of writeMethods) {
|
|
7803
|
+
resource.useMiddleware(method, async (ctx, next) => {
|
|
7804
|
+
const result = await next();
|
|
7805
|
+
if (method === "insert") {
|
|
7806
|
+
await this.clearCacheForResource(resource, ctx.args[0]);
|
|
7807
|
+
} else if (method === "update") {
|
|
7808
|
+
await this.clearCacheForResource(resource, { id: ctx.args[0], ...ctx.args[1] });
|
|
7809
|
+
} else if (method === "delete") {
|
|
7810
|
+
let data = { id: ctx.args[0] };
|
|
7811
|
+
if (typeof resource.get === "function") {
|
|
7812
|
+
const [ok, err, full] = await try_fn_default(() => resource.get(ctx.args[0]));
|
|
7813
|
+
if (ok && full) data = full;
|
|
7814
|
+
}
|
|
7815
|
+
await this.clearCacheForResource(resource, data);
|
|
7816
|
+
} else if (method === "deleteMany") {
|
|
7817
|
+
await this.clearCacheForResource(resource);
|
|
7818
|
+
}
|
|
7819
|
+
return result;
|
|
7820
|
+
});
|
|
7821
|
+
}
|
|
7822
|
+
}
|
|
7823
|
+
async clearCacheForResource(resource, data) {
|
|
7824
|
+
if (!resource.cache) return;
|
|
7825
|
+
const keyPrefix = `resource=${resource.name}`;
|
|
7826
|
+
await resource.cache.clear(keyPrefix);
|
|
7827
|
+
if (this.config.includePartitions === true && resource.config?.partitions && Object.keys(resource.config.partitions).length > 0) {
|
|
7828
|
+
if (!data) {
|
|
7829
|
+
for (const partitionName of Object.keys(resource.config.partitions)) {
|
|
7830
|
+
const partitionKeyPrefix = join(keyPrefix, `partition=${partitionName}`);
|
|
7831
|
+
await resource.cache.clear(partitionKeyPrefix);
|
|
7832
|
+
}
|
|
7833
|
+
} else {
|
|
7834
|
+
const partitionValues = this.getPartitionValues(data, resource);
|
|
7835
|
+
for (const [partitionName, values] of Object.entries(partitionValues)) {
|
|
7836
|
+
if (values && Object.keys(values).length > 0 && Object.values(values).some((v) => v !== null && v !== void 0)) {
|
|
7837
|
+
const partitionKeyPrefix = join(keyPrefix, `partition=${partitionName}`);
|
|
7838
|
+
await resource.cache.clear(partitionKeyPrefix);
|
|
7839
|
+
}
|
|
7840
|
+
}
|
|
7841
|
+
}
|
|
7842
|
+
}
|
|
7843
|
+
}
|
|
7844
|
+
async generateCacheKey(resource, action, params = {}, partition = null, partitionValues = null) {
|
|
7845
|
+
const keyParts = [
|
|
7846
|
+
`resource=${resource.name}`,
|
|
7847
|
+
`action=${action}`
|
|
7848
|
+
];
|
|
7849
|
+
if (partition && partitionValues && Object.keys(partitionValues).length > 0) {
|
|
7850
|
+
keyParts.push(`partition:${partition}`);
|
|
7851
|
+
for (const [field, value] of Object.entries(partitionValues)) {
|
|
7852
|
+
if (value !== null && value !== void 0) {
|
|
7853
|
+
keyParts.push(`${field}:${value}`);
|
|
7854
|
+
}
|
|
7855
|
+
}
|
|
7856
|
+
}
|
|
7857
|
+
if (Object.keys(params).length > 0) {
|
|
7858
|
+
const paramsHash = await this.hashParams(params);
|
|
7859
|
+
keyParts.push(paramsHash);
|
|
7860
|
+
}
|
|
7861
|
+
return join(...keyParts) + ".json.gz";
|
|
7862
|
+
}
|
|
7863
|
+
async hashParams(params) {
|
|
7864
|
+
const sortedParams = Object.keys(params).sort().map((key) => `${key}:${params[key]}`).join("|") || "empty";
|
|
7865
|
+
return await sha256(sortedParams);
|
|
7866
|
+
}
|
|
7867
|
+
// Utility methods
|
|
7868
|
+
async getCacheStats() {
|
|
7869
|
+
if (!this.driver) return null;
|
|
7870
|
+
return {
|
|
7871
|
+
size: await this.driver.size(),
|
|
7872
|
+
keys: await this.driver.keys(),
|
|
7873
|
+
driver: this.driver.constructor.name
|
|
7874
|
+
};
|
|
7875
|
+
}
|
|
7876
|
+
async clearAllCache() {
|
|
7877
|
+
if (!this.driver) return;
|
|
7878
|
+
for (const resource of Object.values(this.database.resources)) {
|
|
7879
|
+
if (resource.cache) {
|
|
7880
|
+
const keyPrefix = `resource=${resource.name}`;
|
|
7881
|
+
await resource.cache.clear(keyPrefix);
|
|
7882
|
+
}
|
|
7883
|
+
}
|
|
7884
|
+
}
|
|
7885
|
+
async warmCache(resourceName, options = {}) {
|
|
7886
|
+
const resource = this.database.resources[resourceName];
|
|
7887
|
+
if (!resource) {
|
|
7888
|
+
throw new Error(`Resource '${resourceName}' not found`);
|
|
7889
|
+
}
|
|
7890
|
+
const { includePartitions = true } = options;
|
|
7891
|
+
if (this.driver instanceof PartitionAwareFilesystemCache && resource.warmPartitionCache) {
|
|
7892
|
+
const partitionNames = resource.config.partitions ? Object.keys(resource.config.partitions) : [];
|
|
7893
|
+
return await resource.warmPartitionCache(partitionNames, options);
|
|
7894
|
+
}
|
|
6983
7895
|
await resource.getAll();
|
|
6984
7896
|
if (includePartitions && resource.config.partitions) {
|
|
6985
7897
|
for (const [partitionName, partitionDef] of Object.entries(resource.config.partitions)) {
|
|
@@ -7001,6 +7913,57 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
7001
7913
|
}
|
|
7002
7914
|
}
|
|
7003
7915
|
}
|
|
7916
|
+
// Partition-specific methods
|
|
7917
|
+
async getPartitionCacheStats(resourceName, partition = null) {
|
|
7918
|
+
if (!(this.driver instanceof PartitionAwareFilesystemCache)) {
|
|
7919
|
+
throw new Error("Partition cache statistics are only available with PartitionAwareFilesystemCache");
|
|
7920
|
+
}
|
|
7921
|
+
return await this.driver.getPartitionStats(resourceName, partition);
|
|
7922
|
+
}
|
|
7923
|
+
async getCacheRecommendations(resourceName) {
|
|
7924
|
+
if (!(this.driver instanceof PartitionAwareFilesystemCache)) {
|
|
7925
|
+
throw new Error("Cache recommendations are only available with PartitionAwareFilesystemCache");
|
|
7926
|
+
}
|
|
7927
|
+
return await this.driver.getCacheRecommendations(resourceName);
|
|
7928
|
+
}
|
|
7929
|
+
async clearPartitionCache(resourceName, partition, partitionValues = {}) {
|
|
7930
|
+
if (!(this.driver instanceof PartitionAwareFilesystemCache)) {
|
|
7931
|
+
throw new Error("Partition cache clearing is only available with PartitionAwareFilesystemCache");
|
|
7932
|
+
}
|
|
7933
|
+
return await this.driver.clearPartition(resourceName, partition, partitionValues);
|
|
7934
|
+
}
|
|
7935
|
+
async analyzeCacheUsage() {
|
|
7936
|
+
if (!(this.driver instanceof PartitionAwareFilesystemCache)) {
|
|
7937
|
+
return { message: "Cache usage analysis is only available with PartitionAwareFilesystemCache" };
|
|
7938
|
+
}
|
|
7939
|
+
const analysis = {
|
|
7940
|
+
totalResources: Object.keys(this.database.resources).length,
|
|
7941
|
+
resourceStats: {},
|
|
7942
|
+
recommendations: {},
|
|
7943
|
+
summary: {
|
|
7944
|
+
mostUsedPartitions: [],
|
|
7945
|
+
leastUsedPartitions: [],
|
|
7946
|
+
suggestedOptimizations: []
|
|
7947
|
+
}
|
|
7948
|
+
};
|
|
7949
|
+
for (const [resourceName, resource] of Object.entries(this.database.resources)) {
|
|
7950
|
+
try {
|
|
7951
|
+
analysis.resourceStats[resourceName] = await this.driver.getPartitionStats(resourceName);
|
|
7952
|
+
analysis.recommendations[resourceName] = await this.driver.getCacheRecommendations(resourceName);
|
|
7953
|
+
} catch (error) {
|
|
7954
|
+
analysis.resourceStats[resourceName] = { error: error.message };
|
|
7955
|
+
}
|
|
7956
|
+
}
|
|
7957
|
+
const allRecommendations = Object.values(analysis.recommendations).flat();
|
|
7958
|
+
analysis.summary.mostUsedPartitions = allRecommendations.filter((r) => r.recommendation === "preload").sort((a, b) => b.priority - a.priority).slice(0, 5);
|
|
7959
|
+
analysis.summary.leastUsedPartitions = allRecommendations.filter((r) => r.recommendation === "archive").slice(0, 5);
|
|
7960
|
+
analysis.summary.suggestedOptimizations = [
|
|
7961
|
+
`Consider preloading ${analysis.summary.mostUsedPartitions.length} high-usage partitions`,
|
|
7962
|
+
`Archive ${analysis.summary.leastUsedPartitions.length} unused partitions`,
|
|
7963
|
+
`Monitor cache hit rates for partition efficiency`
|
|
7964
|
+
];
|
|
7965
|
+
return analysis;
|
|
7966
|
+
}
|
|
7004
7967
|
}
|
|
7005
7968
|
|
|
7006
7969
|
const CostsPlugin = {
|
|
@@ -7177,24 +8140,29 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
7177
8140
|
resource._deleteMany = resource.deleteMany;
|
|
7178
8141
|
this.wrapResourceMethod(resource, "insert", async (result, args, methodName) => {
|
|
7179
8142
|
const [data] = args;
|
|
7180
|
-
this.indexRecord(resource.name, result.id, data).catch(
|
|
8143
|
+
this.indexRecord(resource.name, result.id, data).catch(() => {
|
|
8144
|
+
});
|
|
7181
8145
|
return result;
|
|
7182
8146
|
});
|
|
7183
8147
|
this.wrapResourceMethod(resource, "update", async (result, args, methodName) => {
|
|
7184
8148
|
const [id, data] = args;
|
|
7185
|
-
this.removeRecordFromIndex(resource.name, id).catch(
|
|
7186
|
-
|
|
8149
|
+
this.removeRecordFromIndex(resource.name, id).catch(() => {
|
|
8150
|
+
});
|
|
8151
|
+
this.indexRecord(resource.name, id, result).catch(() => {
|
|
8152
|
+
});
|
|
7187
8153
|
return result;
|
|
7188
8154
|
});
|
|
7189
8155
|
this.wrapResourceMethod(resource, "delete", async (result, args, methodName) => {
|
|
7190
8156
|
const [id] = args;
|
|
7191
|
-
this.removeRecordFromIndex(resource.name, id).catch(
|
|
8157
|
+
this.removeRecordFromIndex(resource.name, id).catch(() => {
|
|
8158
|
+
});
|
|
7192
8159
|
return result;
|
|
7193
8160
|
});
|
|
7194
8161
|
this.wrapResourceMethod(resource, "deleteMany", async (result, args, methodName) => {
|
|
7195
8162
|
const [ids] = args;
|
|
7196
8163
|
for (const id of ids) {
|
|
7197
|
-
this.removeRecordFromIndex(resource.name, id).catch(
|
|
8164
|
+
this.removeRecordFromIndex(resource.name, id).catch(() => {
|
|
8165
|
+
});
|
|
7198
8166
|
}
|
|
7199
8167
|
return result;
|
|
7200
8168
|
});
|
|
@@ -7684,7 +8652,8 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
7684
8652
|
}
|
|
7685
8653
|
if (this.config.flushInterval > 0) {
|
|
7686
8654
|
this.flushTimer = setInterval(() => {
|
|
7687
|
-
this.flushMetrics().catch(
|
|
8655
|
+
this.flushMetrics().catch(() => {
|
|
8656
|
+
});
|
|
7688
8657
|
}, this.config.flushInterval);
|
|
7689
8658
|
}
|
|
7690
8659
|
}
|
|
@@ -7756,9 +8725,6 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
7756
8725
|
}
|
|
7757
8726
|
this.resetMetrics();
|
|
7758
8727
|
});
|
|
7759
|
-
if (!ok) {
|
|
7760
|
-
console.error("Failed to flush metrics:", err);
|
|
7761
|
-
}
|
|
7762
8728
|
}
|
|
7763
8729
|
resetMetrics() {
|
|
7764
8730
|
for (const operation of Object.keys(this.metrics.operations)) {
|
|
@@ -7890,20 +8856,628 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
7890
8856
|
await this.metricsResource.delete(metric.id);
|
|
7891
8857
|
}
|
|
7892
8858
|
}
|
|
7893
|
-
if (this.errorsResource) {
|
|
7894
|
-
const oldErrors = await this.getErrorLogs({ endDate: cutoffDate.toISOString() });
|
|
7895
|
-
for (const error of oldErrors) {
|
|
7896
|
-
await this.errorsResource.delete(error.id);
|
|
7897
|
-
}
|
|
8859
|
+
if (this.errorsResource) {
|
|
8860
|
+
const oldErrors = await this.getErrorLogs({ endDate: cutoffDate.toISOString() });
|
|
8861
|
+
for (const error of oldErrors) {
|
|
8862
|
+
await this.errorsResource.delete(error.id);
|
|
8863
|
+
}
|
|
8864
|
+
}
|
|
8865
|
+
if (this.performanceResource) {
|
|
8866
|
+
const oldPerformance = await this.getPerformanceLogs({ endDate: cutoffDate.toISOString() });
|
|
8867
|
+
for (const perf of oldPerformance) {
|
|
8868
|
+
await this.performanceResource.delete(perf.id);
|
|
8869
|
+
}
|
|
8870
|
+
}
|
|
8871
|
+
}
|
|
8872
|
+
}
|
|
8873
|
+
|
|
8874
|
+
class BaseReplicator extends EventEmitter {
|
|
8875
|
+
constructor(config = {}) {
|
|
8876
|
+
super();
|
|
8877
|
+
this.config = config;
|
|
8878
|
+
this.name = this.constructor.name;
|
|
8879
|
+
this.enabled = config.enabled !== false;
|
|
8880
|
+
}
|
|
8881
|
+
/**
|
|
8882
|
+
* Initialize the replicator
|
|
8883
|
+
* @param {Object} database - The s3db database instance
|
|
8884
|
+
* @returns {Promise<void>}
|
|
8885
|
+
*/
|
|
8886
|
+
async initialize(database) {
|
|
8887
|
+
this.database = database;
|
|
8888
|
+
this.emit("initialized", { replicator: this.name });
|
|
8889
|
+
}
|
|
8890
|
+
/**
|
|
8891
|
+
* Replicate data to the target
|
|
8892
|
+
* @param {string} resourceName - Name of the resource being replicated
|
|
8893
|
+
* @param {string} operation - Operation type (insert, update, delete)
|
|
8894
|
+
* @param {Object} data - The data to replicate
|
|
8895
|
+
* @param {string} id - Record ID
|
|
8896
|
+
* @returns {Promise<Object>} replicator result
|
|
8897
|
+
*/
|
|
8898
|
+
async replicate(resourceName, operation, data, id) {
|
|
8899
|
+
throw new Error(`replicate() method must be implemented by ${this.name}`);
|
|
8900
|
+
}
|
|
8901
|
+
/**
|
|
8902
|
+
* Replicate multiple records in batch
|
|
8903
|
+
* @param {string} resourceName - Name of the resource being replicated
|
|
8904
|
+
* @param {Array} records - Array of records to replicate
|
|
8905
|
+
* @returns {Promise<Object>} Batch replicator result
|
|
8906
|
+
*/
|
|
8907
|
+
async replicateBatch(resourceName, records) {
|
|
8908
|
+
throw new Error(`replicateBatch() method must be implemented by ${this.name}`);
|
|
8909
|
+
}
|
|
8910
|
+
/**
|
|
8911
|
+
* Test the connection to the target
|
|
8912
|
+
* @returns {Promise<boolean>} True if connection is successful
|
|
8913
|
+
*/
|
|
8914
|
+
async testConnection() {
|
|
8915
|
+
throw new Error(`testConnection() method must be implemented by ${this.name}`);
|
|
8916
|
+
}
|
|
8917
|
+
/**
|
|
8918
|
+
* Get replicator status and statistics
|
|
8919
|
+
* @returns {Promise<Object>} Status information
|
|
8920
|
+
*/
|
|
8921
|
+
async getStatus() {
|
|
8922
|
+
return {
|
|
8923
|
+
name: this.name,
|
|
8924
|
+
// Removed: enabled: this.enabled,
|
|
8925
|
+
config: this.config,
|
|
8926
|
+
connected: false
|
|
8927
|
+
};
|
|
8928
|
+
}
|
|
8929
|
+
/**
|
|
8930
|
+
* Cleanup resources
|
|
8931
|
+
* @returns {Promise<void>}
|
|
8932
|
+
*/
|
|
8933
|
+
async cleanup() {
|
|
8934
|
+
this.emit("cleanup", { replicator: this.name });
|
|
8935
|
+
}
|
|
8936
|
+
/**
|
|
8937
|
+
* Validate replicator configuration
|
|
8938
|
+
* @returns {Object} Validation result
|
|
8939
|
+
*/
|
|
8940
|
+
validateConfig() {
|
|
8941
|
+
return { isValid: true, errors: [] };
|
|
8942
|
+
}
|
|
8943
|
+
}
|
|
8944
|
+
var base_replicator_class_default = BaseReplicator;
|
|
8945
|
+
|
|
8946
|
+
class BigqueryReplicator extends base_replicator_class_default {
|
|
8947
|
+
constructor(config = {}, resources = {}) {
|
|
8948
|
+
super(config);
|
|
8949
|
+
this.projectId = config.projectId;
|
|
8950
|
+
this.datasetId = config.datasetId;
|
|
8951
|
+
this.bigqueryClient = null;
|
|
8952
|
+
this.credentials = config.credentials;
|
|
8953
|
+
this.location = config.location || "US";
|
|
8954
|
+
this.logTable = config.logTable;
|
|
8955
|
+
this.resources = this.parseResourcesConfig(resources);
|
|
8956
|
+
}
|
|
8957
|
+
parseResourcesConfig(resources) {
|
|
8958
|
+
const parsed = {};
|
|
8959
|
+
for (const [resourceName, config] of Object.entries(resources)) {
|
|
8960
|
+
if (typeof config === "string") {
|
|
8961
|
+
parsed[resourceName] = [{
|
|
8962
|
+
table: config,
|
|
8963
|
+
actions: ["insert"],
|
|
8964
|
+
transform: null
|
|
8965
|
+
}];
|
|
8966
|
+
} else if (Array.isArray(config)) {
|
|
8967
|
+
parsed[resourceName] = config.map((item) => {
|
|
8968
|
+
if (typeof item === "string") {
|
|
8969
|
+
return { table: item, actions: ["insert"], transform: null };
|
|
8970
|
+
}
|
|
8971
|
+
return {
|
|
8972
|
+
table: item.table,
|
|
8973
|
+
actions: item.actions || ["insert"],
|
|
8974
|
+
transform: item.transform || null
|
|
8975
|
+
};
|
|
8976
|
+
});
|
|
8977
|
+
} else if (typeof config === "object") {
|
|
8978
|
+
parsed[resourceName] = [{
|
|
8979
|
+
table: config.table,
|
|
8980
|
+
actions: config.actions || ["insert"],
|
|
8981
|
+
transform: config.transform || null
|
|
8982
|
+
}];
|
|
8983
|
+
}
|
|
8984
|
+
}
|
|
8985
|
+
return parsed;
|
|
8986
|
+
}
|
|
8987
|
+
validateConfig() {
|
|
8988
|
+
const errors = [];
|
|
8989
|
+
if (!this.projectId) errors.push("projectId is required");
|
|
8990
|
+
if (!this.datasetId) errors.push("datasetId is required");
|
|
8991
|
+
if (Object.keys(this.resources).length === 0) errors.push("At least one resource must be configured");
|
|
8992
|
+
for (const [resourceName, tables] of Object.entries(this.resources)) {
|
|
8993
|
+
for (const tableConfig of tables) {
|
|
8994
|
+
if (!tableConfig.table) {
|
|
8995
|
+
errors.push(`Table name is required for resource '${resourceName}'`);
|
|
8996
|
+
}
|
|
8997
|
+
if (!Array.isArray(tableConfig.actions) || tableConfig.actions.length === 0) {
|
|
8998
|
+
errors.push(`Actions array is required for resource '${resourceName}'`);
|
|
8999
|
+
}
|
|
9000
|
+
const validActions = ["insert", "update", "delete"];
|
|
9001
|
+
const invalidActions = tableConfig.actions.filter((action) => !validActions.includes(action));
|
|
9002
|
+
if (invalidActions.length > 0) {
|
|
9003
|
+
errors.push(`Invalid actions for resource '${resourceName}': ${invalidActions.join(", ")}. Valid actions: ${validActions.join(", ")}`);
|
|
9004
|
+
}
|
|
9005
|
+
if (tableConfig.transform && typeof tableConfig.transform !== "function") {
|
|
9006
|
+
errors.push(`Transform must be a function for resource '${resourceName}'`);
|
|
9007
|
+
}
|
|
9008
|
+
}
|
|
9009
|
+
}
|
|
9010
|
+
return { isValid: errors.length === 0, errors };
|
|
9011
|
+
}
|
|
9012
|
+
async initialize(database) {
|
|
9013
|
+
await super.initialize(database);
|
|
9014
|
+
const [ok, err, sdk] = await try_fn_default(() => import('@google-cloud/bigquery'));
|
|
9015
|
+
if (!ok) {
|
|
9016
|
+
this.emit("initialization_error", { replicator: this.name, error: err.message });
|
|
9017
|
+
throw err;
|
|
9018
|
+
}
|
|
9019
|
+
const { BigQuery } = sdk;
|
|
9020
|
+
this.bigqueryClient = new BigQuery({
|
|
9021
|
+
projectId: this.projectId,
|
|
9022
|
+
credentials: this.credentials,
|
|
9023
|
+
location: this.location
|
|
9024
|
+
});
|
|
9025
|
+
this.emit("initialized", {
|
|
9026
|
+
replicator: this.name,
|
|
9027
|
+
projectId: this.projectId,
|
|
9028
|
+
datasetId: this.datasetId,
|
|
9029
|
+
resources: Object.keys(this.resources)
|
|
9030
|
+
});
|
|
9031
|
+
}
|
|
9032
|
+
shouldReplicateResource(resourceName) {
|
|
9033
|
+
return this.resources.hasOwnProperty(resourceName);
|
|
9034
|
+
}
|
|
9035
|
+
shouldReplicateAction(resourceName, operation) {
|
|
9036
|
+
if (!this.resources[resourceName]) return false;
|
|
9037
|
+
return this.resources[resourceName].some(
|
|
9038
|
+
(tableConfig) => tableConfig.actions.includes(operation)
|
|
9039
|
+
);
|
|
9040
|
+
}
|
|
9041
|
+
getTablesForResource(resourceName, operation) {
|
|
9042
|
+
if (!this.resources[resourceName]) return [];
|
|
9043
|
+
return this.resources[resourceName].filter((tableConfig) => tableConfig.actions.includes(operation)).map((tableConfig) => ({
|
|
9044
|
+
table: tableConfig.table,
|
|
9045
|
+
transform: tableConfig.transform
|
|
9046
|
+
}));
|
|
9047
|
+
}
|
|
9048
|
+
applyTransform(data, transformFn) {
|
|
9049
|
+
if (!transformFn) return data;
|
|
9050
|
+
let transformedData = JSON.parse(JSON.stringify(data));
|
|
9051
|
+
if (transformedData._length) delete transformedData._length;
|
|
9052
|
+
return transformFn(transformedData);
|
|
9053
|
+
}
|
|
9054
|
+
async replicate(resourceName, operation, data, id, beforeData = null) {
|
|
9055
|
+
if (!this.enabled || !this.shouldReplicateResource(resourceName)) {
|
|
9056
|
+
return { skipped: true, reason: "resource_not_included" };
|
|
9057
|
+
}
|
|
9058
|
+
if (!this.shouldReplicateAction(resourceName, operation)) {
|
|
9059
|
+
return { skipped: true, reason: "action_not_included" };
|
|
9060
|
+
}
|
|
9061
|
+
const tableConfigs = this.getTablesForResource(resourceName, operation);
|
|
9062
|
+
if (tableConfigs.length === 0) {
|
|
9063
|
+
return { skipped: true, reason: "no_tables_for_action" };
|
|
9064
|
+
}
|
|
9065
|
+
const results = [];
|
|
9066
|
+
const errors = [];
|
|
9067
|
+
const [ok, err, result] = await try_fn_default(async () => {
|
|
9068
|
+
const dataset = this.bigqueryClient.dataset(this.datasetId);
|
|
9069
|
+
for (const tableConfig of tableConfigs) {
|
|
9070
|
+
const [okTable, errTable] = await try_fn_default(async () => {
|
|
9071
|
+
const table = dataset.table(tableConfig.table);
|
|
9072
|
+
let job;
|
|
9073
|
+
if (operation === "insert") {
|
|
9074
|
+
const transformedData = this.applyTransform(data, tableConfig.transform);
|
|
9075
|
+
job = await table.insert([transformedData]);
|
|
9076
|
+
} else if (operation === "update") {
|
|
9077
|
+
const transformedData = this.applyTransform(data, tableConfig.transform);
|
|
9078
|
+
const keys = Object.keys(transformedData).filter((k) => k !== "id");
|
|
9079
|
+
const setClause = keys.map((k) => `${k} = @${k}`).join(", ");
|
|
9080
|
+
const params = { id, ...transformedData };
|
|
9081
|
+
const query = `UPDATE \`${this.projectId}.${this.datasetId}.${tableConfig.table}\` SET ${setClause} WHERE id = @id`;
|
|
9082
|
+
const maxRetries = 2;
|
|
9083
|
+
let lastError = null;
|
|
9084
|
+
for (let attempt = 1; attempt <= maxRetries; attempt++) {
|
|
9085
|
+
try {
|
|
9086
|
+
const [updateJob] = await this.bigqueryClient.createQueryJob({
|
|
9087
|
+
query,
|
|
9088
|
+
params,
|
|
9089
|
+
location: this.location
|
|
9090
|
+
});
|
|
9091
|
+
await updateJob.getQueryResults();
|
|
9092
|
+
job = [updateJob];
|
|
9093
|
+
break;
|
|
9094
|
+
} catch (error) {
|
|
9095
|
+
lastError = error;
|
|
9096
|
+
if (error?.message?.includes("streaming buffer") && attempt < maxRetries) {
|
|
9097
|
+
const delaySeconds = 30;
|
|
9098
|
+
await new Promise((resolve) => setTimeout(resolve, delaySeconds * 1e3));
|
|
9099
|
+
continue;
|
|
9100
|
+
}
|
|
9101
|
+
throw error;
|
|
9102
|
+
}
|
|
9103
|
+
}
|
|
9104
|
+
if (!job) throw lastError;
|
|
9105
|
+
} else if (operation === "delete") {
|
|
9106
|
+
const query = `DELETE FROM \`${this.projectId}.${this.datasetId}.${tableConfig.table}\` WHERE id = @id`;
|
|
9107
|
+
const [deleteJob] = await this.bigqueryClient.createQueryJob({
|
|
9108
|
+
query,
|
|
9109
|
+
params: { id },
|
|
9110
|
+
location: this.location
|
|
9111
|
+
});
|
|
9112
|
+
await deleteJob.getQueryResults();
|
|
9113
|
+
job = [deleteJob];
|
|
9114
|
+
} else {
|
|
9115
|
+
throw new Error(`Unsupported operation: ${operation}`);
|
|
9116
|
+
}
|
|
9117
|
+
results.push({
|
|
9118
|
+
table: tableConfig.table,
|
|
9119
|
+
success: true,
|
|
9120
|
+
jobId: job[0]?.id
|
|
9121
|
+
});
|
|
9122
|
+
});
|
|
9123
|
+
if (!okTable) {
|
|
9124
|
+
errors.push({
|
|
9125
|
+
table: tableConfig.table,
|
|
9126
|
+
error: errTable.message
|
|
9127
|
+
});
|
|
9128
|
+
}
|
|
9129
|
+
}
|
|
9130
|
+
if (this.logTable) {
|
|
9131
|
+
const [okLog, errLog] = await try_fn_default(async () => {
|
|
9132
|
+
const logTable = dataset.table(this.logTable);
|
|
9133
|
+
await logTable.insert([{
|
|
9134
|
+
resource_name: resourceName,
|
|
9135
|
+
operation,
|
|
9136
|
+
record_id: id,
|
|
9137
|
+
data: JSON.stringify(data),
|
|
9138
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
9139
|
+
source: "s3db-replicator"
|
|
9140
|
+
}]);
|
|
9141
|
+
});
|
|
9142
|
+
if (!okLog) {
|
|
9143
|
+
}
|
|
9144
|
+
}
|
|
9145
|
+
const success = errors.length === 0;
|
|
9146
|
+
this.emit("replicated", {
|
|
9147
|
+
replicator: this.name,
|
|
9148
|
+
resourceName,
|
|
9149
|
+
operation,
|
|
9150
|
+
id,
|
|
9151
|
+
tables: tableConfigs.map((t) => t.table),
|
|
9152
|
+
results,
|
|
9153
|
+
errors,
|
|
9154
|
+
success
|
|
9155
|
+
});
|
|
9156
|
+
return {
|
|
9157
|
+
success,
|
|
9158
|
+
results,
|
|
9159
|
+
errors,
|
|
9160
|
+
tables: tableConfigs.map((t) => t.table)
|
|
9161
|
+
};
|
|
9162
|
+
});
|
|
9163
|
+
if (ok) return result;
|
|
9164
|
+
this.emit("replicator_error", {
|
|
9165
|
+
replicator: this.name,
|
|
9166
|
+
resourceName,
|
|
9167
|
+
operation,
|
|
9168
|
+
id,
|
|
9169
|
+
error: err.message
|
|
9170
|
+
});
|
|
9171
|
+
return { success: false, error: err.message };
|
|
9172
|
+
}
|
|
9173
|
+
async replicateBatch(resourceName, records) {
|
|
9174
|
+
const results = [];
|
|
9175
|
+
const errors = [];
|
|
9176
|
+
for (const record of records) {
|
|
9177
|
+
const [ok, err, res] = await try_fn_default(() => this.replicate(
|
|
9178
|
+
resourceName,
|
|
9179
|
+
record.operation,
|
|
9180
|
+
record.data,
|
|
9181
|
+
record.id,
|
|
9182
|
+
record.beforeData
|
|
9183
|
+
));
|
|
9184
|
+
if (ok) results.push(res);
|
|
9185
|
+
else errors.push({ id: record.id, error: err.message });
|
|
9186
|
+
}
|
|
9187
|
+
return {
|
|
9188
|
+
success: errors.length === 0,
|
|
9189
|
+
results,
|
|
9190
|
+
errors
|
|
9191
|
+
};
|
|
9192
|
+
}
|
|
9193
|
+
async testConnection() {
|
|
9194
|
+
const [ok, err] = await try_fn_default(async () => {
|
|
9195
|
+
if (!this.bigqueryClient) await this.initialize();
|
|
9196
|
+
const dataset = this.bigqueryClient.dataset(this.datasetId);
|
|
9197
|
+
await dataset.getMetadata();
|
|
9198
|
+
return true;
|
|
9199
|
+
});
|
|
9200
|
+
if (ok) return true;
|
|
9201
|
+
this.emit("connection_error", { replicator: this.name, error: err.message });
|
|
9202
|
+
return false;
|
|
9203
|
+
}
|
|
9204
|
+
async cleanup() {
|
|
9205
|
+
}
|
|
9206
|
+
getStatus() {
|
|
9207
|
+
return {
|
|
9208
|
+
...super.getStatus(),
|
|
9209
|
+
projectId: this.projectId,
|
|
9210
|
+
datasetId: this.datasetId,
|
|
9211
|
+
resources: this.resources,
|
|
9212
|
+
logTable: this.logTable
|
|
9213
|
+
};
|
|
9214
|
+
}
|
|
9215
|
+
}
|
|
9216
|
+
var bigquery_replicator_class_default = BigqueryReplicator;
|
|
9217
|
+
|
|
9218
|
+
class PostgresReplicator extends base_replicator_class_default {
|
|
9219
|
+
constructor(config = {}, resources = {}) {
|
|
9220
|
+
super(config);
|
|
9221
|
+
this.connectionString = config.connectionString;
|
|
9222
|
+
this.host = config.host;
|
|
9223
|
+
this.port = config.port || 5432;
|
|
9224
|
+
this.database = config.database;
|
|
9225
|
+
this.user = config.user;
|
|
9226
|
+
this.password = config.password;
|
|
9227
|
+
this.client = null;
|
|
9228
|
+
this.ssl = config.ssl;
|
|
9229
|
+
this.logTable = config.logTable;
|
|
9230
|
+
this.resources = this.parseResourcesConfig(resources);
|
|
9231
|
+
}
|
|
9232
|
+
parseResourcesConfig(resources) {
|
|
9233
|
+
const parsed = {};
|
|
9234
|
+
for (const [resourceName, config] of Object.entries(resources)) {
|
|
9235
|
+
if (typeof config === "string") {
|
|
9236
|
+
parsed[resourceName] = [{
|
|
9237
|
+
table: config,
|
|
9238
|
+
actions: ["insert"]
|
|
9239
|
+
}];
|
|
9240
|
+
} else if (Array.isArray(config)) {
|
|
9241
|
+
parsed[resourceName] = config.map((item) => {
|
|
9242
|
+
if (typeof item === "string") {
|
|
9243
|
+
return { table: item, actions: ["insert"] };
|
|
9244
|
+
}
|
|
9245
|
+
return {
|
|
9246
|
+
table: item.table,
|
|
9247
|
+
actions: item.actions || ["insert"]
|
|
9248
|
+
};
|
|
9249
|
+
});
|
|
9250
|
+
} else if (typeof config === "object") {
|
|
9251
|
+
parsed[resourceName] = [{
|
|
9252
|
+
table: config.table,
|
|
9253
|
+
actions: config.actions || ["insert"]
|
|
9254
|
+
}];
|
|
9255
|
+
}
|
|
9256
|
+
}
|
|
9257
|
+
return parsed;
|
|
9258
|
+
}
|
|
9259
|
+
validateConfig() {
|
|
9260
|
+
const errors = [];
|
|
9261
|
+
if (!this.connectionString && (!this.host || !this.database)) {
|
|
9262
|
+
errors.push("Either connectionString or host+database must be provided");
|
|
9263
|
+
}
|
|
9264
|
+
if (Object.keys(this.resources).length === 0) {
|
|
9265
|
+
errors.push("At least one resource must be configured");
|
|
9266
|
+
}
|
|
9267
|
+
for (const [resourceName, tables] of Object.entries(this.resources)) {
|
|
9268
|
+
for (const tableConfig of tables) {
|
|
9269
|
+
if (!tableConfig.table) {
|
|
9270
|
+
errors.push(`Table name is required for resource '${resourceName}'`);
|
|
9271
|
+
}
|
|
9272
|
+
if (!Array.isArray(tableConfig.actions) || tableConfig.actions.length === 0) {
|
|
9273
|
+
errors.push(`Actions array is required for resource '${resourceName}'`);
|
|
9274
|
+
}
|
|
9275
|
+
const validActions = ["insert", "update", "delete"];
|
|
9276
|
+
const invalidActions = tableConfig.actions.filter((action) => !validActions.includes(action));
|
|
9277
|
+
if (invalidActions.length > 0) {
|
|
9278
|
+
errors.push(`Invalid actions for resource '${resourceName}': ${invalidActions.join(", ")}. Valid actions: ${validActions.join(", ")}`);
|
|
9279
|
+
}
|
|
9280
|
+
}
|
|
9281
|
+
}
|
|
9282
|
+
return { isValid: errors.length === 0, errors };
|
|
9283
|
+
}
|
|
9284
|
+
async initialize(database) {
|
|
9285
|
+
await super.initialize(database);
|
|
9286
|
+
const [ok, err, sdk] = await try_fn_default(() => import('pg'));
|
|
9287
|
+
if (!ok) {
|
|
9288
|
+
this.emit("initialization_error", {
|
|
9289
|
+
replicator: this.name,
|
|
9290
|
+
error: err.message
|
|
9291
|
+
});
|
|
9292
|
+
throw err;
|
|
9293
|
+
}
|
|
9294
|
+
const { Client } = sdk;
|
|
9295
|
+
const config = this.connectionString ? {
|
|
9296
|
+
connectionString: this.connectionString,
|
|
9297
|
+
ssl: this.ssl
|
|
9298
|
+
} : {
|
|
9299
|
+
host: this.host,
|
|
9300
|
+
port: this.port,
|
|
9301
|
+
database: this.database,
|
|
9302
|
+
user: this.user,
|
|
9303
|
+
password: this.password,
|
|
9304
|
+
ssl: this.ssl
|
|
9305
|
+
};
|
|
9306
|
+
this.client = new Client(config);
|
|
9307
|
+
await this.client.connect();
|
|
9308
|
+
if (this.logTable) {
|
|
9309
|
+
await this.createLogTableIfNotExists();
|
|
9310
|
+
}
|
|
9311
|
+
this.emit("initialized", {
|
|
9312
|
+
replicator: this.name,
|
|
9313
|
+
database: this.database || "postgres",
|
|
9314
|
+
resources: Object.keys(this.resources)
|
|
9315
|
+
});
|
|
9316
|
+
}
|
|
9317
|
+
async createLogTableIfNotExists() {
|
|
9318
|
+
const createTableQuery = `
|
|
9319
|
+
CREATE TABLE IF NOT EXISTS ${this.logTable} (
|
|
9320
|
+
id SERIAL PRIMARY KEY,
|
|
9321
|
+
resource_name VARCHAR(255) NOT NULL,
|
|
9322
|
+
operation VARCHAR(50) NOT NULL,
|
|
9323
|
+
record_id VARCHAR(255) NOT NULL,
|
|
9324
|
+
data JSONB,
|
|
9325
|
+
timestamp TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
|
|
9326
|
+
source VARCHAR(100) DEFAULT 's3db-replicator',
|
|
9327
|
+
created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
|
|
9328
|
+
);
|
|
9329
|
+
CREATE INDEX IF NOT EXISTS idx_${this.logTable}_resource_name ON ${this.logTable}(resource_name);
|
|
9330
|
+
CREATE INDEX IF NOT EXISTS idx_${this.logTable}_operation ON ${this.logTable}(operation);
|
|
9331
|
+
CREATE INDEX IF NOT EXISTS idx_${this.logTable}_record_id ON ${this.logTable}(record_id);
|
|
9332
|
+
CREATE INDEX IF NOT EXISTS idx_${this.logTable}_timestamp ON ${this.logTable}(timestamp);
|
|
9333
|
+
`;
|
|
9334
|
+
await this.client.query(createTableQuery);
|
|
9335
|
+
}
|
|
9336
|
+
shouldReplicateResource(resourceName) {
|
|
9337
|
+
return this.resources.hasOwnProperty(resourceName);
|
|
9338
|
+
}
|
|
9339
|
+
shouldReplicateAction(resourceName, operation) {
|
|
9340
|
+
if (!this.resources[resourceName]) return false;
|
|
9341
|
+
return this.resources[resourceName].some(
|
|
9342
|
+
(tableConfig) => tableConfig.actions.includes(operation)
|
|
9343
|
+
);
|
|
9344
|
+
}
|
|
9345
|
+
getTablesForResource(resourceName, operation) {
|
|
9346
|
+
if (!this.resources[resourceName]) return [];
|
|
9347
|
+
return this.resources[resourceName].filter((tableConfig) => tableConfig.actions.includes(operation)).map((tableConfig) => tableConfig.table);
|
|
9348
|
+
}
|
|
9349
|
+
async replicate(resourceName, operation, data, id, beforeData = null) {
|
|
9350
|
+
if (!this.enabled || !this.shouldReplicateResource(resourceName)) {
|
|
9351
|
+
return { skipped: true, reason: "resource_not_included" };
|
|
9352
|
+
}
|
|
9353
|
+
if (!this.shouldReplicateAction(resourceName, operation)) {
|
|
9354
|
+
return { skipped: true, reason: "action_not_included" };
|
|
7898
9355
|
}
|
|
7899
|
-
|
|
7900
|
-
|
|
7901
|
-
|
|
7902
|
-
|
|
9356
|
+
const tables = this.getTablesForResource(resourceName, operation);
|
|
9357
|
+
if (tables.length === 0) {
|
|
9358
|
+
return { skipped: true, reason: "no_tables_for_action" };
|
|
9359
|
+
}
|
|
9360
|
+
const results = [];
|
|
9361
|
+
const errors = [];
|
|
9362
|
+
const [ok, err, result] = await try_fn_default(async () => {
|
|
9363
|
+
for (const table of tables) {
|
|
9364
|
+
const [okTable, errTable] = await try_fn_default(async () => {
|
|
9365
|
+
let result2;
|
|
9366
|
+
if (operation === "insert") {
|
|
9367
|
+
const keys = Object.keys(data);
|
|
9368
|
+
const values = keys.map((k) => data[k]);
|
|
9369
|
+
const columns = keys.map((k) => `"${k}"`).join(", ");
|
|
9370
|
+
const params = keys.map((_, i) => `$${i + 1}`).join(", ");
|
|
9371
|
+
const sql = `INSERT INTO ${table} (${columns}) VALUES (${params}) ON CONFLICT (id) DO NOTHING RETURNING *`;
|
|
9372
|
+
result2 = await this.client.query(sql, values);
|
|
9373
|
+
} else if (operation === "update") {
|
|
9374
|
+
const keys = Object.keys(data).filter((k) => k !== "id");
|
|
9375
|
+
const setClause = keys.map((k, i) => `"${k}"=$${i + 1}`).join(", ");
|
|
9376
|
+
const values = keys.map((k) => data[k]);
|
|
9377
|
+
values.push(id);
|
|
9378
|
+
const sql = `UPDATE ${table} SET ${setClause} WHERE id=$${keys.length + 1} RETURNING *`;
|
|
9379
|
+
result2 = await this.client.query(sql, values);
|
|
9380
|
+
} else if (operation === "delete") {
|
|
9381
|
+
const sql = `DELETE FROM ${table} WHERE id=$1 RETURNING *`;
|
|
9382
|
+
result2 = await this.client.query(sql, [id]);
|
|
9383
|
+
} else {
|
|
9384
|
+
throw new Error(`Unsupported operation: ${operation}`);
|
|
9385
|
+
}
|
|
9386
|
+
results.push({
|
|
9387
|
+
table,
|
|
9388
|
+
success: true,
|
|
9389
|
+
rows: result2.rows,
|
|
9390
|
+
rowCount: result2.rowCount
|
|
9391
|
+
});
|
|
9392
|
+
});
|
|
9393
|
+
if (!okTable) {
|
|
9394
|
+
errors.push({
|
|
9395
|
+
table,
|
|
9396
|
+
error: errTable.message
|
|
9397
|
+
});
|
|
9398
|
+
}
|
|
9399
|
+
}
|
|
9400
|
+
if (this.logTable) {
|
|
9401
|
+
const [okLog, errLog] = await try_fn_default(async () => {
|
|
9402
|
+
await this.client.query(
|
|
9403
|
+
`INSERT INTO ${this.logTable} (resource_name, operation, record_id, data, timestamp, source) VALUES ($1, $2, $3, $4, $5, $6)`,
|
|
9404
|
+
[resourceName, operation, id, JSON.stringify(data), (/* @__PURE__ */ new Date()).toISOString(), "s3db-replicator"]
|
|
9405
|
+
);
|
|
9406
|
+
});
|
|
9407
|
+
if (!okLog) {
|
|
9408
|
+
}
|
|
7903
9409
|
}
|
|
9410
|
+
const success = errors.length === 0;
|
|
9411
|
+
this.emit("replicated", {
|
|
9412
|
+
replicator: this.name,
|
|
9413
|
+
resourceName,
|
|
9414
|
+
operation,
|
|
9415
|
+
id,
|
|
9416
|
+
tables,
|
|
9417
|
+
results,
|
|
9418
|
+
errors,
|
|
9419
|
+
success
|
|
9420
|
+
});
|
|
9421
|
+
return {
|
|
9422
|
+
success,
|
|
9423
|
+
results,
|
|
9424
|
+
errors,
|
|
9425
|
+
tables
|
|
9426
|
+
};
|
|
9427
|
+
});
|
|
9428
|
+
if (ok) return result;
|
|
9429
|
+
this.emit("replicator_error", {
|
|
9430
|
+
replicator: this.name,
|
|
9431
|
+
resourceName,
|
|
9432
|
+
operation,
|
|
9433
|
+
id,
|
|
9434
|
+
error: err.message
|
|
9435
|
+
});
|
|
9436
|
+
return { success: false, error: err.message };
|
|
9437
|
+
}
|
|
9438
|
+
async replicateBatch(resourceName, records) {
|
|
9439
|
+
const results = [];
|
|
9440
|
+
const errors = [];
|
|
9441
|
+
for (const record of records) {
|
|
9442
|
+
const [ok, err, res] = await try_fn_default(() => this.replicate(
|
|
9443
|
+
resourceName,
|
|
9444
|
+
record.operation,
|
|
9445
|
+
record.data,
|
|
9446
|
+
record.id,
|
|
9447
|
+
record.beforeData
|
|
9448
|
+
));
|
|
9449
|
+
if (ok) results.push(res);
|
|
9450
|
+
else errors.push({ id: record.id, error: err.message });
|
|
7904
9451
|
}
|
|
9452
|
+
return {
|
|
9453
|
+
success: errors.length === 0,
|
|
9454
|
+
results,
|
|
9455
|
+
errors
|
|
9456
|
+
};
|
|
9457
|
+
}
|
|
9458
|
+
async testConnection() {
|
|
9459
|
+
const [ok, err] = await try_fn_default(async () => {
|
|
9460
|
+
if (!this.client) await this.initialize();
|
|
9461
|
+
await this.client.query("SELECT 1");
|
|
9462
|
+
return true;
|
|
9463
|
+
});
|
|
9464
|
+
if (ok) return true;
|
|
9465
|
+
this.emit("connection_error", { replicator: this.name, error: err.message });
|
|
9466
|
+
return false;
|
|
9467
|
+
}
|
|
9468
|
+
async cleanup() {
|
|
9469
|
+
if (this.client) await this.client.end();
|
|
9470
|
+
}
|
|
9471
|
+
getStatus() {
|
|
9472
|
+
return {
|
|
9473
|
+
...super.getStatus(),
|
|
9474
|
+
database: this.database || "postgres",
|
|
9475
|
+
resources: this.resources,
|
|
9476
|
+
logTable: this.logTable
|
|
9477
|
+
};
|
|
7905
9478
|
}
|
|
7906
9479
|
}
|
|
9480
|
+
var postgres_replicator_class_default = PostgresReplicator;
|
|
7907
9481
|
|
|
7908
9482
|
const S3_DEFAULT_REGION = "us-east-1";
|
|
7909
9483
|
const S3_DEFAULT_ENDPOINT = "https://s3.us-east-1.amazonaws.com";
|
|
@@ -9987,7 +11561,7 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
9987
11561
|
if (okParse) contentType = "application/json";
|
|
9988
11562
|
}
|
|
9989
11563
|
if (this.behavior === "body-only" && (!body || body === "")) {
|
|
9990
|
-
throw new Error(`[Resource.insert]
|
|
11564
|
+
throw new Error(`[Resource.insert] Attempt to save object without body! Data: id=${finalId}, resource=${this.name}`);
|
|
9991
11565
|
}
|
|
9992
11566
|
const [okPut, errPut, putResult] = await try_fn_default(() => this.client.putObject({
|
|
9993
11567
|
key,
|
|
@@ -11334,8 +12908,8 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
11334
12908
|
return mappedData;
|
|
11335
12909
|
}
|
|
11336
12910
|
/**
|
|
11337
|
-
* Compose the full object (metadata + body) as
|
|
11338
|
-
*
|
|
12911
|
+
* Compose the full object (metadata + body) as returned by .get(),
|
|
12912
|
+
* using in-memory data after insert/update, according to behavior
|
|
11339
12913
|
*/
|
|
11340
12914
|
async composeFullObjectFromWrite({ id, metadata, body, behavior }) {
|
|
11341
12915
|
const behaviorFlags = {};
|
|
@@ -11490,7 +13064,7 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
11490
13064
|
if (!this._middlewares.has(method)) throw new ResourceError(`No such method for middleware: ${method}`, { operation: "useMiddleware", method });
|
|
11491
13065
|
this._middlewares.get(method).push(fn);
|
|
11492
13066
|
}
|
|
11493
|
-
//
|
|
13067
|
+
// Utility to apply schema default values
|
|
11494
13068
|
applyDefaults(data) {
|
|
11495
13069
|
const out = { ...data };
|
|
11496
13070
|
for (const [key, def] of Object.entries(this.attributes)) {
|
|
@@ -11622,7 +13196,7 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
11622
13196
|
super();
|
|
11623
13197
|
this.version = "1";
|
|
11624
13198
|
this.s3dbVersion = (() => {
|
|
11625
|
-
const [ok, err, version] = try_fn_default(() => true ? "7.
|
|
13199
|
+
const [ok, err, version] = try_fn_default(() => true ? "7.3.3" : "latest");
|
|
11626
13200
|
return ok ? version : "latest";
|
|
11627
13201
|
})();
|
|
11628
13202
|
this.resources = {};
|
|
@@ -11695,7 +13269,7 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
11695
13269
|
name,
|
|
11696
13270
|
client: this.client,
|
|
11697
13271
|
database: this,
|
|
11698
|
-
//
|
|
13272
|
+
// ensure reference
|
|
11699
13273
|
version: currentVersion,
|
|
11700
13274
|
attributes: versionData.attributes,
|
|
11701
13275
|
behavior: versionData.behavior || "user-managed",
|
|
@@ -12097,6 +13671,570 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
12097
13671
|
class S3db extends Database {
|
|
12098
13672
|
}
|
|
12099
13673
|
|
|
13674
|
+
function normalizeResourceName$1(name) {
|
|
13675
|
+
return typeof name === "string" ? name.trim().toLowerCase() : name;
|
|
13676
|
+
}
|
|
13677
|
+
class S3dbReplicator extends base_replicator_class_default {
|
|
13678
|
+
constructor(config = {}, resources = [], client = null) {
|
|
13679
|
+
super(config);
|
|
13680
|
+
this.instanceId = Math.random().toString(36).slice(2, 10);
|
|
13681
|
+
this.client = client;
|
|
13682
|
+
this.connectionString = config.connectionString;
|
|
13683
|
+
let normalizedResources = resources;
|
|
13684
|
+
if (!resources) normalizedResources = {};
|
|
13685
|
+
else if (Array.isArray(resources)) {
|
|
13686
|
+
normalizedResources = {};
|
|
13687
|
+
for (const res of resources) {
|
|
13688
|
+
if (typeof res === "string") normalizedResources[normalizeResourceName$1(res)] = res;
|
|
13689
|
+
}
|
|
13690
|
+
} else if (typeof resources === "string") {
|
|
13691
|
+
normalizedResources[normalizeResourceName$1(resources)] = resources;
|
|
13692
|
+
}
|
|
13693
|
+
this.resourcesMap = this._normalizeResources(normalizedResources);
|
|
13694
|
+
}
|
|
13695
|
+
_normalizeResources(resources) {
|
|
13696
|
+
if (!resources) return {};
|
|
13697
|
+
if (Array.isArray(resources)) {
|
|
13698
|
+
const map = {};
|
|
13699
|
+
for (const res of resources) {
|
|
13700
|
+
if (typeof res === "string") map[normalizeResourceName$1(res)] = res;
|
|
13701
|
+
else if (Array.isArray(res) && typeof res[0] === "string") map[normalizeResourceName$1(res[0])] = res;
|
|
13702
|
+
else if (typeof res === "object" && res.resource) {
|
|
13703
|
+
map[normalizeResourceName$1(res.resource)] = { ...res };
|
|
13704
|
+
}
|
|
13705
|
+
}
|
|
13706
|
+
return map;
|
|
13707
|
+
}
|
|
13708
|
+
if (typeof resources === "object") {
|
|
13709
|
+
const map = {};
|
|
13710
|
+
for (const [src, dest] of Object.entries(resources)) {
|
|
13711
|
+
const normSrc = normalizeResourceName$1(src);
|
|
13712
|
+
if (typeof dest === "string") map[normSrc] = dest;
|
|
13713
|
+
else if (Array.isArray(dest)) {
|
|
13714
|
+
map[normSrc] = dest.map((item) => {
|
|
13715
|
+
if (typeof item === "string") return item;
|
|
13716
|
+
if (typeof item === "function") return item;
|
|
13717
|
+
if (typeof item === "object" && item.resource) {
|
|
13718
|
+
return { ...item };
|
|
13719
|
+
}
|
|
13720
|
+
return item;
|
|
13721
|
+
});
|
|
13722
|
+
} else if (typeof dest === "function") map[normSrc] = dest;
|
|
13723
|
+
else if (typeof dest === "object" && dest.resource) {
|
|
13724
|
+
map[normSrc] = { ...dest };
|
|
13725
|
+
}
|
|
13726
|
+
}
|
|
13727
|
+
return map;
|
|
13728
|
+
}
|
|
13729
|
+
if (typeof resources === "function") {
|
|
13730
|
+
return resources;
|
|
13731
|
+
}
|
|
13732
|
+
if (typeof resources === "string") {
|
|
13733
|
+
const map = { [normalizeResourceName$1(resources)]: resources };
|
|
13734
|
+
return map;
|
|
13735
|
+
}
|
|
13736
|
+
return {};
|
|
13737
|
+
}
|
|
13738
|
+
validateConfig() {
|
|
13739
|
+
const errors = [];
|
|
13740
|
+
if (!this.client && !this.connectionString) {
|
|
13741
|
+
errors.push("You must provide a client or a connectionString");
|
|
13742
|
+
}
|
|
13743
|
+
if (!this.resourcesMap || typeof this.resourcesMap === "object" && Object.keys(this.resourcesMap).length === 0) {
|
|
13744
|
+
errors.push("You must provide a resources map or array");
|
|
13745
|
+
}
|
|
13746
|
+
return { isValid: errors.length === 0, errors };
|
|
13747
|
+
}
|
|
13748
|
+
async initialize(database) {
|
|
13749
|
+
try {
|
|
13750
|
+
await super.initialize(database);
|
|
13751
|
+
if (this.client) {
|
|
13752
|
+
this.targetDatabase = this.client;
|
|
13753
|
+
} else if (this.connectionString) {
|
|
13754
|
+
const targetConfig = {
|
|
13755
|
+
connectionString: this.connectionString,
|
|
13756
|
+
region: this.region,
|
|
13757
|
+
keyPrefix: this.keyPrefix,
|
|
13758
|
+
verbose: this.config.verbose || false
|
|
13759
|
+
};
|
|
13760
|
+
this.targetDatabase = new S3db(targetConfig);
|
|
13761
|
+
await this.targetDatabase.connect();
|
|
13762
|
+
} else {
|
|
13763
|
+
throw new Error("S3dbReplicator: No client or connectionString provided");
|
|
13764
|
+
}
|
|
13765
|
+
this.emit("connected", {
|
|
13766
|
+
replicator: this.name,
|
|
13767
|
+
target: this.connectionString || "client-provided"
|
|
13768
|
+
});
|
|
13769
|
+
} catch (err) {
|
|
13770
|
+
throw err;
|
|
13771
|
+
}
|
|
13772
|
+
}
|
|
13773
|
+
// Support both object and parameter signatures for flexibility
|
|
13774
|
+
async replicate(resourceOrObj, operation, data, recordId, beforeData) {
|
|
13775
|
+
let resource, op, payload, id;
|
|
13776
|
+
if (typeof resourceOrObj === "object" && resourceOrObj.resource) {
|
|
13777
|
+
resource = resourceOrObj.resource;
|
|
13778
|
+
op = resourceOrObj.operation;
|
|
13779
|
+
payload = resourceOrObj.data;
|
|
13780
|
+
id = resourceOrObj.id;
|
|
13781
|
+
} else {
|
|
13782
|
+
resource = resourceOrObj;
|
|
13783
|
+
op = operation;
|
|
13784
|
+
payload = data;
|
|
13785
|
+
id = recordId;
|
|
13786
|
+
}
|
|
13787
|
+
const normResource = normalizeResourceName$1(resource);
|
|
13788
|
+
const destResource = this._resolveDestResource(normResource, payload);
|
|
13789
|
+
const destResourceObj = this._getDestResourceObj(destResource);
|
|
13790
|
+
const transformedData = this._applyTransformer(normResource, payload);
|
|
13791
|
+
let result;
|
|
13792
|
+
if (op === "insert") {
|
|
13793
|
+
result = await destResourceObj.insert(transformedData);
|
|
13794
|
+
} else if (op === "update") {
|
|
13795
|
+
result = await destResourceObj.update(id, transformedData);
|
|
13796
|
+
} else if (op === "delete") {
|
|
13797
|
+
result = await destResourceObj.delete(id);
|
|
13798
|
+
} else {
|
|
13799
|
+
throw new Error(`Invalid operation: ${op}. Supported operations are: insert, update, delete`);
|
|
13800
|
+
}
|
|
13801
|
+
return result;
|
|
13802
|
+
}
|
|
13803
|
+
_applyTransformer(resource, data) {
|
|
13804
|
+
const normResource = normalizeResourceName$1(resource);
|
|
13805
|
+
const entry = this.resourcesMap[normResource];
|
|
13806
|
+
let result;
|
|
13807
|
+
if (!entry) return data;
|
|
13808
|
+
if (Array.isArray(entry) && typeof entry[1] === "function") {
|
|
13809
|
+
result = entry[1](data);
|
|
13810
|
+
} else if (typeof entry === "function") {
|
|
13811
|
+
result = entry(data);
|
|
13812
|
+
} else if (typeof entry === "object") {
|
|
13813
|
+
if (typeof entry.transform === "function") result = entry.transform(data);
|
|
13814
|
+
else if (typeof entry.transformer === "function") result = entry.transformer(data);
|
|
13815
|
+
} else {
|
|
13816
|
+
result = data;
|
|
13817
|
+
}
|
|
13818
|
+
if (result && data && data.id && !result.id) result.id = data.id;
|
|
13819
|
+
if (!result && data) result = data;
|
|
13820
|
+
return result;
|
|
13821
|
+
}
|
|
13822
|
+
_resolveDestResource(resource, data) {
|
|
13823
|
+
const normResource = normalizeResourceName$1(resource);
|
|
13824
|
+
const entry = this.resourcesMap[normResource];
|
|
13825
|
+
if (!entry) return resource;
|
|
13826
|
+
if (Array.isArray(entry)) {
|
|
13827
|
+
if (typeof entry[0] === "string") return entry[0];
|
|
13828
|
+
if (typeof entry[0] === "object" && entry[0].resource) return entry[0].resource;
|
|
13829
|
+
if (typeof entry[0] === "function") return resource;
|
|
13830
|
+
}
|
|
13831
|
+
if (typeof entry === "string") return entry;
|
|
13832
|
+
if (resource && !targetResourceName) targetResourceName = resource;
|
|
13833
|
+
if (typeof entry === "object" && entry.resource) return entry.resource;
|
|
13834
|
+
return resource;
|
|
13835
|
+
}
|
|
13836
|
+
_getDestResourceObj(resource) {
|
|
13837
|
+
if (!this.client || !this.client.resources) return null;
|
|
13838
|
+
const available = Object.keys(this.client.resources);
|
|
13839
|
+
const norm = normalizeResourceName$1(resource);
|
|
13840
|
+
const found = available.find((r) => normalizeResourceName$1(r) === norm);
|
|
13841
|
+
if (!found) {
|
|
13842
|
+
throw new Error(`[S3dbReplicator] Destination resource not found: ${resource}. Available: ${available.join(", ")}`);
|
|
13843
|
+
}
|
|
13844
|
+
return this.client.resources[found];
|
|
13845
|
+
}
|
|
13846
|
+
async replicateBatch(resourceName, records) {
|
|
13847
|
+
if (!this.enabled || !this.shouldReplicateResource(resourceName)) {
|
|
13848
|
+
return { skipped: true, reason: "resource_not_included" };
|
|
13849
|
+
}
|
|
13850
|
+
const results = [];
|
|
13851
|
+
const errors = [];
|
|
13852
|
+
for (const record of records) {
|
|
13853
|
+
const [ok, err, result] = await try_fn_default(() => this.replicate({
|
|
13854
|
+
resource: resourceName,
|
|
13855
|
+
operation: record.operation,
|
|
13856
|
+
id: record.id,
|
|
13857
|
+
data: record.data,
|
|
13858
|
+
beforeData: record.beforeData
|
|
13859
|
+
}));
|
|
13860
|
+
if (ok) results.push(result);
|
|
13861
|
+
else errors.push({ id: record.id, error: err.message });
|
|
13862
|
+
}
|
|
13863
|
+
this.emit("batch_replicated", {
|
|
13864
|
+
replicator: this.name,
|
|
13865
|
+
resourceName,
|
|
13866
|
+
total: records.length,
|
|
13867
|
+
successful: results.length,
|
|
13868
|
+
errors: errors.length
|
|
13869
|
+
});
|
|
13870
|
+
return {
|
|
13871
|
+
success: errors.length === 0,
|
|
13872
|
+
results,
|
|
13873
|
+
errors,
|
|
13874
|
+
total: records.length
|
|
13875
|
+
};
|
|
13876
|
+
}
|
|
13877
|
+
async testConnection() {
|
|
13878
|
+
const [ok, err] = await try_fn_default(async () => {
|
|
13879
|
+
if (!this.targetDatabase) {
|
|
13880
|
+
await this.initialize(this.database);
|
|
13881
|
+
}
|
|
13882
|
+
await this.targetDatabase.listResources();
|
|
13883
|
+
return true;
|
|
13884
|
+
});
|
|
13885
|
+
if (ok) return true;
|
|
13886
|
+
this.emit("connection_error", {
|
|
13887
|
+
replicator: this.name,
|
|
13888
|
+
error: err.message
|
|
13889
|
+
});
|
|
13890
|
+
return false;
|
|
13891
|
+
}
|
|
13892
|
+
async getStatus() {
|
|
13893
|
+
const baseStatus = await super.getStatus();
|
|
13894
|
+
return {
|
|
13895
|
+
...baseStatus,
|
|
13896
|
+
connected: !!this.targetDatabase,
|
|
13897
|
+
targetDatabase: this.connectionString || "client-provided",
|
|
13898
|
+
resources: Object.keys(this.resourcesMap || {}),
|
|
13899
|
+
totalreplicators: this.listenerCount("replicated"),
|
|
13900
|
+
totalErrors: this.listenerCount("replicator_error")
|
|
13901
|
+
};
|
|
13902
|
+
}
|
|
13903
|
+
async cleanup() {
|
|
13904
|
+
if (this.targetDatabase) {
|
|
13905
|
+
this.targetDatabase.removeAllListeners();
|
|
13906
|
+
}
|
|
13907
|
+
await super.cleanup();
|
|
13908
|
+
}
|
|
13909
|
+
shouldReplicateResource(resource, action) {
|
|
13910
|
+
const normResource = normalizeResourceName$1(resource);
|
|
13911
|
+
const entry = this.resourcesMap[normResource];
|
|
13912
|
+
if (!entry) return false;
|
|
13913
|
+
if (!action) return true;
|
|
13914
|
+
if (Array.isArray(entry)) {
|
|
13915
|
+
for (const item of entry) {
|
|
13916
|
+
if (typeof item === "object" && item.resource) {
|
|
13917
|
+
if (item.actions && Array.isArray(item.actions)) {
|
|
13918
|
+
if (item.actions.includes(action)) return true;
|
|
13919
|
+
} else {
|
|
13920
|
+
return true;
|
|
13921
|
+
}
|
|
13922
|
+
} else if (typeof item === "string" || typeof item === "function") {
|
|
13923
|
+
return true;
|
|
13924
|
+
}
|
|
13925
|
+
}
|
|
13926
|
+
return false;
|
|
13927
|
+
}
|
|
13928
|
+
if (typeof entry === "object" && entry.resource) {
|
|
13929
|
+
if (entry.actions && Array.isArray(entry.actions)) {
|
|
13930
|
+
return entry.actions.includes(action);
|
|
13931
|
+
}
|
|
13932
|
+
return true;
|
|
13933
|
+
}
|
|
13934
|
+
if (typeof entry === "string" || typeof entry === "function") {
|
|
13935
|
+
return true;
|
|
13936
|
+
}
|
|
13937
|
+
return false;
|
|
13938
|
+
}
|
|
13939
|
+
}
|
|
13940
|
+
var s3db_replicator_class_default = S3dbReplicator;
|
|
13941
|
+
|
|
13942
|
+
class SqsReplicator extends base_replicator_class_default {
|
|
13943
|
+
constructor(config = {}, resources = [], client = null) {
|
|
13944
|
+
super(config);
|
|
13945
|
+
this.client = client;
|
|
13946
|
+
this.queueUrl = config.queueUrl;
|
|
13947
|
+
this.queues = config.queues || {};
|
|
13948
|
+
this.defaultQueue = config.defaultQueue || config.defaultQueueUrl || config.queueUrlDefault;
|
|
13949
|
+
this.region = config.region || "us-east-1";
|
|
13950
|
+
this.sqsClient = client || null;
|
|
13951
|
+
this.messageGroupId = config.messageGroupId;
|
|
13952
|
+
this.deduplicationId = config.deduplicationId;
|
|
13953
|
+
if (Array.isArray(resources)) {
|
|
13954
|
+
this.resources = {};
|
|
13955
|
+
for (const resource of resources) {
|
|
13956
|
+
if (typeof resource === "string") {
|
|
13957
|
+
this.resources[resource] = true;
|
|
13958
|
+
} else if (typeof resource === "object" && resource.name) {
|
|
13959
|
+
this.resources[resource.name] = resource;
|
|
13960
|
+
}
|
|
13961
|
+
}
|
|
13962
|
+
} else if (typeof resources === "object") {
|
|
13963
|
+
this.resources = resources;
|
|
13964
|
+
for (const [resourceName, resourceConfig] of Object.entries(resources)) {
|
|
13965
|
+
if (resourceConfig && resourceConfig.queueUrl) {
|
|
13966
|
+
this.queues[resourceName] = resourceConfig.queueUrl;
|
|
13967
|
+
}
|
|
13968
|
+
}
|
|
13969
|
+
} else {
|
|
13970
|
+
this.resources = {};
|
|
13971
|
+
}
|
|
13972
|
+
}
|
|
13973
|
+
validateConfig() {
|
|
13974
|
+
const errors = [];
|
|
13975
|
+
if (!this.queueUrl && Object.keys(this.queues).length === 0 && !this.defaultQueue && !this.resourceQueueMap) {
|
|
13976
|
+
errors.push("Either queueUrl, queues object, defaultQueue, or resourceQueueMap must be provided");
|
|
13977
|
+
}
|
|
13978
|
+
return {
|
|
13979
|
+
isValid: errors.length === 0,
|
|
13980
|
+
errors
|
|
13981
|
+
};
|
|
13982
|
+
}
|
|
13983
|
+
getQueueUrlsForResource(resource) {
|
|
13984
|
+
if (this.resourceQueueMap && this.resourceQueueMap[resource]) {
|
|
13985
|
+
return this.resourceQueueMap[resource];
|
|
13986
|
+
}
|
|
13987
|
+
if (this.queues[resource]) {
|
|
13988
|
+
return [this.queues[resource]];
|
|
13989
|
+
}
|
|
13990
|
+
if (this.queueUrl) {
|
|
13991
|
+
return [this.queueUrl];
|
|
13992
|
+
}
|
|
13993
|
+
if (this.defaultQueue) {
|
|
13994
|
+
return [this.defaultQueue];
|
|
13995
|
+
}
|
|
13996
|
+
throw new Error(`No queue URL found for resource '${resource}'`);
|
|
13997
|
+
}
|
|
13998
|
+
_applyTransformer(resource, data) {
|
|
13999
|
+
const entry = this.resources[resource];
|
|
14000
|
+
let result = data;
|
|
14001
|
+
if (!entry) return data;
|
|
14002
|
+
if (typeof entry.transform === "function") {
|
|
14003
|
+
result = entry.transform(data);
|
|
14004
|
+
} else if (typeof entry.transformer === "function") {
|
|
14005
|
+
result = entry.transformer(data);
|
|
14006
|
+
}
|
|
14007
|
+
return result || data;
|
|
14008
|
+
}
|
|
14009
|
+
/**
|
|
14010
|
+
* Create standardized message structure
|
|
14011
|
+
*/
|
|
14012
|
+
createMessage(resource, operation, data, id, beforeData = null) {
|
|
14013
|
+
const baseMessage = {
|
|
14014
|
+
resource,
|
|
14015
|
+
// padronizado para 'resource'
|
|
14016
|
+
action: operation,
|
|
14017
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
14018
|
+
source: "s3db-replicator"
|
|
14019
|
+
};
|
|
14020
|
+
switch (operation) {
|
|
14021
|
+
case "insert":
|
|
14022
|
+
return {
|
|
14023
|
+
...baseMessage,
|
|
14024
|
+
data
|
|
14025
|
+
};
|
|
14026
|
+
case "update":
|
|
14027
|
+
return {
|
|
14028
|
+
...baseMessage,
|
|
14029
|
+
before: beforeData,
|
|
14030
|
+
data
|
|
14031
|
+
};
|
|
14032
|
+
case "delete":
|
|
14033
|
+
return {
|
|
14034
|
+
...baseMessage,
|
|
14035
|
+
data
|
|
14036
|
+
};
|
|
14037
|
+
default:
|
|
14038
|
+
return {
|
|
14039
|
+
...baseMessage,
|
|
14040
|
+
data
|
|
14041
|
+
};
|
|
14042
|
+
}
|
|
14043
|
+
}
|
|
14044
|
+
async initialize(database, client) {
|
|
14045
|
+
await super.initialize(database);
|
|
14046
|
+
if (!this.sqsClient) {
|
|
14047
|
+
const [ok, err, sdk] = await try_fn_default(() => import('@aws-sdk/client-sqs'));
|
|
14048
|
+
if (!ok) {
|
|
14049
|
+
this.emit("initialization_error", {
|
|
14050
|
+
replicator: this.name,
|
|
14051
|
+
error: err.message
|
|
14052
|
+
});
|
|
14053
|
+
throw err;
|
|
14054
|
+
}
|
|
14055
|
+
const { SQSClient } = sdk;
|
|
14056
|
+
this.sqsClient = client || new SQSClient({
|
|
14057
|
+
region: this.region,
|
|
14058
|
+
credentials: this.config.credentials
|
|
14059
|
+
});
|
|
14060
|
+
this.emit("initialized", {
|
|
14061
|
+
replicator: this.name,
|
|
14062
|
+
queueUrl: this.queueUrl,
|
|
14063
|
+
queues: this.queues,
|
|
14064
|
+
defaultQueue: this.defaultQueue
|
|
14065
|
+
});
|
|
14066
|
+
}
|
|
14067
|
+
}
|
|
14068
|
+
async replicate(resource, operation, data, id, beforeData = null) {
|
|
14069
|
+
if (!this.enabled || !this.shouldReplicateResource(resource)) {
|
|
14070
|
+
return { skipped: true, reason: "resource_not_included" };
|
|
14071
|
+
}
|
|
14072
|
+
const [ok, err, result] = await try_fn_default(async () => {
|
|
14073
|
+
const { SendMessageCommand } = await import('@aws-sdk/client-sqs');
|
|
14074
|
+
const queueUrls = this.getQueueUrlsForResource(resource);
|
|
14075
|
+
const transformedData = this._applyTransformer(resource, data);
|
|
14076
|
+
const message = this.createMessage(resource, operation, transformedData, id, beforeData);
|
|
14077
|
+
const results = [];
|
|
14078
|
+
for (const queueUrl of queueUrls) {
|
|
14079
|
+
const command = new SendMessageCommand({
|
|
14080
|
+
QueueUrl: queueUrl,
|
|
14081
|
+
MessageBody: JSON.stringify(message),
|
|
14082
|
+
MessageGroupId: this.messageGroupId,
|
|
14083
|
+
MessageDeduplicationId: this.deduplicationId ? `${resource}:${operation}:${id}` : void 0
|
|
14084
|
+
});
|
|
14085
|
+
const result2 = await this.sqsClient.send(command);
|
|
14086
|
+
results.push({ queueUrl, messageId: result2.MessageId });
|
|
14087
|
+
this.emit("replicated", {
|
|
14088
|
+
replicator: this.name,
|
|
14089
|
+
resource,
|
|
14090
|
+
operation,
|
|
14091
|
+
id,
|
|
14092
|
+
queueUrl,
|
|
14093
|
+
messageId: result2.MessageId,
|
|
14094
|
+
success: true
|
|
14095
|
+
});
|
|
14096
|
+
}
|
|
14097
|
+
return { success: true, results };
|
|
14098
|
+
});
|
|
14099
|
+
if (ok) return result;
|
|
14100
|
+
this.emit("replicator_error", {
|
|
14101
|
+
replicator: this.name,
|
|
14102
|
+
resource,
|
|
14103
|
+
operation,
|
|
14104
|
+
id,
|
|
14105
|
+
error: err.message
|
|
14106
|
+
});
|
|
14107
|
+
return { success: false, error: err.message };
|
|
14108
|
+
}
|
|
14109
|
+
async replicateBatch(resource, records) {
|
|
14110
|
+
if (!this.enabled || !this.shouldReplicateResource(resource)) {
|
|
14111
|
+
return { skipped: true, reason: "resource_not_included" };
|
|
14112
|
+
}
|
|
14113
|
+
const [ok, err, result] = await try_fn_default(async () => {
|
|
14114
|
+
const { SendMessageBatchCommand } = await import('@aws-sdk/client-sqs');
|
|
14115
|
+
const queueUrls = this.getQueueUrlsForResource(resource);
|
|
14116
|
+
const batchSize = 10;
|
|
14117
|
+
const batches = [];
|
|
14118
|
+
for (let i = 0; i < records.length; i += batchSize) {
|
|
14119
|
+
batches.push(records.slice(i, i + batchSize));
|
|
14120
|
+
}
|
|
14121
|
+
const results = [];
|
|
14122
|
+
const errors = [];
|
|
14123
|
+
for (const batch of batches) {
|
|
14124
|
+
const [okBatch, errBatch] = await try_fn_default(async () => {
|
|
14125
|
+
const entries = batch.map((record, index) => ({
|
|
14126
|
+
Id: `${record.id}-${index}`,
|
|
14127
|
+
MessageBody: JSON.stringify(this.createMessage(
|
|
14128
|
+
resource,
|
|
14129
|
+
record.operation,
|
|
14130
|
+
record.data,
|
|
14131
|
+
record.id,
|
|
14132
|
+
record.beforeData
|
|
14133
|
+
)),
|
|
14134
|
+
MessageGroupId: this.messageGroupId,
|
|
14135
|
+
MessageDeduplicationId: this.deduplicationId ? `${resource}:${record.operation}:${record.id}` : void 0
|
|
14136
|
+
}));
|
|
14137
|
+
const command = new SendMessageBatchCommand({
|
|
14138
|
+
QueueUrl: queueUrls[0],
|
|
14139
|
+
// Assuming all queueUrls in a batch are the same for batching
|
|
14140
|
+
Entries: entries
|
|
14141
|
+
});
|
|
14142
|
+
const result2 = await this.sqsClient.send(command);
|
|
14143
|
+
results.push(result2);
|
|
14144
|
+
});
|
|
14145
|
+
if (!okBatch) {
|
|
14146
|
+
errors.push({ batch: batch.length, error: errBatch.message });
|
|
14147
|
+
if (errBatch.message && (errBatch.message.includes("Batch error") || errBatch.message.includes("Connection") || errBatch.message.includes("Network"))) {
|
|
14148
|
+
throw errBatch;
|
|
14149
|
+
}
|
|
14150
|
+
}
|
|
14151
|
+
}
|
|
14152
|
+
this.emit("batch_replicated", {
|
|
14153
|
+
replicator: this.name,
|
|
14154
|
+
resource,
|
|
14155
|
+
queueUrl: queueUrls[0],
|
|
14156
|
+
// Assuming all queueUrls in a batch are the same for batching
|
|
14157
|
+
total: records.length,
|
|
14158
|
+
successful: results.length,
|
|
14159
|
+
errors: errors.length
|
|
14160
|
+
});
|
|
14161
|
+
return {
|
|
14162
|
+
success: errors.length === 0,
|
|
14163
|
+
results,
|
|
14164
|
+
errors,
|
|
14165
|
+
total: records.length,
|
|
14166
|
+
queueUrl: queueUrls[0]
|
|
14167
|
+
// Assuming all queueUrls in a batch are the same for batching
|
|
14168
|
+
};
|
|
14169
|
+
});
|
|
14170
|
+
if (ok) return result;
|
|
14171
|
+
const errorMessage = err?.message || err || "Unknown error";
|
|
14172
|
+
this.emit("batch_replicator_error", {
|
|
14173
|
+
replicator: this.name,
|
|
14174
|
+
resource,
|
|
14175
|
+
error: errorMessage
|
|
14176
|
+
});
|
|
14177
|
+
return { success: false, error: errorMessage };
|
|
14178
|
+
}
|
|
14179
|
+
async testConnection() {
|
|
14180
|
+
const [ok, err] = await try_fn_default(async () => {
|
|
14181
|
+
if (!this.sqsClient) {
|
|
14182
|
+
await this.initialize(this.database);
|
|
14183
|
+
}
|
|
14184
|
+
const { GetQueueAttributesCommand } = await import('@aws-sdk/client-sqs');
|
|
14185
|
+
const command = new GetQueueAttributesCommand({
|
|
14186
|
+
QueueUrl: this.queueUrl,
|
|
14187
|
+
AttributeNames: ["QueueArn"]
|
|
14188
|
+
});
|
|
14189
|
+
await this.sqsClient.send(command);
|
|
14190
|
+
return true;
|
|
14191
|
+
});
|
|
14192
|
+
if (ok) return true;
|
|
14193
|
+
this.emit("connection_error", {
|
|
14194
|
+
replicator: this.name,
|
|
14195
|
+
error: err.message
|
|
14196
|
+
});
|
|
14197
|
+
return false;
|
|
14198
|
+
}
|
|
14199
|
+
async getStatus() {
|
|
14200
|
+
const baseStatus = await super.getStatus();
|
|
14201
|
+
return {
|
|
14202
|
+
...baseStatus,
|
|
14203
|
+
connected: !!this.sqsClient,
|
|
14204
|
+
queueUrl: this.queueUrl,
|
|
14205
|
+
region: this.region,
|
|
14206
|
+
resources: Object.keys(this.resources || {}),
|
|
14207
|
+
totalreplicators: this.listenerCount("replicated"),
|
|
14208
|
+
totalErrors: this.listenerCount("replicator_error")
|
|
14209
|
+
};
|
|
14210
|
+
}
|
|
14211
|
+
async cleanup() {
|
|
14212
|
+
if (this.sqsClient) {
|
|
14213
|
+
this.sqsClient.destroy();
|
|
14214
|
+
}
|
|
14215
|
+
await super.cleanup();
|
|
14216
|
+
}
|
|
14217
|
+
shouldReplicateResource(resource) {
|
|
14218
|
+
const result = this.resourceQueueMap && Object.keys(this.resourceQueueMap).includes(resource) || this.queues && Object.keys(this.queues).includes(resource) || !!(this.defaultQueue || this.queueUrl) || this.resources && Object.keys(this.resources).includes(resource) || false;
|
|
14219
|
+
return result;
|
|
14220
|
+
}
|
|
14221
|
+
}
|
|
14222
|
+
var sqs_replicator_class_default = SqsReplicator;
|
|
14223
|
+
|
|
14224
|
+
const REPLICATOR_DRIVERS = {
|
|
14225
|
+
s3db: s3db_replicator_class_default,
|
|
14226
|
+
sqs: sqs_replicator_class_default,
|
|
14227
|
+
bigquery: bigquery_replicator_class_default,
|
|
14228
|
+
postgres: postgres_replicator_class_default
|
|
14229
|
+
};
|
|
14230
|
+
function createReplicator(driver, config = {}, resources = [], client = null) {
|
|
14231
|
+
const ReplicatorClass = REPLICATOR_DRIVERS[driver];
|
|
14232
|
+
if (!ReplicatorClass) {
|
|
14233
|
+
throw new Error(`Unknown replicator driver: ${driver}. Available drivers: ${Object.keys(REPLICATOR_DRIVERS).join(", ")}`);
|
|
14234
|
+
}
|
|
14235
|
+
return new ReplicatorClass(config, resources, client);
|
|
14236
|
+
}
|
|
14237
|
+
|
|
12100
14238
|
function normalizeResourceName(name) {
|
|
12101
14239
|
return typeof name === "string" ? name.trim().toLowerCase() : name;
|
|
12102
14240
|
}
|
|
@@ -12144,7 +14282,7 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
12144
14282
|
return filtered;
|
|
12145
14283
|
}
|
|
12146
14284
|
installEventListeners(resource, database, plugin) {
|
|
12147
|
-
if (!resource || this.eventListenersInstalled.has(resource.name)) {
|
|
14285
|
+
if (!resource || this.eventListenersInstalled.has(resource.name) || resource.name === this.config.replicatorLogResource) {
|
|
12148
14286
|
return;
|
|
12149
14287
|
}
|
|
12150
14288
|
resource.on("insert", async (data) => {
|
|
@@ -12221,10 +14359,15 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
12221
14359
|
this.installEventListeners(resource, database, this);
|
|
12222
14360
|
}
|
|
12223
14361
|
}
|
|
14362
|
+
createReplicator(driver, config, resources, client) {
|
|
14363
|
+
return createReplicator(driver, config, resources, client);
|
|
14364
|
+
}
|
|
12224
14365
|
async initializeReplicators(database) {
|
|
12225
14366
|
for (const replicatorConfig of this.config.replicators) {
|
|
12226
|
-
const { driver, config, resources } = replicatorConfig;
|
|
12227
|
-
const
|
|
14367
|
+
const { driver, config = {}, resources, client, ...otherConfig } = replicatorConfig;
|
|
14368
|
+
const replicatorResources = resources || config.resources || {};
|
|
14369
|
+
const mergedConfig = { ...config, ...otherConfig };
|
|
14370
|
+
const replicator = this.createReplicator(driver, mergedConfig, replicatorResources, client);
|
|
12228
14371
|
if (replicator) {
|
|
12229
14372
|
await replicator.initialize(database);
|
|
12230
14373
|
this.replicators.push(replicator);
|
|
@@ -12235,6 +14378,66 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
12235
14378
|
}
|
|
12236
14379
|
async stop() {
|
|
12237
14380
|
}
|
|
14381
|
+
filterInternalFields(data) {
|
|
14382
|
+
if (!data || typeof data !== "object") return data;
|
|
14383
|
+
const filtered = {};
|
|
14384
|
+
for (const [key, value] of Object.entries(data)) {
|
|
14385
|
+
if (!key.startsWith("_") && !key.startsWith("$")) {
|
|
14386
|
+
filtered[key] = value;
|
|
14387
|
+
}
|
|
14388
|
+
}
|
|
14389
|
+
return filtered;
|
|
14390
|
+
}
|
|
14391
|
+
async uploadMetadataFile(database) {
|
|
14392
|
+
if (typeof database.uploadMetadataFile === "function") {
|
|
14393
|
+
await database.uploadMetadataFile();
|
|
14394
|
+
}
|
|
14395
|
+
}
|
|
14396
|
+
async getCompleteData(resource, data) {
|
|
14397
|
+
try {
|
|
14398
|
+
const [ok, err, record] = await try_fn_default(() => resource.get(data.id));
|
|
14399
|
+
if (ok && record) {
|
|
14400
|
+
return record;
|
|
14401
|
+
}
|
|
14402
|
+
} catch (error) {
|
|
14403
|
+
}
|
|
14404
|
+
return data;
|
|
14405
|
+
}
|
|
14406
|
+
async retryWithBackoff(operation, maxRetries = 3) {
|
|
14407
|
+
let lastError;
|
|
14408
|
+
for (let attempt = 1; attempt <= maxRetries; attempt++) {
|
|
14409
|
+
try {
|
|
14410
|
+
return await operation();
|
|
14411
|
+
} catch (error) {
|
|
14412
|
+
lastError = error;
|
|
14413
|
+
if (attempt === maxRetries) {
|
|
14414
|
+
throw error;
|
|
14415
|
+
}
|
|
14416
|
+
const delay = Math.pow(2, attempt - 1) * 1e3;
|
|
14417
|
+
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
14418
|
+
}
|
|
14419
|
+
}
|
|
14420
|
+
throw lastError;
|
|
14421
|
+
}
|
|
14422
|
+
async logError(replicator, resourceName, operation, recordId, data, error) {
|
|
14423
|
+
try {
|
|
14424
|
+
const logResourceName = this.config.replicatorLogResource;
|
|
14425
|
+
if (this.database && this.database.resources && this.database.resources[logResourceName]) {
|
|
14426
|
+
const logResource = this.database.resources[logResourceName];
|
|
14427
|
+
await logResource.insert({
|
|
14428
|
+
replicator: replicator.name || replicator.id,
|
|
14429
|
+
resourceName,
|
|
14430
|
+
operation,
|
|
14431
|
+
recordId,
|
|
14432
|
+
data: JSON.stringify(data),
|
|
14433
|
+
error: error.message,
|
|
14434
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
14435
|
+
status: "error"
|
|
14436
|
+
});
|
|
14437
|
+
}
|
|
14438
|
+
} catch (logError) {
|
|
14439
|
+
}
|
|
14440
|
+
}
|
|
12238
14441
|
async processReplicatorEvent(operation, resourceName, recordId, data, beforeData = null) {
|
|
12239
14442
|
if (!this.config.enabled) return;
|
|
12240
14443
|
const applicableReplicators = this.replicators.filter((replicator) => {
|
|
@@ -12365,7 +14568,7 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
12365
14568
|
async getreplicatorStats() {
|
|
12366
14569
|
const replicatorStats = await Promise.all(
|
|
12367
14570
|
this.replicators.map(async (replicator) => {
|
|
12368
|
-
const status = await replicator.
|
|
14571
|
+
const status = await replicator.getStatus();
|
|
12369
14572
|
return {
|
|
12370
14573
|
id: replicator.id,
|
|
12371
14574
|
driver: replicator.driver,
|
|
@@ -12439,12 +14642,12 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
12439
14642
|
this.stats.lastSync = (/* @__PURE__ */ new Date()).toISOString();
|
|
12440
14643
|
for (const resourceName in this.database.resources) {
|
|
12441
14644
|
if (normalizeResourceName(resourceName) === normalizeResourceName("replicator_logs")) continue;
|
|
12442
|
-
if (replicator.
|
|
14645
|
+
if (replicator.shouldReplicateResource(resourceName)) {
|
|
12443
14646
|
this.emit("replicator.sync.resource", { resourceName, replicatorId });
|
|
12444
14647
|
const resource = this.database.resources[resourceName];
|
|
12445
14648
|
const allRecords = await resource.getAll();
|
|
12446
14649
|
for (const record of allRecords) {
|
|
12447
|
-
await replicator.
|
|
14650
|
+
await replicator.replicate(resourceName, "insert", record, record.id);
|
|
12448
14651
|
}
|
|
12449
14652
|
}
|
|
12450
14653
|
}
|