s3db.js 7.3.1 → 7.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/s3db.cjs.js +2394 -191
- package/dist/s3db.cjs.min.js +1 -1
- package/dist/s3db.es.js +2394 -191
- package/dist/s3db.es.min.js +1 -1
- package/dist/s3db.iife.js +2393 -190
- package/dist/s3db.iife.min.js +1 -1
- package/package.json +25 -26
- package/src/plugins/cache/filesystem-cache.class.js +1 -1
- package/src/plugins/cache/partition-aware-filesystem-cache.class.js +1 -1
- package/src/plugins/cache/s3-cache.class.js +1 -1
package/dist/s3db.es.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { customAlphabet, urlAlphabet } from 'nanoid';
|
|
2
|
-
import zlib from 'zlib';
|
|
2
|
+
import zlib from 'node:zlib';
|
|
3
3
|
import { PromisePool } from '@supercharge/promise-pool';
|
|
4
4
|
import { ReadableStream } from 'node:stream/web';
|
|
5
5
|
import { chunk, merge, isString as isString$1, isEmpty, invert, uniq, cloneDeep, get, set, isObject as isObject$1, isFunction as isFunction$1 } from 'lodash-es';
|
|
@@ -1341,7 +1341,8 @@ class AuditPlugin extends plugin_class_default {
|
|
|
1341
1341
|
version: "2.0"
|
|
1342
1342
|
})
|
|
1343
1343
|
};
|
|
1344
|
-
this.logAudit(auditRecord).catch(
|
|
1344
|
+
this.logAudit(auditRecord).catch(() => {
|
|
1345
|
+
});
|
|
1345
1346
|
});
|
|
1346
1347
|
resource.on("update", async (data) => {
|
|
1347
1348
|
const recordId = data.id;
|
|
@@ -1367,7 +1368,8 @@ class AuditPlugin extends plugin_class_default {
|
|
|
1367
1368
|
version: "2.0"
|
|
1368
1369
|
})
|
|
1369
1370
|
};
|
|
1370
|
-
this.logAudit(auditRecord).catch(
|
|
1371
|
+
this.logAudit(auditRecord).catch(() => {
|
|
1372
|
+
});
|
|
1371
1373
|
});
|
|
1372
1374
|
resource.on("delete", async (data) => {
|
|
1373
1375
|
const recordId = data.id;
|
|
@@ -1393,7 +1395,8 @@ class AuditPlugin extends plugin_class_default {
|
|
|
1393
1395
|
version: "2.0"
|
|
1394
1396
|
})
|
|
1395
1397
|
};
|
|
1396
|
-
this.logAudit(auditRecord).catch(
|
|
1398
|
+
this.logAudit(auditRecord).catch(() => {
|
|
1399
|
+
});
|
|
1397
1400
|
});
|
|
1398
1401
|
resource.useMiddleware("deleteMany", async (ctx, next) => {
|
|
1399
1402
|
const ids = ctx.args[0];
|
|
@@ -1426,7 +1429,8 @@ class AuditPlugin extends plugin_class_default {
|
|
|
1426
1429
|
batchOperation: true
|
|
1427
1430
|
})
|
|
1428
1431
|
};
|
|
1429
|
-
this.logAudit(auditRecord).catch(
|
|
1432
|
+
this.logAudit(auditRecord).catch(() => {
|
|
1433
|
+
});
|
|
1430
1434
|
}
|
|
1431
1435
|
}
|
|
1432
1436
|
return result;
|
|
@@ -2073,6 +2077,16 @@ if (typeof Object.create === 'function'){
|
|
|
2073
2077
|
// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
|
|
2074
2078
|
// USE OR OTHER DEALINGS IN THE SOFTWARE.
|
|
2075
2079
|
|
|
2080
|
+
var getOwnPropertyDescriptors = Object.getOwnPropertyDescriptors ||
|
|
2081
|
+
function getOwnPropertyDescriptors(obj) {
|
|
2082
|
+
var keys = Object.keys(obj);
|
|
2083
|
+
var descriptors = {};
|
|
2084
|
+
for (var i = 0; i < keys.length; i++) {
|
|
2085
|
+
descriptors[keys[i]] = Object.getOwnPropertyDescriptor(obj, keys[i]);
|
|
2086
|
+
}
|
|
2087
|
+
return descriptors;
|
|
2088
|
+
};
|
|
2089
|
+
|
|
2076
2090
|
var formatRegExp = /%[sdj%]/g;
|
|
2077
2091
|
function format(f) {
|
|
2078
2092
|
if (!isString(f)) {
|
|
@@ -2558,6 +2572,64 @@ function hasOwnProperty(obj, prop) {
|
|
|
2558
2572
|
return Object.prototype.hasOwnProperty.call(obj, prop);
|
|
2559
2573
|
}
|
|
2560
2574
|
|
|
2575
|
+
var kCustomPromisifiedSymbol = typeof Symbol !== 'undefined' ? Symbol('util.promisify.custom') : undefined;
|
|
2576
|
+
|
|
2577
|
+
function promisify(original) {
|
|
2578
|
+
if (typeof original !== 'function')
|
|
2579
|
+
throw new TypeError('The "original" argument must be of type Function');
|
|
2580
|
+
|
|
2581
|
+
if (kCustomPromisifiedSymbol && original[kCustomPromisifiedSymbol]) {
|
|
2582
|
+
var fn = original[kCustomPromisifiedSymbol];
|
|
2583
|
+
if (typeof fn !== 'function') {
|
|
2584
|
+
throw new TypeError('The "util.promisify.custom" argument must be of type Function');
|
|
2585
|
+
}
|
|
2586
|
+
Object.defineProperty(fn, kCustomPromisifiedSymbol, {
|
|
2587
|
+
value: fn, enumerable: false, writable: false, configurable: true
|
|
2588
|
+
});
|
|
2589
|
+
return fn;
|
|
2590
|
+
}
|
|
2591
|
+
|
|
2592
|
+
function fn() {
|
|
2593
|
+
var promiseResolve, promiseReject;
|
|
2594
|
+
var promise = new Promise(function (resolve, reject) {
|
|
2595
|
+
promiseResolve = resolve;
|
|
2596
|
+
promiseReject = reject;
|
|
2597
|
+
});
|
|
2598
|
+
|
|
2599
|
+
var args = [];
|
|
2600
|
+
for (var i = 0; i < arguments.length; i++) {
|
|
2601
|
+
args.push(arguments[i]);
|
|
2602
|
+
}
|
|
2603
|
+
args.push(function (err, value) {
|
|
2604
|
+
if (err) {
|
|
2605
|
+
promiseReject(err);
|
|
2606
|
+
} else {
|
|
2607
|
+
promiseResolve(value);
|
|
2608
|
+
}
|
|
2609
|
+
});
|
|
2610
|
+
|
|
2611
|
+
try {
|
|
2612
|
+
original.apply(this, args);
|
|
2613
|
+
} catch (err) {
|
|
2614
|
+
promiseReject(err);
|
|
2615
|
+
}
|
|
2616
|
+
|
|
2617
|
+
return promise;
|
|
2618
|
+
}
|
|
2619
|
+
|
|
2620
|
+
Object.setPrototypeOf(fn, Object.getPrototypeOf(original));
|
|
2621
|
+
|
|
2622
|
+
if (kCustomPromisifiedSymbol) Object.defineProperty(fn, kCustomPromisifiedSymbol, {
|
|
2623
|
+
value: fn, enumerable: false, writable: false, configurable: true
|
|
2624
|
+
});
|
|
2625
|
+
return Object.defineProperties(
|
|
2626
|
+
fn,
|
|
2627
|
+
getOwnPropertyDescriptors(original)
|
|
2628
|
+
);
|
|
2629
|
+
}
|
|
2630
|
+
|
|
2631
|
+
promisify.custom = kCustomPromisifiedSymbol;
|
|
2632
|
+
|
|
2561
2633
|
var lookup = [];
|
|
2562
2634
|
var revLookup = [];
|
|
2563
2635
|
var Arr = typeof Uint8Array !== 'undefined' ? Uint8Array : Array;
|
|
@@ -6808,186 +6880,1026 @@ class MemoryCache extends Cache {
|
|
|
6808
6880
|
}
|
|
6809
6881
|
var memory_cache_class_default = MemoryCache;
|
|
6810
6882
|
|
|
6811
|
-
|
|
6812
|
-
|
|
6813
|
-
|
|
6814
|
-
|
|
6815
|
-
|
|
6816
|
-
|
|
6817
|
-
|
|
6883
|
+
var fs = {};
|
|
6884
|
+
|
|
6885
|
+
const readFile$1 = promisify(fs.readFile);
|
|
6886
|
+
const writeFile$1 = promisify(fs.writeFile);
|
|
6887
|
+
const unlink = promisify(fs.unlink);
|
|
6888
|
+
const readdir$1 = promisify(fs.readdir);
|
|
6889
|
+
const stat$1 = promisify(fs.stat);
|
|
6890
|
+
const mkdir = promisify(fs.mkdir);
|
|
6891
|
+
class FilesystemCache extends Cache {
|
|
6892
|
+
constructor({
|
|
6893
|
+
directory,
|
|
6894
|
+
prefix = "cache",
|
|
6895
|
+
ttl = 36e5,
|
|
6896
|
+
enableCompression = true,
|
|
6897
|
+
compressionThreshold = 1024,
|
|
6898
|
+
createDirectory = true,
|
|
6899
|
+
fileExtension = ".cache",
|
|
6900
|
+
enableMetadata = true,
|
|
6901
|
+
maxFileSize = 10485760,
|
|
6902
|
+
// 10MB
|
|
6903
|
+
enableStats = false,
|
|
6904
|
+
enableCleanup = true,
|
|
6905
|
+
cleanupInterval = 3e5,
|
|
6906
|
+
// 5 minutes
|
|
6907
|
+
encoding = "utf8",
|
|
6908
|
+
fileMode = 420,
|
|
6909
|
+
enableBackup = false,
|
|
6910
|
+
backupSuffix = ".bak",
|
|
6911
|
+
enableLocking = false,
|
|
6912
|
+
lockTimeout = 5e3,
|
|
6913
|
+
enableJournal = false,
|
|
6914
|
+
journalFile = "cache.journal",
|
|
6915
|
+
...config
|
|
6916
|
+
}) {
|
|
6917
|
+
super(config);
|
|
6918
|
+
if (!directory) {
|
|
6919
|
+
throw new Error("FilesystemCache: directory parameter is required");
|
|
6920
|
+
}
|
|
6921
|
+
this.directory = path.resolve(directory);
|
|
6922
|
+
this.prefix = prefix;
|
|
6923
|
+
this.ttl = ttl;
|
|
6924
|
+
this.enableCompression = enableCompression;
|
|
6925
|
+
this.compressionThreshold = compressionThreshold;
|
|
6926
|
+
this.createDirectory = createDirectory;
|
|
6927
|
+
this.fileExtension = fileExtension;
|
|
6928
|
+
this.enableMetadata = enableMetadata;
|
|
6929
|
+
this.maxFileSize = maxFileSize;
|
|
6930
|
+
this.enableStats = enableStats;
|
|
6931
|
+
this.enableCleanup = enableCleanup;
|
|
6932
|
+
this.cleanupInterval = cleanupInterval;
|
|
6933
|
+
this.encoding = encoding;
|
|
6934
|
+
this.fileMode = fileMode;
|
|
6935
|
+
this.enableBackup = enableBackup;
|
|
6936
|
+
this.backupSuffix = backupSuffix;
|
|
6937
|
+
this.enableLocking = enableLocking;
|
|
6938
|
+
this.lockTimeout = lockTimeout;
|
|
6939
|
+
this.enableJournal = enableJournal;
|
|
6940
|
+
this.journalFile = path.join(this.directory, journalFile);
|
|
6941
|
+
this.stats = {
|
|
6942
|
+
hits: 0,
|
|
6943
|
+
misses: 0,
|
|
6944
|
+
sets: 0,
|
|
6945
|
+
deletes: 0,
|
|
6946
|
+
clears: 0,
|
|
6947
|
+
errors: 0
|
|
6818
6948
|
};
|
|
6819
|
-
|
|
6820
|
-
|
|
6821
|
-
|
|
6822
|
-
}
|
|
6823
|
-
async
|
|
6824
|
-
if (this.
|
|
6825
|
-
this.
|
|
6826
|
-
}
|
|
6827
|
-
|
|
6828
|
-
|
|
6829
|
-
|
|
6949
|
+
this.locks = /* @__PURE__ */ new Map();
|
|
6950
|
+
this.cleanupTimer = null;
|
|
6951
|
+
this._init();
|
|
6952
|
+
}
|
|
6953
|
+
async _init() {
|
|
6954
|
+
if (this.createDirectory) {
|
|
6955
|
+
await this._ensureDirectory(this.directory);
|
|
6956
|
+
}
|
|
6957
|
+
if (this.enableCleanup && this.cleanupInterval > 0) {
|
|
6958
|
+
this.cleanupTimer = setInterval(() => {
|
|
6959
|
+
this._cleanup().catch((err) => {
|
|
6960
|
+
console.warn("FilesystemCache cleanup error:", err.message);
|
|
6961
|
+
});
|
|
6962
|
+
}, this.cleanupInterval);
|
|
6830
6963
|
}
|
|
6831
|
-
this.installDatabaseProxy();
|
|
6832
|
-
this.installResourceHooks();
|
|
6833
6964
|
}
|
|
6834
|
-
async
|
|
6965
|
+
async _ensureDirectory(dir) {
|
|
6966
|
+
const [ok, err] = await try_fn_default(async () => {
|
|
6967
|
+
await mkdir(dir, { recursive: true });
|
|
6968
|
+
});
|
|
6969
|
+
if (!ok && err.code !== "EEXIST") {
|
|
6970
|
+
throw new Error(`Failed to create cache directory: ${err.message}`);
|
|
6971
|
+
}
|
|
6835
6972
|
}
|
|
6836
|
-
|
|
6973
|
+
_getFilePath(key) {
|
|
6974
|
+
const sanitizedKey = key.replace(/[<>:"/\\|?*]/g, "_");
|
|
6975
|
+
const filename = `${this.prefix}_${sanitizedKey}${this.fileExtension}`;
|
|
6976
|
+
return path.join(this.directory, filename);
|
|
6837
6977
|
}
|
|
6838
|
-
|
|
6839
|
-
|
|
6840
|
-
return;
|
|
6841
|
-
}
|
|
6842
|
-
const installResourceHooks = this.installResourceHooks.bind(this);
|
|
6843
|
-
this.database._originalCreateResourceForCache = this.database.createResource;
|
|
6844
|
-
this.database.createResource = async function(...args) {
|
|
6845
|
-
const resource = await this._originalCreateResourceForCache(...args);
|
|
6846
|
-
installResourceHooks(resource);
|
|
6847
|
-
return resource;
|
|
6848
|
-
};
|
|
6849
|
-
this.database._cacheProxyInstalled = true;
|
|
6978
|
+
_getMetadataPath(filePath) {
|
|
6979
|
+
return filePath + ".meta";
|
|
6850
6980
|
}
|
|
6851
|
-
|
|
6852
|
-
|
|
6853
|
-
|
|
6981
|
+
async _set(key, data) {
|
|
6982
|
+
const filePath = this._getFilePath(key);
|
|
6983
|
+
try {
|
|
6984
|
+
let serialized = JSON.stringify(data);
|
|
6985
|
+
const originalSize = Buffer.byteLength(serialized, this.encoding);
|
|
6986
|
+
if (originalSize > this.maxFileSize) {
|
|
6987
|
+
throw new Error(`Cache data exceeds maximum file size: ${originalSize} > ${this.maxFileSize}`);
|
|
6988
|
+
}
|
|
6989
|
+
let compressed = false;
|
|
6990
|
+
let finalData = serialized;
|
|
6991
|
+
if (this.enableCompression && originalSize >= this.compressionThreshold) {
|
|
6992
|
+
const compressedBuffer = zlib.gzipSync(Buffer.from(serialized, this.encoding));
|
|
6993
|
+
finalData = compressedBuffer.toString("base64");
|
|
6994
|
+
compressed = true;
|
|
6995
|
+
}
|
|
6996
|
+
if (this.enableBackup && await this._fileExists(filePath)) {
|
|
6997
|
+
const backupPath = filePath + this.backupSuffix;
|
|
6998
|
+
await this._copyFile(filePath, backupPath);
|
|
6999
|
+
}
|
|
7000
|
+
if (this.enableLocking) {
|
|
7001
|
+
await this._acquireLock(filePath);
|
|
7002
|
+
}
|
|
7003
|
+
try {
|
|
7004
|
+
await writeFile$1(filePath, finalData, {
|
|
7005
|
+
encoding: compressed ? "utf8" : this.encoding,
|
|
7006
|
+
mode: this.fileMode
|
|
7007
|
+
});
|
|
7008
|
+
if (this.enableMetadata) {
|
|
7009
|
+
const metadata = {
|
|
7010
|
+
key,
|
|
7011
|
+
timestamp: Date.now(),
|
|
7012
|
+
ttl: this.ttl,
|
|
7013
|
+
compressed,
|
|
7014
|
+
originalSize,
|
|
7015
|
+
compressedSize: compressed ? Buffer.byteLength(finalData, "utf8") : originalSize,
|
|
7016
|
+
compressionRatio: compressed ? (Buffer.byteLength(finalData, "utf8") / originalSize).toFixed(2) : 1
|
|
7017
|
+
};
|
|
7018
|
+
await writeFile$1(this._getMetadataPath(filePath), JSON.stringify(metadata), {
|
|
7019
|
+
encoding: this.encoding,
|
|
7020
|
+
mode: this.fileMode
|
|
7021
|
+
});
|
|
7022
|
+
}
|
|
7023
|
+
if (this.enableStats) {
|
|
7024
|
+
this.stats.sets++;
|
|
7025
|
+
}
|
|
7026
|
+
if (this.enableJournal) {
|
|
7027
|
+
await this._journalOperation("set", key, { size: originalSize, compressed });
|
|
7028
|
+
}
|
|
7029
|
+
} finally {
|
|
7030
|
+
if (this.enableLocking) {
|
|
7031
|
+
this._releaseLock(filePath);
|
|
7032
|
+
}
|
|
7033
|
+
}
|
|
7034
|
+
return data;
|
|
7035
|
+
} catch (error) {
|
|
7036
|
+
if (this.enableStats) {
|
|
7037
|
+
this.stats.errors++;
|
|
7038
|
+
}
|
|
7039
|
+
throw new Error(`Failed to set cache key '${key}': ${error.message}`);
|
|
6854
7040
|
}
|
|
6855
7041
|
}
|
|
6856
|
-
|
|
6857
|
-
|
|
6858
|
-
|
|
6859
|
-
|
|
6860
|
-
|
|
6861
|
-
|
|
6862
|
-
enumerable: false
|
|
6863
|
-
});
|
|
6864
|
-
resource.cacheKeyFor = async (options = {}) => {
|
|
6865
|
-
const { action, params = {}, partition, partitionValues } = options;
|
|
6866
|
-
return this.generateCacheKey(resource, action, params, partition, partitionValues);
|
|
6867
|
-
};
|
|
6868
|
-
const cacheMethods = [
|
|
6869
|
-
"count",
|
|
6870
|
-
"listIds",
|
|
6871
|
-
"getMany",
|
|
6872
|
-
"getAll",
|
|
6873
|
-
"page",
|
|
6874
|
-
"list",
|
|
6875
|
-
"get"
|
|
6876
|
-
];
|
|
6877
|
-
for (const method of cacheMethods) {
|
|
6878
|
-
resource.useMiddleware(method, async (ctx, next) => {
|
|
6879
|
-
let key;
|
|
6880
|
-
if (method === "getMany") {
|
|
6881
|
-
key = await resource.cacheKeyFor({ action: method, params: { ids: ctx.args[0] } });
|
|
6882
|
-
} else if (method === "page") {
|
|
6883
|
-
const { offset, size, partition, partitionValues } = ctx.args[0] || {};
|
|
6884
|
-
key = await resource.cacheKeyFor({ action: method, params: { offset, size }, partition, partitionValues });
|
|
6885
|
-
} else if (method === "list" || method === "listIds" || method === "count") {
|
|
6886
|
-
const { partition, partitionValues } = ctx.args[0] || {};
|
|
6887
|
-
key = await resource.cacheKeyFor({ action: method, partition, partitionValues });
|
|
6888
|
-
} else if (method === "getAll") {
|
|
6889
|
-
key = await resource.cacheKeyFor({ action: method });
|
|
6890
|
-
} else if (method === "get") {
|
|
6891
|
-
key = await resource.cacheKeyFor({ action: method, params: { id: ctx.args[0] } });
|
|
7042
|
+
async _get(key) {
|
|
7043
|
+
const filePath = this._getFilePath(key);
|
|
7044
|
+
try {
|
|
7045
|
+
if (!await this._fileExists(filePath)) {
|
|
7046
|
+
if (this.enableStats) {
|
|
7047
|
+
this.stats.misses++;
|
|
6892
7048
|
}
|
|
6893
|
-
|
|
6894
|
-
|
|
6895
|
-
|
|
6896
|
-
|
|
6897
|
-
|
|
6898
|
-
|
|
6899
|
-
|
|
6900
|
-
|
|
6901
|
-
|
|
6902
|
-
|
|
6903
|
-
|
|
6904
|
-
|
|
6905
|
-
|
|
6906
|
-
await this.clearCacheForResource(resource, ctx.args[0]);
|
|
6907
|
-
} else if (method === "update") {
|
|
6908
|
-
await this.clearCacheForResource(resource, { id: ctx.args[0], ...ctx.args[1] });
|
|
6909
|
-
} else if (method === "delete") {
|
|
6910
|
-
let data = { id: ctx.args[0] };
|
|
6911
|
-
if (typeof resource.get === "function") {
|
|
6912
|
-
const [ok, err, full] = await try_fn_default(() => resource.get(ctx.args[0]));
|
|
6913
|
-
if (ok && full) data = full;
|
|
7049
|
+
return null;
|
|
7050
|
+
}
|
|
7051
|
+
let isExpired = false;
|
|
7052
|
+
if (this.enableMetadata) {
|
|
7053
|
+
const metadataPath = this._getMetadataPath(filePath);
|
|
7054
|
+
if (await this._fileExists(metadataPath)) {
|
|
7055
|
+
const [ok, err, metadata] = await try_fn_default(async () => {
|
|
7056
|
+
const metaContent = await readFile$1(metadataPath, this.encoding);
|
|
7057
|
+
return JSON.parse(metaContent);
|
|
7058
|
+
});
|
|
7059
|
+
if (ok && metadata.ttl > 0) {
|
|
7060
|
+
const age = Date.now() - metadata.timestamp;
|
|
7061
|
+
isExpired = age > metadata.ttl;
|
|
6914
7062
|
}
|
|
6915
|
-
await this.clearCacheForResource(resource, data);
|
|
6916
|
-
} else if (method === "deleteMany") {
|
|
6917
|
-
await this.clearCacheForResource(resource);
|
|
6918
7063
|
}
|
|
6919
|
-
|
|
6920
|
-
|
|
6921
|
-
|
|
6922
|
-
|
|
6923
|
-
|
|
6924
|
-
|
|
6925
|
-
|
|
6926
|
-
|
|
6927
|
-
|
|
6928
|
-
if (!data) {
|
|
6929
|
-
for (const partitionName of Object.keys(resource.config.partitions)) {
|
|
6930
|
-
const partitionKeyPrefix = join(keyPrefix, `partition=${partitionName}`);
|
|
6931
|
-
await resource.cache.clear(partitionKeyPrefix);
|
|
7064
|
+
} else if (this.ttl > 0) {
|
|
7065
|
+
const stats = await stat$1(filePath);
|
|
7066
|
+
const age = Date.now() - stats.mtime.getTime();
|
|
7067
|
+
isExpired = age > this.ttl;
|
|
7068
|
+
}
|
|
7069
|
+
if (isExpired) {
|
|
7070
|
+
await this._del(key);
|
|
7071
|
+
if (this.enableStats) {
|
|
7072
|
+
this.stats.misses++;
|
|
6932
7073
|
}
|
|
6933
|
-
|
|
6934
|
-
|
|
6935
|
-
|
|
6936
|
-
|
|
6937
|
-
|
|
6938
|
-
|
|
7074
|
+
return null;
|
|
7075
|
+
}
|
|
7076
|
+
if (this.enableLocking) {
|
|
7077
|
+
await this._acquireLock(filePath);
|
|
7078
|
+
}
|
|
7079
|
+
try {
|
|
7080
|
+
const content = await readFile$1(filePath, this.encoding);
|
|
7081
|
+
let isCompressed = false;
|
|
7082
|
+
if (this.enableMetadata) {
|
|
7083
|
+
const metadataPath = this._getMetadataPath(filePath);
|
|
7084
|
+
if (await this._fileExists(metadataPath)) {
|
|
7085
|
+
const [ok, err, metadata] = await try_fn_default(async () => {
|
|
7086
|
+
const metaContent = await readFile$1(metadataPath, this.encoding);
|
|
7087
|
+
return JSON.parse(metaContent);
|
|
7088
|
+
});
|
|
7089
|
+
if (ok) {
|
|
7090
|
+
isCompressed = metadata.compressed;
|
|
7091
|
+
}
|
|
7092
|
+
}
|
|
7093
|
+
}
|
|
7094
|
+
let finalContent = content;
|
|
7095
|
+
if (isCompressed || this.enableCompression && content.match(/^[A-Za-z0-9+/=]+$/)) {
|
|
7096
|
+
try {
|
|
7097
|
+
const compressedBuffer = Buffer.from(content, "base64");
|
|
7098
|
+
finalContent = zlib.gunzipSync(compressedBuffer).toString(this.encoding);
|
|
7099
|
+
} catch (decompressError) {
|
|
7100
|
+
finalContent = content;
|
|
6939
7101
|
}
|
|
6940
7102
|
}
|
|
7103
|
+
const data = JSON.parse(finalContent);
|
|
7104
|
+
if (this.enableStats) {
|
|
7105
|
+
this.stats.hits++;
|
|
7106
|
+
}
|
|
7107
|
+
return data;
|
|
7108
|
+
} finally {
|
|
7109
|
+
if (this.enableLocking) {
|
|
7110
|
+
this._releaseLock(filePath);
|
|
7111
|
+
}
|
|
6941
7112
|
}
|
|
7113
|
+
} catch (error) {
|
|
7114
|
+
if (this.enableStats) {
|
|
7115
|
+
this.stats.errors++;
|
|
7116
|
+
}
|
|
7117
|
+
await this._del(key);
|
|
7118
|
+
return null;
|
|
6942
7119
|
}
|
|
6943
7120
|
}
|
|
6944
|
-
async
|
|
6945
|
-
const
|
|
6946
|
-
|
|
6947
|
-
|
|
6948
|
-
|
|
6949
|
-
|
|
6950
|
-
|
|
6951
|
-
|
|
6952
|
-
if (
|
|
6953
|
-
|
|
7121
|
+
async _del(key) {
|
|
7122
|
+
const filePath = this._getFilePath(key);
|
|
7123
|
+
try {
|
|
7124
|
+
if (await this._fileExists(filePath)) {
|
|
7125
|
+
await unlink(filePath);
|
|
7126
|
+
}
|
|
7127
|
+
if (this.enableMetadata) {
|
|
7128
|
+
const metadataPath = this._getMetadataPath(filePath);
|
|
7129
|
+
if (await this._fileExists(metadataPath)) {
|
|
7130
|
+
await unlink(metadataPath);
|
|
6954
7131
|
}
|
|
6955
7132
|
}
|
|
7133
|
+
if (this.enableBackup) {
|
|
7134
|
+
const backupPath = filePath + this.backupSuffix;
|
|
7135
|
+
if (await this._fileExists(backupPath)) {
|
|
7136
|
+
await unlink(backupPath);
|
|
7137
|
+
}
|
|
7138
|
+
}
|
|
7139
|
+
if (this.enableStats) {
|
|
7140
|
+
this.stats.deletes++;
|
|
7141
|
+
}
|
|
7142
|
+
if (this.enableJournal) {
|
|
7143
|
+
await this._journalOperation("delete", key);
|
|
7144
|
+
}
|
|
7145
|
+
return true;
|
|
7146
|
+
} catch (error) {
|
|
7147
|
+
if (this.enableStats) {
|
|
7148
|
+
this.stats.errors++;
|
|
7149
|
+
}
|
|
7150
|
+
throw new Error(`Failed to delete cache key '${key}': ${error.message}`);
|
|
6956
7151
|
}
|
|
6957
|
-
if (Object.keys(params).length > 0) {
|
|
6958
|
-
const paramsHash = await this.hashParams(params);
|
|
6959
|
-
keyParts.push(paramsHash);
|
|
6960
|
-
}
|
|
6961
|
-
return join(...keyParts) + ".json.gz";
|
|
6962
|
-
}
|
|
6963
|
-
async hashParams(params) {
|
|
6964
|
-
const sortedParams = Object.keys(params).sort().map((key) => `${key}:${params[key]}`).join("|") || "empty";
|
|
6965
|
-
return await sha256(sortedParams);
|
|
6966
|
-
}
|
|
6967
|
-
// Utility methods
|
|
6968
|
-
async getCacheStats() {
|
|
6969
|
-
if (!this.driver) return null;
|
|
6970
|
-
return {
|
|
6971
|
-
size: await this.driver.size(),
|
|
6972
|
-
keys: await this.driver.keys(),
|
|
6973
|
-
driver: this.driver.constructor.name
|
|
6974
|
-
};
|
|
6975
7152
|
}
|
|
6976
|
-
async
|
|
6977
|
-
|
|
6978
|
-
|
|
6979
|
-
|
|
6980
|
-
|
|
6981
|
-
|
|
7153
|
+
async _clear(prefix) {
|
|
7154
|
+
try {
|
|
7155
|
+
const files = await readdir$1(this.directory);
|
|
7156
|
+
const cacheFiles = files.filter((file) => {
|
|
7157
|
+
if (!file.startsWith(this.prefix)) return false;
|
|
7158
|
+
if (!file.endsWith(this.fileExtension)) return false;
|
|
7159
|
+
if (prefix) {
|
|
7160
|
+
const keyPart = file.slice(this.prefix.length + 1, -this.fileExtension.length);
|
|
7161
|
+
return keyPart.startsWith(prefix);
|
|
7162
|
+
}
|
|
7163
|
+
return true;
|
|
7164
|
+
});
|
|
7165
|
+
for (const file of cacheFiles) {
|
|
7166
|
+
const filePath = path.join(this.directory, file);
|
|
7167
|
+
if (await this._fileExists(filePath)) {
|
|
7168
|
+
await unlink(filePath);
|
|
7169
|
+
}
|
|
7170
|
+
if (this.enableMetadata) {
|
|
7171
|
+
const metadataPath = this._getMetadataPath(filePath);
|
|
7172
|
+
if (await this._fileExists(metadataPath)) {
|
|
7173
|
+
await unlink(metadataPath);
|
|
7174
|
+
}
|
|
7175
|
+
}
|
|
7176
|
+
if (this.enableBackup) {
|
|
7177
|
+
const backupPath = filePath + this.backupSuffix;
|
|
7178
|
+
if (await this._fileExists(backupPath)) {
|
|
7179
|
+
await unlink(backupPath);
|
|
7180
|
+
}
|
|
7181
|
+
}
|
|
7182
|
+
}
|
|
7183
|
+
if (this.enableStats) {
|
|
7184
|
+
this.stats.clears++;
|
|
7185
|
+
}
|
|
7186
|
+
if (this.enableJournal) {
|
|
7187
|
+
await this._journalOperation("clear", prefix || "all", { count: cacheFiles.length });
|
|
7188
|
+
}
|
|
7189
|
+
return true;
|
|
7190
|
+
} catch (error) {
|
|
7191
|
+
if (this.enableStats) {
|
|
7192
|
+
this.stats.errors++;
|
|
6982
7193
|
}
|
|
7194
|
+
throw new Error(`Failed to clear cache: ${error.message}`);
|
|
6983
7195
|
}
|
|
6984
7196
|
}
|
|
6985
|
-
async
|
|
6986
|
-
const
|
|
6987
|
-
|
|
6988
|
-
|
|
6989
|
-
|
|
6990
|
-
|
|
7197
|
+
async size() {
|
|
7198
|
+
const keys = await this.keys();
|
|
7199
|
+
return keys.length;
|
|
7200
|
+
}
|
|
7201
|
+
async keys() {
|
|
7202
|
+
try {
|
|
7203
|
+
const files = await readdir$1(this.directory);
|
|
7204
|
+
const cacheFiles = files.filter(
|
|
7205
|
+
(file) => file.startsWith(this.prefix) && file.endsWith(this.fileExtension)
|
|
7206
|
+
);
|
|
7207
|
+
const keys = cacheFiles.map((file) => {
|
|
7208
|
+
const keyPart = file.slice(this.prefix.length + 1, -this.fileExtension.length);
|
|
7209
|
+
return keyPart;
|
|
7210
|
+
});
|
|
7211
|
+
return keys;
|
|
7212
|
+
} catch (error) {
|
|
7213
|
+
console.warn("FilesystemCache: Failed to list keys:", error.message);
|
|
7214
|
+
return [];
|
|
7215
|
+
}
|
|
7216
|
+
}
|
|
7217
|
+
// Helper methods
|
|
7218
|
+
async _fileExists(filePath) {
|
|
7219
|
+
const [ok] = await try_fn_default(async () => {
|
|
7220
|
+
await stat$1(filePath);
|
|
7221
|
+
});
|
|
7222
|
+
return ok;
|
|
7223
|
+
}
|
|
7224
|
+
async _copyFile(src, dest) {
|
|
7225
|
+
const [ok, err] = await try_fn_default(async () => {
|
|
7226
|
+
const content = await readFile$1(src);
|
|
7227
|
+
await writeFile$1(dest, content);
|
|
7228
|
+
});
|
|
7229
|
+
if (!ok) {
|
|
7230
|
+
console.warn("FilesystemCache: Failed to create backup:", err.message);
|
|
7231
|
+
}
|
|
7232
|
+
}
|
|
7233
|
+
async _cleanup() {
|
|
7234
|
+
if (!this.ttl || this.ttl <= 0) return;
|
|
7235
|
+
try {
|
|
7236
|
+
const files = await readdir$1(this.directory);
|
|
7237
|
+
const now = Date.now();
|
|
7238
|
+
for (const file of files) {
|
|
7239
|
+
if (!file.startsWith(this.prefix) || !file.endsWith(this.fileExtension)) {
|
|
7240
|
+
continue;
|
|
7241
|
+
}
|
|
7242
|
+
const filePath = path.join(this.directory, file);
|
|
7243
|
+
let shouldDelete = false;
|
|
7244
|
+
if (this.enableMetadata) {
|
|
7245
|
+
const metadataPath = this._getMetadataPath(filePath);
|
|
7246
|
+
if (await this._fileExists(metadataPath)) {
|
|
7247
|
+
const [ok, err, metadata] = await try_fn_default(async () => {
|
|
7248
|
+
const metaContent = await readFile$1(metadataPath, this.encoding);
|
|
7249
|
+
return JSON.parse(metaContent);
|
|
7250
|
+
});
|
|
7251
|
+
if (ok && metadata.ttl > 0) {
|
|
7252
|
+
const age = now - metadata.timestamp;
|
|
7253
|
+
shouldDelete = age > metadata.ttl;
|
|
7254
|
+
}
|
|
7255
|
+
}
|
|
7256
|
+
} else {
|
|
7257
|
+
const [ok, err, stats] = await try_fn_default(async () => {
|
|
7258
|
+
return await stat$1(filePath);
|
|
7259
|
+
});
|
|
7260
|
+
if (ok) {
|
|
7261
|
+
const age = now - stats.mtime.getTime();
|
|
7262
|
+
shouldDelete = age > this.ttl;
|
|
7263
|
+
}
|
|
7264
|
+
}
|
|
7265
|
+
if (shouldDelete) {
|
|
7266
|
+
const keyPart = file.slice(this.prefix.length + 1, -this.fileExtension.length);
|
|
7267
|
+
await this._del(keyPart);
|
|
7268
|
+
}
|
|
7269
|
+
}
|
|
7270
|
+
} catch (error) {
|
|
7271
|
+
console.warn("FilesystemCache cleanup error:", error.message);
|
|
7272
|
+
}
|
|
7273
|
+
}
|
|
7274
|
+
async _acquireLock(filePath) {
|
|
7275
|
+
if (!this.enableLocking) return;
|
|
7276
|
+
const lockKey = filePath;
|
|
7277
|
+
const startTime = Date.now();
|
|
7278
|
+
while (this.locks.has(lockKey)) {
|
|
7279
|
+
if (Date.now() - startTime > this.lockTimeout) {
|
|
7280
|
+
throw new Error(`Lock timeout for file: ${filePath}`);
|
|
7281
|
+
}
|
|
7282
|
+
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
7283
|
+
}
|
|
7284
|
+
this.locks.set(lockKey, Date.now());
|
|
7285
|
+
}
|
|
7286
|
+
_releaseLock(filePath) {
|
|
7287
|
+
if (!this.enableLocking) return;
|
|
7288
|
+
this.locks.delete(filePath);
|
|
7289
|
+
}
|
|
7290
|
+
async _journalOperation(operation, key, metadata = {}) {
|
|
7291
|
+
if (!this.enableJournal) return;
|
|
7292
|
+
const entry = {
|
|
7293
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
7294
|
+
operation,
|
|
7295
|
+
key,
|
|
7296
|
+
metadata
|
|
7297
|
+
};
|
|
7298
|
+
const [ok, err] = await try_fn_default(async () => {
|
|
7299
|
+
const line = JSON.stringify(entry) + "\n";
|
|
7300
|
+
await fs.promises.appendFile(this.journalFile, line, this.encoding);
|
|
7301
|
+
});
|
|
7302
|
+
if (!ok) {
|
|
7303
|
+
console.warn("FilesystemCache journal error:", err.message);
|
|
7304
|
+
}
|
|
7305
|
+
}
|
|
7306
|
+
// Cleanup on process exit
|
|
7307
|
+
destroy() {
|
|
7308
|
+
if (this.cleanupTimer) {
|
|
7309
|
+
clearInterval(this.cleanupTimer);
|
|
7310
|
+
this.cleanupTimer = null;
|
|
7311
|
+
}
|
|
7312
|
+
}
|
|
7313
|
+
// Get cache statistics
|
|
7314
|
+
getStats() {
|
|
7315
|
+
return {
|
|
7316
|
+
...this.stats,
|
|
7317
|
+
directory: this.directory,
|
|
7318
|
+
ttl: this.ttl,
|
|
7319
|
+
compression: this.enableCompression,
|
|
7320
|
+
metadata: this.enableMetadata,
|
|
7321
|
+
cleanup: this.enableCleanup,
|
|
7322
|
+
locking: this.enableLocking,
|
|
7323
|
+
journal: this.enableJournal
|
|
7324
|
+
};
|
|
7325
|
+
}
|
|
7326
|
+
}
|
|
7327
|
+
|
|
7328
|
+
promisify(fs.mkdir);
|
|
7329
|
+
const rmdir = promisify(fs.rm);
|
|
7330
|
+
const readdir = promisify(fs.readdir);
|
|
7331
|
+
const stat = promisify(fs.stat);
|
|
7332
|
+
const writeFile = promisify(fs.writeFile);
|
|
7333
|
+
const readFile = promisify(fs.readFile);
|
|
7334
|
+
class PartitionAwareFilesystemCache extends FilesystemCache {
|
|
7335
|
+
constructor({
|
|
7336
|
+
partitionStrategy = "hierarchical",
|
|
7337
|
+
// 'hierarchical', 'flat', 'temporal'
|
|
7338
|
+
trackUsage = true,
|
|
7339
|
+
preloadRelated = false,
|
|
7340
|
+
preloadThreshold = 10,
|
|
7341
|
+
maxCacheSize = null,
|
|
7342
|
+
usageStatsFile = "partition-usage.json",
|
|
7343
|
+
...config
|
|
7344
|
+
}) {
|
|
7345
|
+
super(config);
|
|
7346
|
+
this.partitionStrategy = partitionStrategy;
|
|
7347
|
+
this.trackUsage = trackUsage;
|
|
7348
|
+
this.preloadRelated = preloadRelated;
|
|
7349
|
+
this.preloadThreshold = preloadThreshold;
|
|
7350
|
+
this.maxCacheSize = maxCacheSize;
|
|
7351
|
+
this.usageStatsFile = path.join(this.directory, usageStatsFile);
|
|
7352
|
+
this.partitionUsage = /* @__PURE__ */ new Map();
|
|
7353
|
+
this.loadUsageStats();
|
|
7354
|
+
}
|
|
7355
|
+
/**
|
|
7356
|
+
* Generate partition-aware cache key
|
|
7357
|
+
*/
|
|
7358
|
+
_getPartitionCacheKey(resource, action, partition, partitionValues = {}, params = {}) {
|
|
7359
|
+
const keyParts = [`resource=${resource}`, `action=${action}`];
|
|
7360
|
+
if (partition && Object.keys(partitionValues).length > 0) {
|
|
7361
|
+
keyParts.push(`partition=${partition}`);
|
|
7362
|
+
const sortedFields = Object.entries(partitionValues).sort(([a], [b]) => a.localeCompare(b));
|
|
7363
|
+
for (const [field, value] of sortedFields) {
|
|
7364
|
+
if (value !== null && value !== void 0) {
|
|
7365
|
+
keyParts.push(`${field}=${value}`);
|
|
7366
|
+
}
|
|
7367
|
+
}
|
|
7368
|
+
}
|
|
7369
|
+
if (Object.keys(params).length > 0) {
|
|
7370
|
+
const paramsStr = Object.entries(params).sort(([a], [b]) => a.localeCompare(b)).map(([k, v]) => `${k}=${v}`).join("|");
|
|
7371
|
+
keyParts.push(`params=${Buffer.from(paramsStr).toString("base64")}`);
|
|
7372
|
+
}
|
|
7373
|
+
return keyParts.join("/") + this.fileExtension;
|
|
7374
|
+
}
|
|
7375
|
+
/**
|
|
7376
|
+
* Get directory path for partition cache
|
|
7377
|
+
*/
|
|
7378
|
+
_getPartitionDirectory(resource, partition, partitionValues = {}) {
|
|
7379
|
+
const basePath = path.join(this.directory, `resource=${resource}`);
|
|
7380
|
+
if (!partition) {
|
|
7381
|
+
return basePath;
|
|
7382
|
+
}
|
|
7383
|
+
if (this.partitionStrategy === "flat") {
|
|
7384
|
+
return path.join(basePath, "partitions");
|
|
7385
|
+
}
|
|
7386
|
+
if (this.partitionStrategy === "temporal" && this._isTemporalPartition(partition, partitionValues)) {
|
|
7387
|
+
return this._getTemporalDirectory(basePath, partition, partitionValues);
|
|
7388
|
+
}
|
|
7389
|
+
const pathParts = [basePath, `partition=${partition}`];
|
|
7390
|
+
const sortedFields = Object.entries(partitionValues).sort(([a], [b]) => a.localeCompare(b));
|
|
7391
|
+
for (const [field, value] of sortedFields) {
|
|
7392
|
+
if (value !== null && value !== void 0) {
|
|
7393
|
+
pathParts.push(`${field}=${this._sanitizePathValue(value)}`);
|
|
7394
|
+
}
|
|
7395
|
+
}
|
|
7396
|
+
return path.join(...pathParts);
|
|
7397
|
+
}
|
|
7398
|
+
/**
|
|
7399
|
+
* Enhanced set method with partition awareness
|
|
7400
|
+
*/
|
|
7401
|
+
async _set(key, data, options = {}) {
|
|
7402
|
+
const { resource, action, partition, partitionValues, params } = options;
|
|
7403
|
+
if (resource && partition) {
|
|
7404
|
+
const partitionKey = this._getPartitionCacheKey(resource, action, partition, partitionValues, params);
|
|
7405
|
+
const partitionDir = this._getPartitionDirectory(resource, partition, partitionValues);
|
|
7406
|
+
await this._ensureDirectory(partitionDir);
|
|
7407
|
+
const filePath = path.join(partitionDir, this._sanitizeFileName(partitionKey));
|
|
7408
|
+
if (this.trackUsage) {
|
|
7409
|
+
await this._trackPartitionUsage(resource, partition, partitionValues);
|
|
7410
|
+
}
|
|
7411
|
+
const partitionData = {
|
|
7412
|
+
data,
|
|
7413
|
+
metadata: {
|
|
7414
|
+
resource,
|
|
7415
|
+
partition,
|
|
7416
|
+
partitionValues,
|
|
7417
|
+
timestamp: Date.now(),
|
|
7418
|
+
ttl: this.ttl
|
|
7419
|
+
}
|
|
7420
|
+
};
|
|
7421
|
+
return this._writeFileWithMetadata(filePath, partitionData);
|
|
7422
|
+
}
|
|
7423
|
+
return super._set(key, data);
|
|
7424
|
+
}
|
|
7425
|
+
/**
|
|
7426
|
+
* Enhanced get method with partition awareness
|
|
7427
|
+
*/
|
|
7428
|
+
async _get(key, options = {}) {
|
|
7429
|
+
const { resource, action, partition, partitionValues, params } = options;
|
|
7430
|
+
if (resource && partition) {
|
|
7431
|
+
const partitionKey = this._getPartitionCacheKey(resource, action, partition, partitionValues, params);
|
|
7432
|
+
const partitionDir = this._getPartitionDirectory(resource, partition, partitionValues);
|
|
7433
|
+
const filePath = path.join(partitionDir, this._sanitizeFileName(partitionKey));
|
|
7434
|
+
if (!await this._fileExists(filePath)) {
|
|
7435
|
+
if (this.preloadRelated) {
|
|
7436
|
+
await this._preloadRelatedPartitions(resource, partition, partitionValues);
|
|
7437
|
+
}
|
|
7438
|
+
return null;
|
|
7439
|
+
}
|
|
7440
|
+
const result = await this._readFileWithMetadata(filePath);
|
|
7441
|
+
if (result && this.trackUsage) {
|
|
7442
|
+
await this._trackPartitionUsage(resource, partition, partitionValues);
|
|
7443
|
+
}
|
|
7444
|
+
return result?.data || null;
|
|
7445
|
+
}
|
|
7446
|
+
return super._get(key);
|
|
7447
|
+
}
|
|
7448
|
+
/**
|
|
7449
|
+
* Clear cache for specific partition
|
|
7450
|
+
*/
|
|
7451
|
+
async clearPartition(resource, partition, partitionValues = {}) {
|
|
7452
|
+
const partitionDir = this._getPartitionDirectory(resource, partition, partitionValues);
|
|
7453
|
+
const [ok, err] = await try_fn_default(async () => {
|
|
7454
|
+
if (await this._fileExists(partitionDir)) {
|
|
7455
|
+
await rmdir(partitionDir, { recursive: true });
|
|
7456
|
+
}
|
|
7457
|
+
});
|
|
7458
|
+
if (!ok) {
|
|
7459
|
+
console.warn(`Failed to clear partition cache: ${err.message}`);
|
|
7460
|
+
}
|
|
7461
|
+
const usageKey = this._getUsageKey(resource, partition, partitionValues);
|
|
7462
|
+
this.partitionUsage.delete(usageKey);
|
|
7463
|
+
await this._saveUsageStats();
|
|
7464
|
+
return ok;
|
|
7465
|
+
}
|
|
7466
|
+
/**
|
|
7467
|
+
* Clear all partitions for a resource
|
|
7468
|
+
*/
|
|
7469
|
+
async clearResourcePartitions(resource) {
|
|
7470
|
+
const resourceDir = path.join(this.directory, `resource=${resource}`);
|
|
7471
|
+
const [ok, err] = await try_fn_default(async () => {
|
|
7472
|
+
if (await this._fileExists(resourceDir)) {
|
|
7473
|
+
await rmdir(resourceDir, { recursive: true });
|
|
7474
|
+
}
|
|
7475
|
+
});
|
|
7476
|
+
for (const [key] of this.partitionUsage.entries()) {
|
|
7477
|
+
if (key.startsWith(`${resource}/`)) {
|
|
7478
|
+
this.partitionUsage.delete(key);
|
|
7479
|
+
}
|
|
7480
|
+
}
|
|
7481
|
+
await this._saveUsageStats();
|
|
7482
|
+
return ok;
|
|
7483
|
+
}
|
|
7484
|
+
/**
|
|
7485
|
+
* Get partition cache statistics
|
|
7486
|
+
*/
|
|
7487
|
+
async getPartitionStats(resource, partition = null) {
|
|
7488
|
+
const stats = {
|
|
7489
|
+
totalFiles: 0,
|
|
7490
|
+
totalSize: 0,
|
|
7491
|
+
partitions: {},
|
|
7492
|
+
usage: {}
|
|
7493
|
+
};
|
|
7494
|
+
const resourceDir = path.join(this.directory, `resource=${resource}`);
|
|
7495
|
+
if (!await this._fileExists(resourceDir)) {
|
|
7496
|
+
return stats;
|
|
7497
|
+
}
|
|
7498
|
+
await this._calculateDirectoryStats(resourceDir, stats);
|
|
7499
|
+
for (const [key, usage] of this.partitionUsage.entries()) {
|
|
7500
|
+
if (key.startsWith(`${resource}/`)) {
|
|
7501
|
+
const partitionName = key.split("/")[1];
|
|
7502
|
+
if (!partition || partitionName === partition) {
|
|
7503
|
+
stats.usage[partitionName] = usage;
|
|
7504
|
+
}
|
|
7505
|
+
}
|
|
7506
|
+
}
|
|
7507
|
+
return stats;
|
|
7508
|
+
}
|
|
7509
|
+
/**
|
|
7510
|
+
* Get cache recommendations based on usage patterns
|
|
7511
|
+
*/
|
|
7512
|
+
async getCacheRecommendations(resource) {
|
|
7513
|
+
const recommendations = [];
|
|
7514
|
+
const now = Date.now();
|
|
7515
|
+
const dayMs = 24 * 60 * 60 * 1e3;
|
|
7516
|
+
for (const [key, usage] of this.partitionUsage.entries()) {
|
|
7517
|
+
if (key.startsWith(`${resource}/`)) {
|
|
7518
|
+
const [, partition] = key.split("/");
|
|
7519
|
+
const daysSinceLastAccess = (now - usage.lastAccess) / dayMs;
|
|
7520
|
+
const accessesPerDay = usage.count / Math.max(1, daysSinceLastAccess);
|
|
7521
|
+
let recommendation = "keep";
|
|
7522
|
+
let priority = usage.count;
|
|
7523
|
+
if (daysSinceLastAccess > 30) {
|
|
7524
|
+
recommendation = "archive";
|
|
7525
|
+
priority = 0;
|
|
7526
|
+
} else if (accessesPerDay < 0.1) {
|
|
7527
|
+
recommendation = "reduce_ttl";
|
|
7528
|
+
priority = 1;
|
|
7529
|
+
} else if (accessesPerDay > 10) {
|
|
7530
|
+
recommendation = "preload";
|
|
7531
|
+
priority = 100;
|
|
7532
|
+
}
|
|
7533
|
+
recommendations.push({
|
|
7534
|
+
partition,
|
|
7535
|
+
recommendation,
|
|
7536
|
+
priority,
|
|
7537
|
+
usage: accessesPerDay,
|
|
7538
|
+
lastAccess: new Date(usage.lastAccess).toISOString()
|
|
7539
|
+
});
|
|
7540
|
+
}
|
|
7541
|
+
}
|
|
7542
|
+
return recommendations.sort((a, b) => b.priority - a.priority);
|
|
7543
|
+
}
|
|
7544
|
+
/**
|
|
7545
|
+
* Preload frequently accessed partitions
|
|
7546
|
+
*/
|
|
7547
|
+
async warmPartitionCache(resource, options = {}) {
|
|
7548
|
+
const { partitions = [], maxFiles = 1e3 } = options;
|
|
7549
|
+
let warmedCount = 0;
|
|
7550
|
+
for (const partition of partitions) {
|
|
7551
|
+
const usageKey = `${resource}/${partition}`;
|
|
7552
|
+
const usage = this.partitionUsage.get(usageKey);
|
|
7553
|
+
if (usage && usage.count >= this.preloadThreshold) {
|
|
7554
|
+
console.log(`\u{1F525} Warming cache for ${resource}/${partition} (${usage.count} accesses)`);
|
|
7555
|
+
warmedCount++;
|
|
7556
|
+
}
|
|
7557
|
+
if (warmedCount >= maxFiles) break;
|
|
7558
|
+
}
|
|
7559
|
+
return warmedCount;
|
|
7560
|
+
}
|
|
7561
|
+
// Private helper methods
|
|
7562
|
+
async _trackPartitionUsage(resource, partition, partitionValues) {
|
|
7563
|
+
const usageKey = this._getUsageKey(resource, partition, partitionValues);
|
|
7564
|
+
const current = this.partitionUsage.get(usageKey) || {
|
|
7565
|
+
count: 0,
|
|
7566
|
+
firstAccess: Date.now(),
|
|
7567
|
+
lastAccess: Date.now()
|
|
7568
|
+
};
|
|
7569
|
+
current.count++;
|
|
7570
|
+
current.lastAccess = Date.now();
|
|
7571
|
+
this.partitionUsage.set(usageKey, current);
|
|
7572
|
+
if (current.count % 10 === 0) {
|
|
7573
|
+
await this._saveUsageStats();
|
|
7574
|
+
}
|
|
7575
|
+
}
|
|
7576
|
+
_getUsageKey(resource, partition, partitionValues) {
|
|
7577
|
+
const valuePart = Object.entries(partitionValues).sort(([a], [b]) => a.localeCompare(b)).map(([k, v]) => `${k}=${v}`).join("|");
|
|
7578
|
+
return `${resource}/${partition}/${valuePart}`;
|
|
7579
|
+
}
|
|
7580
|
+
async _preloadRelatedPartitions(resource, partition, partitionValues) {
|
|
7581
|
+
console.log(`\u{1F3AF} Preloading related partitions for ${resource}/${partition}`);
|
|
7582
|
+
if (partitionValues.timestamp || partitionValues.date) ;
|
|
7583
|
+
}
|
|
7584
|
+
_isTemporalPartition(partition, partitionValues) {
|
|
7585
|
+
const temporalFields = ["date", "timestamp", "createdAt", "updatedAt"];
|
|
7586
|
+
return Object.keys(partitionValues).some(
|
|
7587
|
+
(field) => temporalFields.some((tf) => field.toLowerCase().includes(tf))
|
|
7588
|
+
);
|
|
7589
|
+
}
|
|
7590
|
+
_getTemporalDirectory(basePath, partition, partitionValues) {
|
|
7591
|
+
const dateValue = Object.values(partitionValues)[0];
|
|
7592
|
+
if (typeof dateValue === "string" && dateValue.match(/^\d{4}-\d{2}-\d{2}/)) {
|
|
7593
|
+
const [year, month, day] = dateValue.split("-");
|
|
7594
|
+
return path.join(basePath, "temporal", year, month, day);
|
|
7595
|
+
}
|
|
7596
|
+
return path.join(basePath, `partition=${partition}`);
|
|
7597
|
+
}
|
|
7598
|
+
_sanitizePathValue(value) {
|
|
7599
|
+
return String(value).replace(/[<>:"/\\|?*]/g, "_");
|
|
7600
|
+
}
|
|
7601
|
+
_sanitizeFileName(filename) {
|
|
7602
|
+
return filename.replace(/[<>:"/\\|?*]/g, "_");
|
|
7603
|
+
}
|
|
7604
|
+
async _calculateDirectoryStats(dir, stats) {
|
|
7605
|
+
const [ok, err, files] = await try_fn_default(() => readdir(dir));
|
|
7606
|
+
if (!ok) return;
|
|
7607
|
+
for (const file of files) {
|
|
7608
|
+
const filePath = path.join(dir, file);
|
|
7609
|
+
const [statOk, statErr, fileStat] = await try_fn_default(() => stat(filePath));
|
|
7610
|
+
if (statOk) {
|
|
7611
|
+
if (fileStat.isDirectory()) {
|
|
7612
|
+
await this._calculateDirectoryStats(filePath, stats);
|
|
7613
|
+
} else {
|
|
7614
|
+
stats.totalFiles++;
|
|
7615
|
+
stats.totalSize += fileStat.size;
|
|
7616
|
+
}
|
|
7617
|
+
}
|
|
7618
|
+
}
|
|
7619
|
+
}
|
|
7620
|
+
async loadUsageStats() {
|
|
7621
|
+
const [ok, err, content] = await try_fn_default(async () => {
|
|
7622
|
+
const data = await readFile(this.usageStatsFile, "utf8");
|
|
7623
|
+
return JSON.parse(data);
|
|
7624
|
+
});
|
|
7625
|
+
if (ok && content) {
|
|
7626
|
+
this.partitionUsage = new Map(Object.entries(content));
|
|
7627
|
+
}
|
|
7628
|
+
}
|
|
7629
|
+
async _saveUsageStats() {
|
|
7630
|
+
const statsObject = Object.fromEntries(this.partitionUsage);
|
|
7631
|
+
await try_fn_default(async () => {
|
|
7632
|
+
await writeFile(
|
|
7633
|
+
this.usageStatsFile,
|
|
7634
|
+
JSON.stringify(statsObject, null, 2),
|
|
7635
|
+
"utf8"
|
|
7636
|
+
);
|
|
7637
|
+
});
|
|
7638
|
+
}
|
|
7639
|
+
async _writeFileWithMetadata(filePath, data) {
|
|
7640
|
+
const content = JSON.stringify(data);
|
|
7641
|
+
const [ok, err] = await try_fn_default(async () => {
|
|
7642
|
+
await writeFile(filePath, content, {
|
|
7643
|
+
encoding: this.encoding,
|
|
7644
|
+
mode: this.fileMode
|
|
7645
|
+
});
|
|
7646
|
+
});
|
|
7647
|
+
if (!ok) {
|
|
7648
|
+
throw new Error(`Failed to write cache file: ${err.message}`);
|
|
7649
|
+
}
|
|
7650
|
+
return true;
|
|
7651
|
+
}
|
|
7652
|
+
async _readFileWithMetadata(filePath) {
|
|
7653
|
+
const [ok, err, content] = await try_fn_default(async () => {
|
|
7654
|
+
return await readFile(filePath, this.encoding);
|
|
7655
|
+
});
|
|
7656
|
+
if (!ok || !content) return null;
|
|
7657
|
+
try {
|
|
7658
|
+
return JSON.parse(content);
|
|
7659
|
+
} catch (error) {
|
|
7660
|
+
return { data: content };
|
|
7661
|
+
}
|
|
7662
|
+
}
|
|
7663
|
+
}
|
|
7664
|
+
|
|
7665
|
+
class CachePlugin extends plugin_class_default {
|
|
7666
|
+
constructor(options = {}) {
|
|
7667
|
+
super(options);
|
|
7668
|
+
this.driver = options.driver;
|
|
7669
|
+
this.config = {
|
|
7670
|
+
includePartitions: options.includePartitions !== false,
|
|
7671
|
+
partitionStrategy: options.partitionStrategy || "hierarchical",
|
|
7672
|
+
partitionAware: options.partitionAware !== false,
|
|
7673
|
+
trackUsage: options.trackUsage !== false,
|
|
7674
|
+
preloadRelated: options.preloadRelated !== false,
|
|
7675
|
+
...options
|
|
7676
|
+
};
|
|
7677
|
+
}
|
|
7678
|
+
async setup(database) {
|
|
7679
|
+
await super.setup(database);
|
|
7680
|
+
}
|
|
7681
|
+
async onSetup() {
|
|
7682
|
+
if (this.config.driver) {
|
|
7683
|
+
this.driver = this.config.driver;
|
|
7684
|
+
} else if (this.config.driverType === "memory") {
|
|
7685
|
+
this.driver = new memory_cache_class_default(this.config.memoryOptions || {});
|
|
7686
|
+
} else if (this.config.driverType === "filesystem") {
|
|
7687
|
+
if (this.config.partitionAware) {
|
|
7688
|
+
this.driver = new PartitionAwareFilesystemCache({
|
|
7689
|
+
partitionStrategy: this.config.partitionStrategy,
|
|
7690
|
+
trackUsage: this.config.trackUsage,
|
|
7691
|
+
preloadRelated: this.config.preloadRelated,
|
|
7692
|
+
...this.config.filesystemOptions
|
|
7693
|
+
});
|
|
7694
|
+
} else {
|
|
7695
|
+
this.driver = new FilesystemCache(this.config.filesystemOptions || {});
|
|
7696
|
+
}
|
|
7697
|
+
} else {
|
|
7698
|
+
this.driver = new s3_cache_class_default({ client: this.database.client, ...this.config.s3Options || {} });
|
|
7699
|
+
}
|
|
7700
|
+
this.installDatabaseProxy();
|
|
7701
|
+
this.installResourceHooks();
|
|
7702
|
+
}
|
|
7703
|
+
async onStart() {
|
|
7704
|
+
}
|
|
7705
|
+
async onStop() {
|
|
7706
|
+
}
|
|
7707
|
+
installDatabaseProxy() {
|
|
7708
|
+
if (this.database._cacheProxyInstalled) {
|
|
7709
|
+
return;
|
|
7710
|
+
}
|
|
7711
|
+
const installResourceHooks = this.installResourceHooks.bind(this);
|
|
7712
|
+
this.database._originalCreateResourceForCache = this.database.createResource;
|
|
7713
|
+
this.database.createResource = async function(...args) {
|
|
7714
|
+
const resource = await this._originalCreateResourceForCache(...args);
|
|
7715
|
+
installResourceHooks(resource);
|
|
7716
|
+
return resource;
|
|
7717
|
+
};
|
|
7718
|
+
this.database._cacheProxyInstalled = true;
|
|
7719
|
+
}
|
|
7720
|
+
installResourceHooks() {
|
|
7721
|
+
for (const resource of Object.values(this.database.resources)) {
|
|
7722
|
+
this.installResourceHooksForResource(resource);
|
|
7723
|
+
}
|
|
7724
|
+
}
|
|
7725
|
+
installResourceHooksForResource(resource) {
|
|
7726
|
+
if (!this.driver) return;
|
|
7727
|
+
Object.defineProperty(resource, "cache", {
|
|
7728
|
+
value: this.driver,
|
|
7729
|
+
writable: true,
|
|
7730
|
+
configurable: true,
|
|
7731
|
+
enumerable: false
|
|
7732
|
+
});
|
|
7733
|
+
resource.cacheKeyFor = async (options = {}) => {
|
|
7734
|
+
const { action, params = {}, partition, partitionValues } = options;
|
|
7735
|
+
return this.generateCacheKey(resource, action, params, partition, partitionValues);
|
|
7736
|
+
};
|
|
7737
|
+
if (this.driver instanceof PartitionAwareFilesystemCache) {
|
|
7738
|
+
resource.clearPartitionCache = async (partition, partitionValues = {}) => {
|
|
7739
|
+
return await this.driver.clearPartition(resource.name, partition, partitionValues);
|
|
7740
|
+
};
|
|
7741
|
+
resource.getPartitionCacheStats = async (partition = null) => {
|
|
7742
|
+
return await this.driver.getPartitionStats(resource.name, partition);
|
|
7743
|
+
};
|
|
7744
|
+
resource.getCacheRecommendations = async () => {
|
|
7745
|
+
return await this.driver.getCacheRecommendations(resource.name);
|
|
7746
|
+
};
|
|
7747
|
+
resource.warmPartitionCache = async (partitions = [], options = {}) => {
|
|
7748
|
+
return await this.driver.warmPartitionCache(resource.name, { partitions, ...options });
|
|
7749
|
+
};
|
|
7750
|
+
}
|
|
7751
|
+
const cacheMethods = [
|
|
7752
|
+
"count",
|
|
7753
|
+
"listIds",
|
|
7754
|
+
"getMany",
|
|
7755
|
+
"getAll",
|
|
7756
|
+
"page",
|
|
7757
|
+
"list",
|
|
7758
|
+
"get"
|
|
7759
|
+
];
|
|
7760
|
+
for (const method of cacheMethods) {
|
|
7761
|
+
resource.useMiddleware(method, async (ctx, next) => {
|
|
7762
|
+
let key;
|
|
7763
|
+
if (method === "getMany") {
|
|
7764
|
+
key = await resource.cacheKeyFor({ action: method, params: { ids: ctx.args[0] } });
|
|
7765
|
+
} else if (method === "page") {
|
|
7766
|
+
const { offset, size, partition, partitionValues } = ctx.args[0] || {};
|
|
7767
|
+
key = await resource.cacheKeyFor({ action: method, params: { offset, size }, partition, partitionValues });
|
|
7768
|
+
} else if (method === "list" || method === "listIds" || method === "count") {
|
|
7769
|
+
const { partition, partitionValues } = ctx.args[0] || {};
|
|
7770
|
+
key = await resource.cacheKeyFor({ action: method, partition, partitionValues });
|
|
7771
|
+
} else if (method === "getAll") {
|
|
7772
|
+
key = await resource.cacheKeyFor({ action: method });
|
|
7773
|
+
} else if (method === "get") {
|
|
7774
|
+
key = await resource.cacheKeyFor({ action: method, params: { id: ctx.args[0] } });
|
|
7775
|
+
}
|
|
7776
|
+
if (this.driver instanceof PartitionAwareFilesystemCache) {
|
|
7777
|
+
let partition, partitionValues;
|
|
7778
|
+
if (method === "list" || method === "listIds" || method === "count" || method === "page") {
|
|
7779
|
+
const args = ctx.args[0] || {};
|
|
7780
|
+
partition = args.partition;
|
|
7781
|
+
partitionValues = args.partitionValues;
|
|
7782
|
+
}
|
|
7783
|
+
const [ok, err, result] = await try_fn_default(() => resource.cache._get(key, {
|
|
7784
|
+
resource: resource.name,
|
|
7785
|
+
action: method,
|
|
7786
|
+
partition,
|
|
7787
|
+
partitionValues
|
|
7788
|
+
}));
|
|
7789
|
+
if (ok && result !== null && result !== void 0) return result;
|
|
7790
|
+
if (!ok && err.name !== "NoSuchKey") throw err;
|
|
7791
|
+
const freshResult = await next();
|
|
7792
|
+
await resource.cache._set(key, freshResult, {
|
|
7793
|
+
resource: resource.name,
|
|
7794
|
+
action: method,
|
|
7795
|
+
partition,
|
|
7796
|
+
partitionValues
|
|
7797
|
+
});
|
|
7798
|
+
return freshResult;
|
|
7799
|
+
} else {
|
|
7800
|
+
const [ok, err, result] = await try_fn_default(() => resource.cache.get(key));
|
|
7801
|
+
if (ok && result !== null && result !== void 0) return result;
|
|
7802
|
+
if (!ok && err.name !== "NoSuchKey") throw err;
|
|
7803
|
+
const freshResult = await next();
|
|
7804
|
+
await resource.cache.set(key, freshResult);
|
|
7805
|
+
return freshResult;
|
|
7806
|
+
}
|
|
7807
|
+
});
|
|
7808
|
+
}
|
|
7809
|
+
const writeMethods = ["insert", "update", "delete", "deleteMany"];
|
|
7810
|
+
for (const method of writeMethods) {
|
|
7811
|
+
resource.useMiddleware(method, async (ctx, next) => {
|
|
7812
|
+
const result = await next();
|
|
7813
|
+
if (method === "insert") {
|
|
7814
|
+
await this.clearCacheForResource(resource, ctx.args[0]);
|
|
7815
|
+
} else if (method === "update") {
|
|
7816
|
+
await this.clearCacheForResource(resource, { id: ctx.args[0], ...ctx.args[1] });
|
|
7817
|
+
} else if (method === "delete") {
|
|
7818
|
+
let data = { id: ctx.args[0] };
|
|
7819
|
+
if (typeof resource.get === "function") {
|
|
7820
|
+
const [ok, err, full] = await try_fn_default(() => resource.get(ctx.args[0]));
|
|
7821
|
+
if (ok && full) data = full;
|
|
7822
|
+
}
|
|
7823
|
+
await this.clearCacheForResource(resource, data);
|
|
7824
|
+
} else if (method === "deleteMany") {
|
|
7825
|
+
await this.clearCacheForResource(resource);
|
|
7826
|
+
}
|
|
7827
|
+
return result;
|
|
7828
|
+
});
|
|
7829
|
+
}
|
|
7830
|
+
}
|
|
7831
|
+
async clearCacheForResource(resource, data) {
|
|
7832
|
+
if (!resource.cache) return;
|
|
7833
|
+
const keyPrefix = `resource=${resource.name}`;
|
|
7834
|
+
await resource.cache.clear(keyPrefix);
|
|
7835
|
+
if (this.config.includePartitions === true && resource.config?.partitions && Object.keys(resource.config.partitions).length > 0) {
|
|
7836
|
+
if (!data) {
|
|
7837
|
+
for (const partitionName of Object.keys(resource.config.partitions)) {
|
|
7838
|
+
const partitionKeyPrefix = join(keyPrefix, `partition=${partitionName}`);
|
|
7839
|
+
await resource.cache.clear(partitionKeyPrefix);
|
|
7840
|
+
}
|
|
7841
|
+
} else {
|
|
7842
|
+
const partitionValues = this.getPartitionValues(data, resource);
|
|
7843
|
+
for (const [partitionName, values] of Object.entries(partitionValues)) {
|
|
7844
|
+
if (values && Object.keys(values).length > 0 && Object.values(values).some((v) => v !== null && v !== void 0)) {
|
|
7845
|
+
const partitionKeyPrefix = join(keyPrefix, `partition=${partitionName}`);
|
|
7846
|
+
await resource.cache.clear(partitionKeyPrefix);
|
|
7847
|
+
}
|
|
7848
|
+
}
|
|
7849
|
+
}
|
|
7850
|
+
}
|
|
7851
|
+
}
|
|
7852
|
+
async generateCacheKey(resource, action, params = {}, partition = null, partitionValues = null) {
|
|
7853
|
+
const keyParts = [
|
|
7854
|
+
`resource=${resource.name}`,
|
|
7855
|
+
`action=${action}`
|
|
7856
|
+
];
|
|
7857
|
+
if (partition && partitionValues && Object.keys(partitionValues).length > 0) {
|
|
7858
|
+
keyParts.push(`partition:${partition}`);
|
|
7859
|
+
for (const [field, value] of Object.entries(partitionValues)) {
|
|
7860
|
+
if (value !== null && value !== void 0) {
|
|
7861
|
+
keyParts.push(`${field}:${value}`);
|
|
7862
|
+
}
|
|
7863
|
+
}
|
|
7864
|
+
}
|
|
7865
|
+
if (Object.keys(params).length > 0) {
|
|
7866
|
+
const paramsHash = await this.hashParams(params);
|
|
7867
|
+
keyParts.push(paramsHash);
|
|
7868
|
+
}
|
|
7869
|
+
return join(...keyParts) + ".json.gz";
|
|
7870
|
+
}
|
|
7871
|
+
async hashParams(params) {
|
|
7872
|
+
const sortedParams = Object.keys(params).sort().map((key) => `${key}:${params[key]}`).join("|") || "empty";
|
|
7873
|
+
return await sha256(sortedParams);
|
|
7874
|
+
}
|
|
7875
|
+
// Utility methods
|
|
7876
|
+
async getCacheStats() {
|
|
7877
|
+
if (!this.driver) return null;
|
|
7878
|
+
return {
|
|
7879
|
+
size: await this.driver.size(),
|
|
7880
|
+
keys: await this.driver.keys(),
|
|
7881
|
+
driver: this.driver.constructor.name
|
|
7882
|
+
};
|
|
7883
|
+
}
|
|
7884
|
+
async clearAllCache() {
|
|
7885
|
+
if (!this.driver) return;
|
|
7886
|
+
for (const resource of Object.values(this.database.resources)) {
|
|
7887
|
+
if (resource.cache) {
|
|
7888
|
+
const keyPrefix = `resource=${resource.name}`;
|
|
7889
|
+
await resource.cache.clear(keyPrefix);
|
|
7890
|
+
}
|
|
7891
|
+
}
|
|
7892
|
+
}
|
|
7893
|
+
async warmCache(resourceName, options = {}) {
|
|
7894
|
+
const resource = this.database.resources[resourceName];
|
|
7895
|
+
if (!resource) {
|
|
7896
|
+
throw new Error(`Resource '${resourceName}' not found`);
|
|
7897
|
+
}
|
|
7898
|
+
const { includePartitions = true } = options;
|
|
7899
|
+
if (this.driver instanceof PartitionAwareFilesystemCache && resource.warmPartitionCache) {
|
|
7900
|
+
const partitionNames = resource.config.partitions ? Object.keys(resource.config.partitions) : [];
|
|
7901
|
+
return await resource.warmPartitionCache(partitionNames, options);
|
|
7902
|
+
}
|
|
6991
7903
|
await resource.getAll();
|
|
6992
7904
|
if (includePartitions && resource.config.partitions) {
|
|
6993
7905
|
for (const [partitionName, partitionDef] of Object.entries(resource.config.partitions)) {
|
|
@@ -7009,6 +7921,57 @@ class CachePlugin extends plugin_class_default {
|
|
|
7009
7921
|
}
|
|
7010
7922
|
}
|
|
7011
7923
|
}
|
|
7924
|
+
// Partition-specific methods
|
|
7925
|
+
async getPartitionCacheStats(resourceName, partition = null) {
|
|
7926
|
+
if (!(this.driver instanceof PartitionAwareFilesystemCache)) {
|
|
7927
|
+
throw new Error("Partition cache statistics are only available with PartitionAwareFilesystemCache");
|
|
7928
|
+
}
|
|
7929
|
+
return await this.driver.getPartitionStats(resourceName, partition);
|
|
7930
|
+
}
|
|
7931
|
+
async getCacheRecommendations(resourceName) {
|
|
7932
|
+
if (!(this.driver instanceof PartitionAwareFilesystemCache)) {
|
|
7933
|
+
throw new Error("Cache recommendations are only available with PartitionAwareFilesystemCache");
|
|
7934
|
+
}
|
|
7935
|
+
return await this.driver.getCacheRecommendations(resourceName);
|
|
7936
|
+
}
|
|
7937
|
+
async clearPartitionCache(resourceName, partition, partitionValues = {}) {
|
|
7938
|
+
if (!(this.driver instanceof PartitionAwareFilesystemCache)) {
|
|
7939
|
+
throw new Error("Partition cache clearing is only available with PartitionAwareFilesystemCache");
|
|
7940
|
+
}
|
|
7941
|
+
return await this.driver.clearPartition(resourceName, partition, partitionValues);
|
|
7942
|
+
}
|
|
7943
|
+
async analyzeCacheUsage() {
|
|
7944
|
+
if (!(this.driver instanceof PartitionAwareFilesystemCache)) {
|
|
7945
|
+
return { message: "Cache usage analysis is only available with PartitionAwareFilesystemCache" };
|
|
7946
|
+
}
|
|
7947
|
+
const analysis = {
|
|
7948
|
+
totalResources: Object.keys(this.database.resources).length,
|
|
7949
|
+
resourceStats: {},
|
|
7950
|
+
recommendations: {},
|
|
7951
|
+
summary: {
|
|
7952
|
+
mostUsedPartitions: [],
|
|
7953
|
+
leastUsedPartitions: [],
|
|
7954
|
+
suggestedOptimizations: []
|
|
7955
|
+
}
|
|
7956
|
+
};
|
|
7957
|
+
for (const [resourceName, resource] of Object.entries(this.database.resources)) {
|
|
7958
|
+
try {
|
|
7959
|
+
analysis.resourceStats[resourceName] = await this.driver.getPartitionStats(resourceName);
|
|
7960
|
+
analysis.recommendations[resourceName] = await this.driver.getCacheRecommendations(resourceName);
|
|
7961
|
+
} catch (error) {
|
|
7962
|
+
analysis.resourceStats[resourceName] = { error: error.message };
|
|
7963
|
+
}
|
|
7964
|
+
}
|
|
7965
|
+
const allRecommendations = Object.values(analysis.recommendations).flat();
|
|
7966
|
+
analysis.summary.mostUsedPartitions = allRecommendations.filter((r) => r.recommendation === "preload").sort((a, b) => b.priority - a.priority).slice(0, 5);
|
|
7967
|
+
analysis.summary.leastUsedPartitions = allRecommendations.filter((r) => r.recommendation === "archive").slice(0, 5);
|
|
7968
|
+
analysis.summary.suggestedOptimizations = [
|
|
7969
|
+
`Consider preloading ${analysis.summary.mostUsedPartitions.length} high-usage partitions`,
|
|
7970
|
+
`Archive ${analysis.summary.leastUsedPartitions.length} unused partitions`,
|
|
7971
|
+
`Monitor cache hit rates for partition efficiency`
|
|
7972
|
+
];
|
|
7973
|
+
return analysis;
|
|
7974
|
+
}
|
|
7012
7975
|
}
|
|
7013
7976
|
|
|
7014
7977
|
const CostsPlugin = {
|
|
@@ -7185,24 +8148,29 @@ class FullTextPlugin extends plugin_class_default {
|
|
|
7185
8148
|
resource._deleteMany = resource.deleteMany;
|
|
7186
8149
|
this.wrapResourceMethod(resource, "insert", async (result, args, methodName) => {
|
|
7187
8150
|
const [data] = args;
|
|
7188
|
-
this.indexRecord(resource.name, result.id, data).catch(
|
|
8151
|
+
this.indexRecord(resource.name, result.id, data).catch(() => {
|
|
8152
|
+
});
|
|
7189
8153
|
return result;
|
|
7190
8154
|
});
|
|
7191
8155
|
this.wrapResourceMethod(resource, "update", async (result, args, methodName) => {
|
|
7192
8156
|
const [id, data] = args;
|
|
7193
|
-
this.removeRecordFromIndex(resource.name, id).catch(
|
|
7194
|
-
|
|
8157
|
+
this.removeRecordFromIndex(resource.name, id).catch(() => {
|
|
8158
|
+
});
|
|
8159
|
+
this.indexRecord(resource.name, id, result).catch(() => {
|
|
8160
|
+
});
|
|
7195
8161
|
return result;
|
|
7196
8162
|
});
|
|
7197
8163
|
this.wrapResourceMethod(resource, "delete", async (result, args, methodName) => {
|
|
7198
8164
|
const [id] = args;
|
|
7199
|
-
this.removeRecordFromIndex(resource.name, id).catch(
|
|
8165
|
+
this.removeRecordFromIndex(resource.name, id).catch(() => {
|
|
8166
|
+
});
|
|
7200
8167
|
return result;
|
|
7201
8168
|
});
|
|
7202
8169
|
this.wrapResourceMethod(resource, "deleteMany", async (result, args, methodName) => {
|
|
7203
8170
|
const [ids] = args;
|
|
7204
8171
|
for (const id of ids) {
|
|
7205
|
-
this.removeRecordFromIndex(resource.name, id).catch(
|
|
8172
|
+
this.removeRecordFromIndex(resource.name, id).catch(() => {
|
|
8173
|
+
});
|
|
7206
8174
|
}
|
|
7207
8175
|
return result;
|
|
7208
8176
|
});
|
|
@@ -7692,7 +8660,8 @@ class MetricsPlugin extends plugin_class_default {
|
|
|
7692
8660
|
}
|
|
7693
8661
|
if (this.config.flushInterval > 0) {
|
|
7694
8662
|
this.flushTimer = setInterval(() => {
|
|
7695
|
-
this.flushMetrics().catch(
|
|
8663
|
+
this.flushMetrics().catch(() => {
|
|
8664
|
+
});
|
|
7696
8665
|
}, this.config.flushInterval);
|
|
7697
8666
|
}
|
|
7698
8667
|
}
|
|
@@ -7764,9 +8733,6 @@ class MetricsPlugin extends plugin_class_default {
|
|
|
7764
8733
|
}
|
|
7765
8734
|
this.resetMetrics();
|
|
7766
8735
|
});
|
|
7767
|
-
if (!ok) {
|
|
7768
|
-
console.error("Failed to flush metrics:", err);
|
|
7769
|
-
}
|
|
7770
8736
|
}
|
|
7771
8737
|
resetMetrics() {
|
|
7772
8738
|
for (const operation of Object.keys(this.metrics.operations)) {
|
|
@@ -7898,20 +8864,628 @@ class MetricsPlugin extends plugin_class_default {
|
|
|
7898
8864
|
await this.metricsResource.delete(metric.id);
|
|
7899
8865
|
}
|
|
7900
8866
|
}
|
|
7901
|
-
if (this.errorsResource) {
|
|
7902
|
-
const oldErrors = await this.getErrorLogs({ endDate: cutoffDate.toISOString() });
|
|
7903
|
-
for (const error of oldErrors) {
|
|
7904
|
-
await this.errorsResource.delete(error.id);
|
|
7905
|
-
}
|
|
8867
|
+
if (this.errorsResource) {
|
|
8868
|
+
const oldErrors = await this.getErrorLogs({ endDate: cutoffDate.toISOString() });
|
|
8869
|
+
for (const error of oldErrors) {
|
|
8870
|
+
await this.errorsResource.delete(error.id);
|
|
8871
|
+
}
|
|
8872
|
+
}
|
|
8873
|
+
if (this.performanceResource) {
|
|
8874
|
+
const oldPerformance = await this.getPerformanceLogs({ endDate: cutoffDate.toISOString() });
|
|
8875
|
+
for (const perf of oldPerformance) {
|
|
8876
|
+
await this.performanceResource.delete(perf.id);
|
|
8877
|
+
}
|
|
8878
|
+
}
|
|
8879
|
+
}
|
|
8880
|
+
}
|
|
8881
|
+
|
|
8882
|
+
class BaseReplicator extends EventEmitter {
|
|
8883
|
+
constructor(config = {}) {
|
|
8884
|
+
super();
|
|
8885
|
+
this.config = config;
|
|
8886
|
+
this.name = this.constructor.name;
|
|
8887
|
+
this.enabled = config.enabled !== false;
|
|
8888
|
+
}
|
|
8889
|
+
/**
|
|
8890
|
+
* Initialize the replicator
|
|
8891
|
+
* @param {Object} database - The s3db database instance
|
|
8892
|
+
* @returns {Promise<void>}
|
|
8893
|
+
*/
|
|
8894
|
+
async initialize(database) {
|
|
8895
|
+
this.database = database;
|
|
8896
|
+
this.emit("initialized", { replicator: this.name });
|
|
8897
|
+
}
|
|
8898
|
+
/**
|
|
8899
|
+
* Replicate data to the target
|
|
8900
|
+
* @param {string} resourceName - Name of the resource being replicated
|
|
8901
|
+
* @param {string} operation - Operation type (insert, update, delete)
|
|
8902
|
+
* @param {Object} data - The data to replicate
|
|
8903
|
+
* @param {string} id - Record ID
|
|
8904
|
+
* @returns {Promise<Object>} replicator result
|
|
8905
|
+
*/
|
|
8906
|
+
async replicate(resourceName, operation, data, id) {
|
|
8907
|
+
throw new Error(`replicate() method must be implemented by ${this.name}`);
|
|
8908
|
+
}
|
|
8909
|
+
/**
|
|
8910
|
+
* Replicate multiple records in batch
|
|
8911
|
+
* @param {string} resourceName - Name of the resource being replicated
|
|
8912
|
+
* @param {Array} records - Array of records to replicate
|
|
8913
|
+
* @returns {Promise<Object>} Batch replicator result
|
|
8914
|
+
*/
|
|
8915
|
+
async replicateBatch(resourceName, records) {
|
|
8916
|
+
throw new Error(`replicateBatch() method must be implemented by ${this.name}`);
|
|
8917
|
+
}
|
|
8918
|
+
/**
|
|
8919
|
+
* Test the connection to the target
|
|
8920
|
+
* @returns {Promise<boolean>} True if connection is successful
|
|
8921
|
+
*/
|
|
8922
|
+
async testConnection() {
|
|
8923
|
+
throw new Error(`testConnection() method must be implemented by ${this.name}`);
|
|
8924
|
+
}
|
|
8925
|
+
/**
|
|
8926
|
+
* Get replicator status and statistics
|
|
8927
|
+
* @returns {Promise<Object>} Status information
|
|
8928
|
+
*/
|
|
8929
|
+
async getStatus() {
|
|
8930
|
+
return {
|
|
8931
|
+
name: this.name,
|
|
8932
|
+
// Removed: enabled: this.enabled,
|
|
8933
|
+
config: this.config,
|
|
8934
|
+
connected: false
|
|
8935
|
+
};
|
|
8936
|
+
}
|
|
8937
|
+
/**
|
|
8938
|
+
* Cleanup resources
|
|
8939
|
+
* @returns {Promise<void>}
|
|
8940
|
+
*/
|
|
8941
|
+
async cleanup() {
|
|
8942
|
+
this.emit("cleanup", { replicator: this.name });
|
|
8943
|
+
}
|
|
8944
|
+
/**
|
|
8945
|
+
* Validate replicator configuration
|
|
8946
|
+
* @returns {Object} Validation result
|
|
8947
|
+
*/
|
|
8948
|
+
validateConfig() {
|
|
8949
|
+
return { isValid: true, errors: [] };
|
|
8950
|
+
}
|
|
8951
|
+
}
|
|
8952
|
+
var base_replicator_class_default = BaseReplicator;
|
|
8953
|
+
|
|
8954
|
+
class BigqueryReplicator extends base_replicator_class_default {
|
|
8955
|
+
constructor(config = {}, resources = {}) {
|
|
8956
|
+
super(config);
|
|
8957
|
+
this.projectId = config.projectId;
|
|
8958
|
+
this.datasetId = config.datasetId;
|
|
8959
|
+
this.bigqueryClient = null;
|
|
8960
|
+
this.credentials = config.credentials;
|
|
8961
|
+
this.location = config.location || "US";
|
|
8962
|
+
this.logTable = config.logTable;
|
|
8963
|
+
this.resources = this.parseResourcesConfig(resources);
|
|
8964
|
+
}
|
|
8965
|
+
parseResourcesConfig(resources) {
|
|
8966
|
+
const parsed = {};
|
|
8967
|
+
for (const [resourceName, config] of Object.entries(resources)) {
|
|
8968
|
+
if (typeof config === "string") {
|
|
8969
|
+
parsed[resourceName] = [{
|
|
8970
|
+
table: config,
|
|
8971
|
+
actions: ["insert"],
|
|
8972
|
+
transform: null
|
|
8973
|
+
}];
|
|
8974
|
+
} else if (Array.isArray(config)) {
|
|
8975
|
+
parsed[resourceName] = config.map((item) => {
|
|
8976
|
+
if (typeof item === "string") {
|
|
8977
|
+
return { table: item, actions: ["insert"], transform: null };
|
|
8978
|
+
}
|
|
8979
|
+
return {
|
|
8980
|
+
table: item.table,
|
|
8981
|
+
actions: item.actions || ["insert"],
|
|
8982
|
+
transform: item.transform || null
|
|
8983
|
+
};
|
|
8984
|
+
});
|
|
8985
|
+
} else if (typeof config === "object") {
|
|
8986
|
+
parsed[resourceName] = [{
|
|
8987
|
+
table: config.table,
|
|
8988
|
+
actions: config.actions || ["insert"],
|
|
8989
|
+
transform: config.transform || null
|
|
8990
|
+
}];
|
|
8991
|
+
}
|
|
8992
|
+
}
|
|
8993
|
+
return parsed;
|
|
8994
|
+
}
|
|
8995
|
+
validateConfig() {
|
|
8996
|
+
const errors = [];
|
|
8997
|
+
if (!this.projectId) errors.push("projectId is required");
|
|
8998
|
+
if (!this.datasetId) errors.push("datasetId is required");
|
|
8999
|
+
if (Object.keys(this.resources).length === 0) errors.push("At least one resource must be configured");
|
|
9000
|
+
for (const [resourceName, tables] of Object.entries(this.resources)) {
|
|
9001
|
+
for (const tableConfig of tables) {
|
|
9002
|
+
if (!tableConfig.table) {
|
|
9003
|
+
errors.push(`Table name is required for resource '${resourceName}'`);
|
|
9004
|
+
}
|
|
9005
|
+
if (!Array.isArray(tableConfig.actions) || tableConfig.actions.length === 0) {
|
|
9006
|
+
errors.push(`Actions array is required for resource '${resourceName}'`);
|
|
9007
|
+
}
|
|
9008
|
+
const validActions = ["insert", "update", "delete"];
|
|
9009
|
+
const invalidActions = tableConfig.actions.filter((action) => !validActions.includes(action));
|
|
9010
|
+
if (invalidActions.length > 0) {
|
|
9011
|
+
errors.push(`Invalid actions for resource '${resourceName}': ${invalidActions.join(", ")}. Valid actions: ${validActions.join(", ")}`);
|
|
9012
|
+
}
|
|
9013
|
+
if (tableConfig.transform && typeof tableConfig.transform !== "function") {
|
|
9014
|
+
errors.push(`Transform must be a function for resource '${resourceName}'`);
|
|
9015
|
+
}
|
|
9016
|
+
}
|
|
9017
|
+
}
|
|
9018
|
+
return { isValid: errors.length === 0, errors };
|
|
9019
|
+
}
|
|
9020
|
+
async initialize(database) {
|
|
9021
|
+
await super.initialize(database);
|
|
9022
|
+
const [ok, err, sdk] = await try_fn_default(() => import('@google-cloud/bigquery'));
|
|
9023
|
+
if (!ok) {
|
|
9024
|
+
this.emit("initialization_error", { replicator: this.name, error: err.message });
|
|
9025
|
+
throw err;
|
|
9026
|
+
}
|
|
9027
|
+
const { BigQuery } = sdk;
|
|
9028
|
+
this.bigqueryClient = new BigQuery({
|
|
9029
|
+
projectId: this.projectId,
|
|
9030
|
+
credentials: this.credentials,
|
|
9031
|
+
location: this.location
|
|
9032
|
+
});
|
|
9033
|
+
this.emit("initialized", {
|
|
9034
|
+
replicator: this.name,
|
|
9035
|
+
projectId: this.projectId,
|
|
9036
|
+
datasetId: this.datasetId,
|
|
9037
|
+
resources: Object.keys(this.resources)
|
|
9038
|
+
});
|
|
9039
|
+
}
|
|
9040
|
+
shouldReplicateResource(resourceName) {
|
|
9041
|
+
return this.resources.hasOwnProperty(resourceName);
|
|
9042
|
+
}
|
|
9043
|
+
shouldReplicateAction(resourceName, operation) {
|
|
9044
|
+
if (!this.resources[resourceName]) return false;
|
|
9045
|
+
return this.resources[resourceName].some(
|
|
9046
|
+
(tableConfig) => tableConfig.actions.includes(operation)
|
|
9047
|
+
);
|
|
9048
|
+
}
|
|
9049
|
+
getTablesForResource(resourceName, operation) {
|
|
9050
|
+
if (!this.resources[resourceName]) return [];
|
|
9051
|
+
return this.resources[resourceName].filter((tableConfig) => tableConfig.actions.includes(operation)).map((tableConfig) => ({
|
|
9052
|
+
table: tableConfig.table,
|
|
9053
|
+
transform: tableConfig.transform
|
|
9054
|
+
}));
|
|
9055
|
+
}
|
|
9056
|
+
applyTransform(data, transformFn) {
|
|
9057
|
+
if (!transformFn) return data;
|
|
9058
|
+
let transformedData = JSON.parse(JSON.stringify(data));
|
|
9059
|
+
if (transformedData._length) delete transformedData._length;
|
|
9060
|
+
return transformFn(transformedData);
|
|
9061
|
+
}
|
|
9062
|
+
async replicate(resourceName, operation, data, id, beforeData = null) {
|
|
9063
|
+
if (!this.enabled || !this.shouldReplicateResource(resourceName)) {
|
|
9064
|
+
return { skipped: true, reason: "resource_not_included" };
|
|
9065
|
+
}
|
|
9066
|
+
if (!this.shouldReplicateAction(resourceName, operation)) {
|
|
9067
|
+
return { skipped: true, reason: "action_not_included" };
|
|
9068
|
+
}
|
|
9069
|
+
const tableConfigs = this.getTablesForResource(resourceName, operation);
|
|
9070
|
+
if (tableConfigs.length === 0) {
|
|
9071
|
+
return { skipped: true, reason: "no_tables_for_action" };
|
|
9072
|
+
}
|
|
9073
|
+
const results = [];
|
|
9074
|
+
const errors = [];
|
|
9075
|
+
const [ok, err, result] = await try_fn_default(async () => {
|
|
9076
|
+
const dataset = this.bigqueryClient.dataset(this.datasetId);
|
|
9077
|
+
for (const tableConfig of tableConfigs) {
|
|
9078
|
+
const [okTable, errTable] = await try_fn_default(async () => {
|
|
9079
|
+
const table = dataset.table(tableConfig.table);
|
|
9080
|
+
let job;
|
|
9081
|
+
if (operation === "insert") {
|
|
9082
|
+
const transformedData = this.applyTransform(data, tableConfig.transform);
|
|
9083
|
+
job = await table.insert([transformedData]);
|
|
9084
|
+
} else if (operation === "update") {
|
|
9085
|
+
const transformedData = this.applyTransform(data, tableConfig.transform);
|
|
9086
|
+
const keys = Object.keys(transformedData).filter((k) => k !== "id");
|
|
9087
|
+
const setClause = keys.map((k) => `${k} = @${k}`).join(", ");
|
|
9088
|
+
const params = { id, ...transformedData };
|
|
9089
|
+
const query = `UPDATE \`${this.projectId}.${this.datasetId}.${tableConfig.table}\` SET ${setClause} WHERE id = @id`;
|
|
9090
|
+
const maxRetries = 2;
|
|
9091
|
+
let lastError = null;
|
|
9092
|
+
for (let attempt = 1; attempt <= maxRetries; attempt++) {
|
|
9093
|
+
try {
|
|
9094
|
+
const [updateJob] = await this.bigqueryClient.createQueryJob({
|
|
9095
|
+
query,
|
|
9096
|
+
params,
|
|
9097
|
+
location: this.location
|
|
9098
|
+
});
|
|
9099
|
+
await updateJob.getQueryResults();
|
|
9100
|
+
job = [updateJob];
|
|
9101
|
+
break;
|
|
9102
|
+
} catch (error) {
|
|
9103
|
+
lastError = error;
|
|
9104
|
+
if (error?.message?.includes("streaming buffer") && attempt < maxRetries) {
|
|
9105
|
+
const delaySeconds = 30;
|
|
9106
|
+
await new Promise((resolve) => setTimeout(resolve, delaySeconds * 1e3));
|
|
9107
|
+
continue;
|
|
9108
|
+
}
|
|
9109
|
+
throw error;
|
|
9110
|
+
}
|
|
9111
|
+
}
|
|
9112
|
+
if (!job) throw lastError;
|
|
9113
|
+
} else if (operation === "delete") {
|
|
9114
|
+
const query = `DELETE FROM \`${this.projectId}.${this.datasetId}.${tableConfig.table}\` WHERE id = @id`;
|
|
9115
|
+
const [deleteJob] = await this.bigqueryClient.createQueryJob({
|
|
9116
|
+
query,
|
|
9117
|
+
params: { id },
|
|
9118
|
+
location: this.location
|
|
9119
|
+
});
|
|
9120
|
+
await deleteJob.getQueryResults();
|
|
9121
|
+
job = [deleteJob];
|
|
9122
|
+
} else {
|
|
9123
|
+
throw new Error(`Unsupported operation: ${operation}`);
|
|
9124
|
+
}
|
|
9125
|
+
results.push({
|
|
9126
|
+
table: tableConfig.table,
|
|
9127
|
+
success: true,
|
|
9128
|
+
jobId: job[0]?.id
|
|
9129
|
+
});
|
|
9130
|
+
});
|
|
9131
|
+
if (!okTable) {
|
|
9132
|
+
errors.push({
|
|
9133
|
+
table: tableConfig.table,
|
|
9134
|
+
error: errTable.message
|
|
9135
|
+
});
|
|
9136
|
+
}
|
|
9137
|
+
}
|
|
9138
|
+
if (this.logTable) {
|
|
9139
|
+
const [okLog, errLog] = await try_fn_default(async () => {
|
|
9140
|
+
const logTable = dataset.table(this.logTable);
|
|
9141
|
+
await logTable.insert([{
|
|
9142
|
+
resource_name: resourceName,
|
|
9143
|
+
operation,
|
|
9144
|
+
record_id: id,
|
|
9145
|
+
data: JSON.stringify(data),
|
|
9146
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
9147
|
+
source: "s3db-replicator"
|
|
9148
|
+
}]);
|
|
9149
|
+
});
|
|
9150
|
+
if (!okLog) {
|
|
9151
|
+
}
|
|
9152
|
+
}
|
|
9153
|
+
const success = errors.length === 0;
|
|
9154
|
+
this.emit("replicated", {
|
|
9155
|
+
replicator: this.name,
|
|
9156
|
+
resourceName,
|
|
9157
|
+
operation,
|
|
9158
|
+
id,
|
|
9159
|
+
tables: tableConfigs.map((t) => t.table),
|
|
9160
|
+
results,
|
|
9161
|
+
errors,
|
|
9162
|
+
success
|
|
9163
|
+
});
|
|
9164
|
+
return {
|
|
9165
|
+
success,
|
|
9166
|
+
results,
|
|
9167
|
+
errors,
|
|
9168
|
+
tables: tableConfigs.map((t) => t.table)
|
|
9169
|
+
};
|
|
9170
|
+
});
|
|
9171
|
+
if (ok) return result;
|
|
9172
|
+
this.emit("replicator_error", {
|
|
9173
|
+
replicator: this.name,
|
|
9174
|
+
resourceName,
|
|
9175
|
+
operation,
|
|
9176
|
+
id,
|
|
9177
|
+
error: err.message
|
|
9178
|
+
});
|
|
9179
|
+
return { success: false, error: err.message };
|
|
9180
|
+
}
|
|
9181
|
+
async replicateBatch(resourceName, records) {
|
|
9182
|
+
const results = [];
|
|
9183
|
+
const errors = [];
|
|
9184
|
+
for (const record of records) {
|
|
9185
|
+
const [ok, err, res] = await try_fn_default(() => this.replicate(
|
|
9186
|
+
resourceName,
|
|
9187
|
+
record.operation,
|
|
9188
|
+
record.data,
|
|
9189
|
+
record.id,
|
|
9190
|
+
record.beforeData
|
|
9191
|
+
));
|
|
9192
|
+
if (ok) results.push(res);
|
|
9193
|
+
else errors.push({ id: record.id, error: err.message });
|
|
9194
|
+
}
|
|
9195
|
+
return {
|
|
9196
|
+
success: errors.length === 0,
|
|
9197
|
+
results,
|
|
9198
|
+
errors
|
|
9199
|
+
};
|
|
9200
|
+
}
|
|
9201
|
+
async testConnection() {
|
|
9202
|
+
const [ok, err] = await try_fn_default(async () => {
|
|
9203
|
+
if (!this.bigqueryClient) await this.initialize();
|
|
9204
|
+
const dataset = this.bigqueryClient.dataset(this.datasetId);
|
|
9205
|
+
await dataset.getMetadata();
|
|
9206
|
+
return true;
|
|
9207
|
+
});
|
|
9208
|
+
if (ok) return true;
|
|
9209
|
+
this.emit("connection_error", { replicator: this.name, error: err.message });
|
|
9210
|
+
return false;
|
|
9211
|
+
}
|
|
9212
|
+
async cleanup() {
|
|
9213
|
+
}
|
|
9214
|
+
getStatus() {
|
|
9215
|
+
return {
|
|
9216
|
+
...super.getStatus(),
|
|
9217
|
+
projectId: this.projectId,
|
|
9218
|
+
datasetId: this.datasetId,
|
|
9219
|
+
resources: this.resources,
|
|
9220
|
+
logTable: this.logTable
|
|
9221
|
+
};
|
|
9222
|
+
}
|
|
9223
|
+
}
|
|
9224
|
+
var bigquery_replicator_class_default = BigqueryReplicator;
|
|
9225
|
+
|
|
9226
|
+
class PostgresReplicator extends base_replicator_class_default {
|
|
9227
|
+
constructor(config = {}, resources = {}) {
|
|
9228
|
+
super(config);
|
|
9229
|
+
this.connectionString = config.connectionString;
|
|
9230
|
+
this.host = config.host;
|
|
9231
|
+
this.port = config.port || 5432;
|
|
9232
|
+
this.database = config.database;
|
|
9233
|
+
this.user = config.user;
|
|
9234
|
+
this.password = config.password;
|
|
9235
|
+
this.client = null;
|
|
9236
|
+
this.ssl = config.ssl;
|
|
9237
|
+
this.logTable = config.logTable;
|
|
9238
|
+
this.resources = this.parseResourcesConfig(resources);
|
|
9239
|
+
}
|
|
9240
|
+
parseResourcesConfig(resources) {
|
|
9241
|
+
const parsed = {};
|
|
9242
|
+
for (const [resourceName, config] of Object.entries(resources)) {
|
|
9243
|
+
if (typeof config === "string") {
|
|
9244
|
+
parsed[resourceName] = [{
|
|
9245
|
+
table: config,
|
|
9246
|
+
actions: ["insert"]
|
|
9247
|
+
}];
|
|
9248
|
+
} else if (Array.isArray(config)) {
|
|
9249
|
+
parsed[resourceName] = config.map((item) => {
|
|
9250
|
+
if (typeof item === "string") {
|
|
9251
|
+
return { table: item, actions: ["insert"] };
|
|
9252
|
+
}
|
|
9253
|
+
return {
|
|
9254
|
+
table: item.table,
|
|
9255
|
+
actions: item.actions || ["insert"]
|
|
9256
|
+
};
|
|
9257
|
+
});
|
|
9258
|
+
} else if (typeof config === "object") {
|
|
9259
|
+
parsed[resourceName] = [{
|
|
9260
|
+
table: config.table,
|
|
9261
|
+
actions: config.actions || ["insert"]
|
|
9262
|
+
}];
|
|
9263
|
+
}
|
|
9264
|
+
}
|
|
9265
|
+
return parsed;
|
|
9266
|
+
}
|
|
9267
|
+
validateConfig() {
|
|
9268
|
+
const errors = [];
|
|
9269
|
+
if (!this.connectionString && (!this.host || !this.database)) {
|
|
9270
|
+
errors.push("Either connectionString or host+database must be provided");
|
|
9271
|
+
}
|
|
9272
|
+
if (Object.keys(this.resources).length === 0) {
|
|
9273
|
+
errors.push("At least one resource must be configured");
|
|
9274
|
+
}
|
|
9275
|
+
for (const [resourceName, tables] of Object.entries(this.resources)) {
|
|
9276
|
+
for (const tableConfig of tables) {
|
|
9277
|
+
if (!tableConfig.table) {
|
|
9278
|
+
errors.push(`Table name is required for resource '${resourceName}'`);
|
|
9279
|
+
}
|
|
9280
|
+
if (!Array.isArray(tableConfig.actions) || tableConfig.actions.length === 0) {
|
|
9281
|
+
errors.push(`Actions array is required for resource '${resourceName}'`);
|
|
9282
|
+
}
|
|
9283
|
+
const validActions = ["insert", "update", "delete"];
|
|
9284
|
+
const invalidActions = tableConfig.actions.filter((action) => !validActions.includes(action));
|
|
9285
|
+
if (invalidActions.length > 0) {
|
|
9286
|
+
errors.push(`Invalid actions for resource '${resourceName}': ${invalidActions.join(", ")}. Valid actions: ${validActions.join(", ")}`);
|
|
9287
|
+
}
|
|
9288
|
+
}
|
|
9289
|
+
}
|
|
9290
|
+
return { isValid: errors.length === 0, errors };
|
|
9291
|
+
}
|
|
9292
|
+
async initialize(database) {
|
|
9293
|
+
await super.initialize(database);
|
|
9294
|
+
const [ok, err, sdk] = await try_fn_default(() => import('pg'));
|
|
9295
|
+
if (!ok) {
|
|
9296
|
+
this.emit("initialization_error", {
|
|
9297
|
+
replicator: this.name,
|
|
9298
|
+
error: err.message
|
|
9299
|
+
});
|
|
9300
|
+
throw err;
|
|
9301
|
+
}
|
|
9302
|
+
const { Client } = sdk;
|
|
9303
|
+
const config = this.connectionString ? {
|
|
9304
|
+
connectionString: this.connectionString,
|
|
9305
|
+
ssl: this.ssl
|
|
9306
|
+
} : {
|
|
9307
|
+
host: this.host,
|
|
9308
|
+
port: this.port,
|
|
9309
|
+
database: this.database,
|
|
9310
|
+
user: this.user,
|
|
9311
|
+
password: this.password,
|
|
9312
|
+
ssl: this.ssl
|
|
9313
|
+
};
|
|
9314
|
+
this.client = new Client(config);
|
|
9315
|
+
await this.client.connect();
|
|
9316
|
+
if (this.logTable) {
|
|
9317
|
+
await this.createLogTableIfNotExists();
|
|
9318
|
+
}
|
|
9319
|
+
this.emit("initialized", {
|
|
9320
|
+
replicator: this.name,
|
|
9321
|
+
database: this.database || "postgres",
|
|
9322
|
+
resources: Object.keys(this.resources)
|
|
9323
|
+
});
|
|
9324
|
+
}
|
|
9325
|
+
async createLogTableIfNotExists() {
|
|
9326
|
+
const createTableQuery = `
|
|
9327
|
+
CREATE TABLE IF NOT EXISTS ${this.logTable} (
|
|
9328
|
+
id SERIAL PRIMARY KEY,
|
|
9329
|
+
resource_name VARCHAR(255) NOT NULL,
|
|
9330
|
+
operation VARCHAR(50) NOT NULL,
|
|
9331
|
+
record_id VARCHAR(255) NOT NULL,
|
|
9332
|
+
data JSONB,
|
|
9333
|
+
timestamp TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
|
|
9334
|
+
source VARCHAR(100) DEFAULT 's3db-replicator',
|
|
9335
|
+
created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
|
|
9336
|
+
);
|
|
9337
|
+
CREATE INDEX IF NOT EXISTS idx_${this.logTable}_resource_name ON ${this.logTable}(resource_name);
|
|
9338
|
+
CREATE INDEX IF NOT EXISTS idx_${this.logTable}_operation ON ${this.logTable}(operation);
|
|
9339
|
+
CREATE INDEX IF NOT EXISTS idx_${this.logTable}_record_id ON ${this.logTable}(record_id);
|
|
9340
|
+
CREATE INDEX IF NOT EXISTS idx_${this.logTable}_timestamp ON ${this.logTable}(timestamp);
|
|
9341
|
+
`;
|
|
9342
|
+
await this.client.query(createTableQuery);
|
|
9343
|
+
}
|
|
9344
|
+
shouldReplicateResource(resourceName) {
|
|
9345
|
+
return this.resources.hasOwnProperty(resourceName);
|
|
9346
|
+
}
|
|
9347
|
+
shouldReplicateAction(resourceName, operation) {
|
|
9348
|
+
if (!this.resources[resourceName]) return false;
|
|
9349
|
+
return this.resources[resourceName].some(
|
|
9350
|
+
(tableConfig) => tableConfig.actions.includes(operation)
|
|
9351
|
+
);
|
|
9352
|
+
}
|
|
9353
|
+
getTablesForResource(resourceName, operation) {
|
|
9354
|
+
if (!this.resources[resourceName]) return [];
|
|
9355
|
+
return this.resources[resourceName].filter((tableConfig) => tableConfig.actions.includes(operation)).map((tableConfig) => tableConfig.table);
|
|
9356
|
+
}
|
|
9357
|
+
async replicate(resourceName, operation, data, id, beforeData = null) {
|
|
9358
|
+
if (!this.enabled || !this.shouldReplicateResource(resourceName)) {
|
|
9359
|
+
return { skipped: true, reason: "resource_not_included" };
|
|
9360
|
+
}
|
|
9361
|
+
if (!this.shouldReplicateAction(resourceName, operation)) {
|
|
9362
|
+
return { skipped: true, reason: "action_not_included" };
|
|
7906
9363
|
}
|
|
7907
|
-
|
|
7908
|
-
|
|
7909
|
-
|
|
7910
|
-
|
|
9364
|
+
const tables = this.getTablesForResource(resourceName, operation);
|
|
9365
|
+
if (tables.length === 0) {
|
|
9366
|
+
return { skipped: true, reason: "no_tables_for_action" };
|
|
9367
|
+
}
|
|
9368
|
+
const results = [];
|
|
9369
|
+
const errors = [];
|
|
9370
|
+
const [ok, err, result] = await try_fn_default(async () => {
|
|
9371
|
+
for (const table of tables) {
|
|
9372
|
+
const [okTable, errTable] = await try_fn_default(async () => {
|
|
9373
|
+
let result2;
|
|
9374
|
+
if (operation === "insert") {
|
|
9375
|
+
const keys = Object.keys(data);
|
|
9376
|
+
const values = keys.map((k) => data[k]);
|
|
9377
|
+
const columns = keys.map((k) => `"${k}"`).join(", ");
|
|
9378
|
+
const params = keys.map((_, i) => `$${i + 1}`).join(", ");
|
|
9379
|
+
const sql = `INSERT INTO ${table} (${columns}) VALUES (${params}) ON CONFLICT (id) DO NOTHING RETURNING *`;
|
|
9380
|
+
result2 = await this.client.query(sql, values);
|
|
9381
|
+
} else if (operation === "update") {
|
|
9382
|
+
const keys = Object.keys(data).filter((k) => k !== "id");
|
|
9383
|
+
const setClause = keys.map((k, i) => `"${k}"=$${i + 1}`).join(", ");
|
|
9384
|
+
const values = keys.map((k) => data[k]);
|
|
9385
|
+
values.push(id);
|
|
9386
|
+
const sql = `UPDATE ${table} SET ${setClause} WHERE id=$${keys.length + 1} RETURNING *`;
|
|
9387
|
+
result2 = await this.client.query(sql, values);
|
|
9388
|
+
} else if (operation === "delete") {
|
|
9389
|
+
const sql = `DELETE FROM ${table} WHERE id=$1 RETURNING *`;
|
|
9390
|
+
result2 = await this.client.query(sql, [id]);
|
|
9391
|
+
} else {
|
|
9392
|
+
throw new Error(`Unsupported operation: ${operation}`);
|
|
9393
|
+
}
|
|
9394
|
+
results.push({
|
|
9395
|
+
table,
|
|
9396
|
+
success: true,
|
|
9397
|
+
rows: result2.rows,
|
|
9398
|
+
rowCount: result2.rowCount
|
|
9399
|
+
});
|
|
9400
|
+
});
|
|
9401
|
+
if (!okTable) {
|
|
9402
|
+
errors.push({
|
|
9403
|
+
table,
|
|
9404
|
+
error: errTable.message
|
|
9405
|
+
});
|
|
9406
|
+
}
|
|
9407
|
+
}
|
|
9408
|
+
if (this.logTable) {
|
|
9409
|
+
const [okLog, errLog] = await try_fn_default(async () => {
|
|
9410
|
+
await this.client.query(
|
|
9411
|
+
`INSERT INTO ${this.logTable} (resource_name, operation, record_id, data, timestamp, source) VALUES ($1, $2, $3, $4, $5, $6)`,
|
|
9412
|
+
[resourceName, operation, id, JSON.stringify(data), (/* @__PURE__ */ new Date()).toISOString(), "s3db-replicator"]
|
|
9413
|
+
);
|
|
9414
|
+
});
|
|
9415
|
+
if (!okLog) {
|
|
9416
|
+
}
|
|
7911
9417
|
}
|
|
9418
|
+
const success = errors.length === 0;
|
|
9419
|
+
this.emit("replicated", {
|
|
9420
|
+
replicator: this.name,
|
|
9421
|
+
resourceName,
|
|
9422
|
+
operation,
|
|
9423
|
+
id,
|
|
9424
|
+
tables,
|
|
9425
|
+
results,
|
|
9426
|
+
errors,
|
|
9427
|
+
success
|
|
9428
|
+
});
|
|
9429
|
+
return {
|
|
9430
|
+
success,
|
|
9431
|
+
results,
|
|
9432
|
+
errors,
|
|
9433
|
+
tables
|
|
9434
|
+
};
|
|
9435
|
+
});
|
|
9436
|
+
if (ok) return result;
|
|
9437
|
+
this.emit("replicator_error", {
|
|
9438
|
+
replicator: this.name,
|
|
9439
|
+
resourceName,
|
|
9440
|
+
operation,
|
|
9441
|
+
id,
|
|
9442
|
+
error: err.message
|
|
9443
|
+
});
|
|
9444
|
+
return { success: false, error: err.message };
|
|
9445
|
+
}
|
|
9446
|
+
async replicateBatch(resourceName, records) {
|
|
9447
|
+
const results = [];
|
|
9448
|
+
const errors = [];
|
|
9449
|
+
for (const record of records) {
|
|
9450
|
+
const [ok, err, res] = await try_fn_default(() => this.replicate(
|
|
9451
|
+
resourceName,
|
|
9452
|
+
record.operation,
|
|
9453
|
+
record.data,
|
|
9454
|
+
record.id,
|
|
9455
|
+
record.beforeData
|
|
9456
|
+
));
|
|
9457
|
+
if (ok) results.push(res);
|
|
9458
|
+
else errors.push({ id: record.id, error: err.message });
|
|
7912
9459
|
}
|
|
9460
|
+
return {
|
|
9461
|
+
success: errors.length === 0,
|
|
9462
|
+
results,
|
|
9463
|
+
errors
|
|
9464
|
+
};
|
|
9465
|
+
}
|
|
9466
|
+
async testConnection() {
|
|
9467
|
+
const [ok, err] = await try_fn_default(async () => {
|
|
9468
|
+
if (!this.client) await this.initialize();
|
|
9469
|
+
await this.client.query("SELECT 1");
|
|
9470
|
+
return true;
|
|
9471
|
+
});
|
|
9472
|
+
if (ok) return true;
|
|
9473
|
+
this.emit("connection_error", { replicator: this.name, error: err.message });
|
|
9474
|
+
return false;
|
|
9475
|
+
}
|
|
9476
|
+
async cleanup() {
|
|
9477
|
+
if (this.client) await this.client.end();
|
|
9478
|
+
}
|
|
9479
|
+
getStatus() {
|
|
9480
|
+
return {
|
|
9481
|
+
...super.getStatus(),
|
|
9482
|
+
database: this.database || "postgres",
|
|
9483
|
+
resources: this.resources,
|
|
9484
|
+
logTable: this.logTable
|
|
9485
|
+
};
|
|
7913
9486
|
}
|
|
7914
9487
|
}
|
|
9488
|
+
var postgres_replicator_class_default = PostgresReplicator;
|
|
7915
9489
|
|
|
7916
9490
|
const S3_DEFAULT_REGION = "us-east-1";
|
|
7917
9491
|
const S3_DEFAULT_ENDPOINT = "https://s3.us-east-1.amazonaws.com";
|
|
@@ -9995,7 +11569,7 @@ class Resource extends EventEmitter {
|
|
|
9995
11569
|
if (okParse) contentType = "application/json";
|
|
9996
11570
|
}
|
|
9997
11571
|
if (this.behavior === "body-only" && (!body || body === "")) {
|
|
9998
|
-
throw new Error(`[Resource.insert]
|
|
11572
|
+
throw new Error(`[Resource.insert] Attempt to save object without body! Data: id=${finalId}, resource=${this.name}`);
|
|
9999
11573
|
}
|
|
10000
11574
|
const [okPut, errPut, putResult] = await try_fn_default(() => this.client.putObject({
|
|
10001
11575
|
key,
|
|
@@ -11342,8 +12916,8 @@ class Resource extends EventEmitter {
|
|
|
11342
12916
|
return mappedData;
|
|
11343
12917
|
}
|
|
11344
12918
|
/**
|
|
11345
|
-
* Compose the full object (metadata + body) as
|
|
11346
|
-
*
|
|
12919
|
+
* Compose the full object (metadata + body) as returned by .get(),
|
|
12920
|
+
* using in-memory data after insert/update, according to behavior
|
|
11347
12921
|
*/
|
|
11348
12922
|
async composeFullObjectFromWrite({ id, metadata, body, behavior }) {
|
|
11349
12923
|
const behaviorFlags = {};
|
|
@@ -11498,7 +13072,7 @@ class Resource extends EventEmitter {
|
|
|
11498
13072
|
if (!this._middlewares.has(method)) throw new ResourceError(`No such method for middleware: ${method}`, { operation: "useMiddleware", method });
|
|
11499
13073
|
this._middlewares.get(method).push(fn);
|
|
11500
13074
|
}
|
|
11501
|
-
//
|
|
13075
|
+
// Utility to apply schema default values
|
|
11502
13076
|
applyDefaults(data) {
|
|
11503
13077
|
const out = { ...data };
|
|
11504
13078
|
for (const [key, def] of Object.entries(this.attributes)) {
|
|
@@ -11630,7 +13204,7 @@ class Database extends EventEmitter {
|
|
|
11630
13204
|
super();
|
|
11631
13205
|
this.version = "1";
|
|
11632
13206
|
this.s3dbVersion = (() => {
|
|
11633
|
-
const [ok, err, version] = try_fn_default(() => true ? "7.
|
|
13207
|
+
const [ok, err, version] = try_fn_default(() => true ? "7.3.3" : "latest");
|
|
11634
13208
|
return ok ? version : "latest";
|
|
11635
13209
|
})();
|
|
11636
13210
|
this.resources = {};
|
|
@@ -11703,7 +13277,7 @@ class Database extends EventEmitter {
|
|
|
11703
13277
|
name,
|
|
11704
13278
|
client: this.client,
|
|
11705
13279
|
database: this,
|
|
11706
|
-
//
|
|
13280
|
+
// ensure reference
|
|
11707
13281
|
version: currentVersion,
|
|
11708
13282
|
attributes: versionData.attributes,
|
|
11709
13283
|
behavior: versionData.behavior || "user-managed",
|
|
@@ -12105,6 +13679,570 @@ class Database extends EventEmitter {
|
|
|
12105
13679
|
class S3db extends Database {
|
|
12106
13680
|
}
|
|
12107
13681
|
|
|
13682
|
+
function normalizeResourceName$1(name) {
|
|
13683
|
+
return typeof name === "string" ? name.trim().toLowerCase() : name;
|
|
13684
|
+
}
|
|
13685
|
+
class S3dbReplicator extends base_replicator_class_default {
|
|
13686
|
+
constructor(config = {}, resources = [], client = null) {
|
|
13687
|
+
super(config);
|
|
13688
|
+
this.instanceId = Math.random().toString(36).slice(2, 10);
|
|
13689
|
+
this.client = client;
|
|
13690
|
+
this.connectionString = config.connectionString;
|
|
13691
|
+
let normalizedResources = resources;
|
|
13692
|
+
if (!resources) normalizedResources = {};
|
|
13693
|
+
else if (Array.isArray(resources)) {
|
|
13694
|
+
normalizedResources = {};
|
|
13695
|
+
for (const res of resources) {
|
|
13696
|
+
if (typeof res === "string") normalizedResources[normalizeResourceName$1(res)] = res;
|
|
13697
|
+
}
|
|
13698
|
+
} else if (typeof resources === "string") {
|
|
13699
|
+
normalizedResources[normalizeResourceName$1(resources)] = resources;
|
|
13700
|
+
}
|
|
13701
|
+
this.resourcesMap = this._normalizeResources(normalizedResources);
|
|
13702
|
+
}
|
|
13703
|
+
_normalizeResources(resources) {
|
|
13704
|
+
if (!resources) return {};
|
|
13705
|
+
if (Array.isArray(resources)) {
|
|
13706
|
+
const map = {};
|
|
13707
|
+
for (const res of resources) {
|
|
13708
|
+
if (typeof res === "string") map[normalizeResourceName$1(res)] = res;
|
|
13709
|
+
else if (Array.isArray(res) && typeof res[0] === "string") map[normalizeResourceName$1(res[0])] = res;
|
|
13710
|
+
else if (typeof res === "object" && res.resource) {
|
|
13711
|
+
map[normalizeResourceName$1(res.resource)] = { ...res };
|
|
13712
|
+
}
|
|
13713
|
+
}
|
|
13714
|
+
return map;
|
|
13715
|
+
}
|
|
13716
|
+
if (typeof resources === "object") {
|
|
13717
|
+
const map = {};
|
|
13718
|
+
for (const [src, dest] of Object.entries(resources)) {
|
|
13719
|
+
const normSrc = normalizeResourceName$1(src);
|
|
13720
|
+
if (typeof dest === "string") map[normSrc] = dest;
|
|
13721
|
+
else if (Array.isArray(dest)) {
|
|
13722
|
+
map[normSrc] = dest.map((item) => {
|
|
13723
|
+
if (typeof item === "string") return item;
|
|
13724
|
+
if (typeof item === "function") return item;
|
|
13725
|
+
if (typeof item === "object" && item.resource) {
|
|
13726
|
+
return { ...item };
|
|
13727
|
+
}
|
|
13728
|
+
return item;
|
|
13729
|
+
});
|
|
13730
|
+
} else if (typeof dest === "function") map[normSrc] = dest;
|
|
13731
|
+
else if (typeof dest === "object" && dest.resource) {
|
|
13732
|
+
map[normSrc] = { ...dest };
|
|
13733
|
+
}
|
|
13734
|
+
}
|
|
13735
|
+
return map;
|
|
13736
|
+
}
|
|
13737
|
+
if (typeof resources === "function") {
|
|
13738
|
+
return resources;
|
|
13739
|
+
}
|
|
13740
|
+
if (typeof resources === "string") {
|
|
13741
|
+
const map = { [normalizeResourceName$1(resources)]: resources };
|
|
13742
|
+
return map;
|
|
13743
|
+
}
|
|
13744
|
+
return {};
|
|
13745
|
+
}
|
|
13746
|
+
validateConfig() {
|
|
13747
|
+
const errors = [];
|
|
13748
|
+
if (!this.client && !this.connectionString) {
|
|
13749
|
+
errors.push("You must provide a client or a connectionString");
|
|
13750
|
+
}
|
|
13751
|
+
if (!this.resourcesMap || typeof this.resourcesMap === "object" && Object.keys(this.resourcesMap).length === 0) {
|
|
13752
|
+
errors.push("You must provide a resources map or array");
|
|
13753
|
+
}
|
|
13754
|
+
return { isValid: errors.length === 0, errors };
|
|
13755
|
+
}
|
|
13756
|
+
async initialize(database) {
|
|
13757
|
+
try {
|
|
13758
|
+
await super.initialize(database);
|
|
13759
|
+
if (this.client) {
|
|
13760
|
+
this.targetDatabase = this.client;
|
|
13761
|
+
} else if (this.connectionString) {
|
|
13762
|
+
const targetConfig = {
|
|
13763
|
+
connectionString: this.connectionString,
|
|
13764
|
+
region: this.region,
|
|
13765
|
+
keyPrefix: this.keyPrefix,
|
|
13766
|
+
verbose: this.config.verbose || false
|
|
13767
|
+
};
|
|
13768
|
+
this.targetDatabase = new S3db(targetConfig);
|
|
13769
|
+
await this.targetDatabase.connect();
|
|
13770
|
+
} else {
|
|
13771
|
+
throw new Error("S3dbReplicator: No client or connectionString provided");
|
|
13772
|
+
}
|
|
13773
|
+
this.emit("connected", {
|
|
13774
|
+
replicator: this.name,
|
|
13775
|
+
target: this.connectionString || "client-provided"
|
|
13776
|
+
});
|
|
13777
|
+
} catch (err) {
|
|
13778
|
+
throw err;
|
|
13779
|
+
}
|
|
13780
|
+
}
|
|
13781
|
+
// Support both object and parameter signatures for flexibility
|
|
13782
|
+
async replicate(resourceOrObj, operation, data, recordId, beforeData) {
|
|
13783
|
+
let resource, op, payload, id;
|
|
13784
|
+
if (typeof resourceOrObj === "object" && resourceOrObj.resource) {
|
|
13785
|
+
resource = resourceOrObj.resource;
|
|
13786
|
+
op = resourceOrObj.operation;
|
|
13787
|
+
payload = resourceOrObj.data;
|
|
13788
|
+
id = resourceOrObj.id;
|
|
13789
|
+
} else {
|
|
13790
|
+
resource = resourceOrObj;
|
|
13791
|
+
op = operation;
|
|
13792
|
+
payload = data;
|
|
13793
|
+
id = recordId;
|
|
13794
|
+
}
|
|
13795
|
+
const normResource = normalizeResourceName$1(resource);
|
|
13796
|
+
const destResource = this._resolveDestResource(normResource, payload);
|
|
13797
|
+
const destResourceObj = this._getDestResourceObj(destResource);
|
|
13798
|
+
const transformedData = this._applyTransformer(normResource, payload);
|
|
13799
|
+
let result;
|
|
13800
|
+
if (op === "insert") {
|
|
13801
|
+
result = await destResourceObj.insert(transformedData);
|
|
13802
|
+
} else if (op === "update") {
|
|
13803
|
+
result = await destResourceObj.update(id, transformedData);
|
|
13804
|
+
} else if (op === "delete") {
|
|
13805
|
+
result = await destResourceObj.delete(id);
|
|
13806
|
+
} else {
|
|
13807
|
+
throw new Error(`Invalid operation: ${op}. Supported operations are: insert, update, delete`);
|
|
13808
|
+
}
|
|
13809
|
+
return result;
|
|
13810
|
+
}
|
|
13811
|
+
_applyTransformer(resource, data) {
|
|
13812
|
+
const normResource = normalizeResourceName$1(resource);
|
|
13813
|
+
const entry = this.resourcesMap[normResource];
|
|
13814
|
+
let result;
|
|
13815
|
+
if (!entry) return data;
|
|
13816
|
+
if (Array.isArray(entry) && typeof entry[1] === "function") {
|
|
13817
|
+
result = entry[1](data);
|
|
13818
|
+
} else if (typeof entry === "function") {
|
|
13819
|
+
result = entry(data);
|
|
13820
|
+
} else if (typeof entry === "object") {
|
|
13821
|
+
if (typeof entry.transform === "function") result = entry.transform(data);
|
|
13822
|
+
else if (typeof entry.transformer === "function") result = entry.transformer(data);
|
|
13823
|
+
} else {
|
|
13824
|
+
result = data;
|
|
13825
|
+
}
|
|
13826
|
+
if (result && data && data.id && !result.id) result.id = data.id;
|
|
13827
|
+
if (!result && data) result = data;
|
|
13828
|
+
return result;
|
|
13829
|
+
}
|
|
13830
|
+
_resolveDestResource(resource, data) {
|
|
13831
|
+
const normResource = normalizeResourceName$1(resource);
|
|
13832
|
+
const entry = this.resourcesMap[normResource];
|
|
13833
|
+
if (!entry) return resource;
|
|
13834
|
+
if (Array.isArray(entry)) {
|
|
13835
|
+
if (typeof entry[0] === "string") return entry[0];
|
|
13836
|
+
if (typeof entry[0] === "object" && entry[0].resource) return entry[0].resource;
|
|
13837
|
+
if (typeof entry[0] === "function") return resource;
|
|
13838
|
+
}
|
|
13839
|
+
if (typeof entry === "string") return entry;
|
|
13840
|
+
if (resource && !targetResourceName) targetResourceName = resource;
|
|
13841
|
+
if (typeof entry === "object" && entry.resource) return entry.resource;
|
|
13842
|
+
return resource;
|
|
13843
|
+
}
|
|
13844
|
+
_getDestResourceObj(resource) {
|
|
13845
|
+
if (!this.client || !this.client.resources) return null;
|
|
13846
|
+
const available = Object.keys(this.client.resources);
|
|
13847
|
+
const norm = normalizeResourceName$1(resource);
|
|
13848
|
+
const found = available.find((r) => normalizeResourceName$1(r) === norm);
|
|
13849
|
+
if (!found) {
|
|
13850
|
+
throw new Error(`[S3dbReplicator] Destination resource not found: ${resource}. Available: ${available.join(", ")}`);
|
|
13851
|
+
}
|
|
13852
|
+
return this.client.resources[found];
|
|
13853
|
+
}
|
|
13854
|
+
async replicateBatch(resourceName, records) {
|
|
13855
|
+
if (!this.enabled || !this.shouldReplicateResource(resourceName)) {
|
|
13856
|
+
return { skipped: true, reason: "resource_not_included" };
|
|
13857
|
+
}
|
|
13858
|
+
const results = [];
|
|
13859
|
+
const errors = [];
|
|
13860
|
+
for (const record of records) {
|
|
13861
|
+
const [ok, err, result] = await try_fn_default(() => this.replicate({
|
|
13862
|
+
resource: resourceName,
|
|
13863
|
+
operation: record.operation,
|
|
13864
|
+
id: record.id,
|
|
13865
|
+
data: record.data,
|
|
13866
|
+
beforeData: record.beforeData
|
|
13867
|
+
}));
|
|
13868
|
+
if (ok) results.push(result);
|
|
13869
|
+
else errors.push({ id: record.id, error: err.message });
|
|
13870
|
+
}
|
|
13871
|
+
this.emit("batch_replicated", {
|
|
13872
|
+
replicator: this.name,
|
|
13873
|
+
resourceName,
|
|
13874
|
+
total: records.length,
|
|
13875
|
+
successful: results.length,
|
|
13876
|
+
errors: errors.length
|
|
13877
|
+
});
|
|
13878
|
+
return {
|
|
13879
|
+
success: errors.length === 0,
|
|
13880
|
+
results,
|
|
13881
|
+
errors,
|
|
13882
|
+
total: records.length
|
|
13883
|
+
};
|
|
13884
|
+
}
|
|
13885
|
+
async testConnection() {
|
|
13886
|
+
const [ok, err] = await try_fn_default(async () => {
|
|
13887
|
+
if (!this.targetDatabase) {
|
|
13888
|
+
await this.initialize(this.database);
|
|
13889
|
+
}
|
|
13890
|
+
await this.targetDatabase.listResources();
|
|
13891
|
+
return true;
|
|
13892
|
+
});
|
|
13893
|
+
if (ok) return true;
|
|
13894
|
+
this.emit("connection_error", {
|
|
13895
|
+
replicator: this.name,
|
|
13896
|
+
error: err.message
|
|
13897
|
+
});
|
|
13898
|
+
return false;
|
|
13899
|
+
}
|
|
13900
|
+
async getStatus() {
|
|
13901
|
+
const baseStatus = await super.getStatus();
|
|
13902
|
+
return {
|
|
13903
|
+
...baseStatus,
|
|
13904
|
+
connected: !!this.targetDatabase,
|
|
13905
|
+
targetDatabase: this.connectionString || "client-provided",
|
|
13906
|
+
resources: Object.keys(this.resourcesMap || {}),
|
|
13907
|
+
totalreplicators: this.listenerCount("replicated"),
|
|
13908
|
+
totalErrors: this.listenerCount("replicator_error")
|
|
13909
|
+
};
|
|
13910
|
+
}
|
|
13911
|
+
async cleanup() {
|
|
13912
|
+
if (this.targetDatabase) {
|
|
13913
|
+
this.targetDatabase.removeAllListeners();
|
|
13914
|
+
}
|
|
13915
|
+
await super.cleanup();
|
|
13916
|
+
}
|
|
13917
|
+
shouldReplicateResource(resource, action) {
|
|
13918
|
+
const normResource = normalizeResourceName$1(resource);
|
|
13919
|
+
const entry = this.resourcesMap[normResource];
|
|
13920
|
+
if (!entry) return false;
|
|
13921
|
+
if (!action) return true;
|
|
13922
|
+
if (Array.isArray(entry)) {
|
|
13923
|
+
for (const item of entry) {
|
|
13924
|
+
if (typeof item === "object" && item.resource) {
|
|
13925
|
+
if (item.actions && Array.isArray(item.actions)) {
|
|
13926
|
+
if (item.actions.includes(action)) return true;
|
|
13927
|
+
} else {
|
|
13928
|
+
return true;
|
|
13929
|
+
}
|
|
13930
|
+
} else if (typeof item === "string" || typeof item === "function") {
|
|
13931
|
+
return true;
|
|
13932
|
+
}
|
|
13933
|
+
}
|
|
13934
|
+
return false;
|
|
13935
|
+
}
|
|
13936
|
+
if (typeof entry === "object" && entry.resource) {
|
|
13937
|
+
if (entry.actions && Array.isArray(entry.actions)) {
|
|
13938
|
+
return entry.actions.includes(action);
|
|
13939
|
+
}
|
|
13940
|
+
return true;
|
|
13941
|
+
}
|
|
13942
|
+
if (typeof entry === "string" || typeof entry === "function") {
|
|
13943
|
+
return true;
|
|
13944
|
+
}
|
|
13945
|
+
return false;
|
|
13946
|
+
}
|
|
13947
|
+
}
|
|
13948
|
+
var s3db_replicator_class_default = S3dbReplicator;
|
|
13949
|
+
|
|
13950
|
+
class SqsReplicator extends base_replicator_class_default {
|
|
13951
|
+
constructor(config = {}, resources = [], client = null) {
|
|
13952
|
+
super(config);
|
|
13953
|
+
this.client = client;
|
|
13954
|
+
this.queueUrl = config.queueUrl;
|
|
13955
|
+
this.queues = config.queues || {};
|
|
13956
|
+
this.defaultQueue = config.defaultQueue || config.defaultQueueUrl || config.queueUrlDefault;
|
|
13957
|
+
this.region = config.region || "us-east-1";
|
|
13958
|
+
this.sqsClient = client || null;
|
|
13959
|
+
this.messageGroupId = config.messageGroupId;
|
|
13960
|
+
this.deduplicationId = config.deduplicationId;
|
|
13961
|
+
if (Array.isArray(resources)) {
|
|
13962
|
+
this.resources = {};
|
|
13963
|
+
for (const resource of resources) {
|
|
13964
|
+
if (typeof resource === "string") {
|
|
13965
|
+
this.resources[resource] = true;
|
|
13966
|
+
} else if (typeof resource === "object" && resource.name) {
|
|
13967
|
+
this.resources[resource.name] = resource;
|
|
13968
|
+
}
|
|
13969
|
+
}
|
|
13970
|
+
} else if (typeof resources === "object") {
|
|
13971
|
+
this.resources = resources;
|
|
13972
|
+
for (const [resourceName, resourceConfig] of Object.entries(resources)) {
|
|
13973
|
+
if (resourceConfig && resourceConfig.queueUrl) {
|
|
13974
|
+
this.queues[resourceName] = resourceConfig.queueUrl;
|
|
13975
|
+
}
|
|
13976
|
+
}
|
|
13977
|
+
} else {
|
|
13978
|
+
this.resources = {};
|
|
13979
|
+
}
|
|
13980
|
+
}
|
|
13981
|
+
validateConfig() {
|
|
13982
|
+
const errors = [];
|
|
13983
|
+
if (!this.queueUrl && Object.keys(this.queues).length === 0 && !this.defaultQueue && !this.resourceQueueMap) {
|
|
13984
|
+
errors.push("Either queueUrl, queues object, defaultQueue, or resourceQueueMap must be provided");
|
|
13985
|
+
}
|
|
13986
|
+
return {
|
|
13987
|
+
isValid: errors.length === 0,
|
|
13988
|
+
errors
|
|
13989
|
+
};
|
|
13990
|
+
}
|
|
13991
|
+
getQueueUrlsForResource(resource) {
|
|
13992
|
+
if (this.resourceQueueMap && this.resourceQueueMap[resource]) {
|
|
13993
|
+
return this.resourceQueueMap[resource];
|
|
13994
|
+
}
|
|
13995
|
+
if (this.queues[resource]) {
|
|
13996
|
+
return [this.queues[resource]];
|
|
13997
|
+
}
|
|
13998
|
+
if (this.queueUrl) {
|
|
13999
|
+
return [this.queueUrl];
|
|
14000
|
+
}
|
|
14001
|
+
if (this.defaultQueue) {
|
|
14002
|
+
return [this.defaultQueue];
|
|
14003
|
+
}
|
|
14004
|
+
throw new Error(`No queue URL found for resource '${resource}'`);
|
|
14005
|
+
}
|
|
14006
|
+
_applyTransformer(resource, data) {
|
|
14007
|
+
const entry = this.resources[resource];
|
|
14008
|
+
let result = data;
|
|
14009
|
+
if (!entry) return data;
|
|
14010
|
+
if (typeof entry.transform === "function") {
|
|
14011
|
+
result = entry.transform(data);
|
|
14012
|
+
} else if (typeof entry.transformer === "function") {
|
|
14013
|
+
result = entry.transformer(data);
|
|
14014
|
+
}
|
|
14015
|
+
return result || data;
|
|
14016
|
+
}
|
|
14017
|
+
/**
|
|
14018
|
+
* Create standardized message structure
|
|
14019
|
+
*/
|
|
14020
|
+
createMessage(resource, operation, data, id, beforeData = null) {
|
|
14021
|
+
const baseMessage = {
|
|
14022
|
+
resource,
|
|
14023
|
+
// padronizado para 'resource'
|
|
14024
|
+
action: operation,
|
|
14025
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
14026
|
+
source: "s3db-replicator"
|
|
14027
|
+
};
|
|
14028
|
+
switch (operation) {
|
|
14029
|
+
case "insert":
|
|
14030
|
+
return {
|
|
14031
|
+
...baseMessage,
|
|
14032
|
+
data
|
|
14033
|
+
};
|
|
14034
|
+
case "update":
|
|
14035
|
+
return {
|
|
14036
|
+
...baseMessage,
|
|
14037
|
+
before: beforeData,
|
|
14038
|
+
data
|
|
14039
|
+
};
|
|
14040
|
+
case "delete":
|
|
14041
|
+
return {
|
|
14042
|
+
...baseMessage,
|
|
14043
|
+
data
|
|
14044
|
+
};
|
|
14045
|
+
default:
|
|
14046
|
+
return {
|
|
14047
|
+
...baseMessage,
|
|
14048
|
+
data
|
|
14049
|
+
};
|
|
14050
|
+
}
|
|
14051
|
+
}
|
|
14052
|
+
async initialize(database, client) {
|
|
14053
|
+
await super.initialize(database);
|
|
14054
|
+
if (!this.sqsClient) {
|
|
14055
|
+
const [ok, err, sdk] = await try_fn_default(() => import('@aws-sdk/client-sqs'));
|
|
14056
|
+
if (!ok) {
|
|
14057
|
+
this.emit("initialization_error", {
|
|
14058
|
+
replicator: this.name,
|
|
14059
|
+
error: err.message
|
|
14060
|
+
});
|
|
14061
|
+
throw err;
|
|
14062
|
+
}
|
|
14063
|
+
const { SQSClient } = sdk;
|
|
14064
|
+
this.sqsClient = client || new SQSClient({
|
|
14065
|
+
region: this.region,
|
|
14066
|
+
credentials: this.config.credentials
|
|
14067
|
+
});
|
|
14068
|
+
this.emit("initialized", {
|
|
14069
|
+
replicator: this.name,
|
|
14070
|
+
queueUrl: this.queueUrl,
|
|
14071
|
+
queues: this.queues,
|
|
14072
|
+
defaultQueue: this.defaultQueue
|
|
14073
|
+
});
|
|
14074
|
+
}
|
|
14075
|
+
}
|
|
14076
|
+
async replicate(resource, operation, data, id, beforeData = null) {
|
|
14077
|
+
if (!this.enabled || !this.shouldReplicateResource(resource)) {
|
|
14078
|
+
return { skipped: true, reason: "resource_not_included" };
|
|
14079
|
+
}
|
|
14080
|
+
const [ok, err, result] = await try_fn_default(async () => {
|
|
14081
|
+
const { SendMessageCommand } = await import('@aws-sdk/client-sqs');
|
|
14082
|
+
const queueUrls = this.getQueueUrlsForResource(resource);
|
|
14083
|
+
const transformedData = this._applyTransformer(resource, data);
|
|
14084
|
+
const message = this.createMessage(resource, operation, transformedData, id, beforeData);
|
|
14085
|
+
const results = [];
|
|
14086
|
+
for (const queueUrl of queueUrls) {
|
|
14087
|
+
const command = new SendMessageCommand({
|
|
14088
|
+
QueueUrl: queueUrl,
|
|
14089
|
+
MessageBody: JSON.stringify(message),
|
|
14090
|
+
MessageGroupId: this.messageGroupId,
|
|
14091
|
+
MessageDeduplicationId: this.deduplicationId ? `${resource}:${operation}:${id}` : void 0
|
|
14092
|
+
});
|
|
14093
|
+
const result2 = await this.sqsClient.send(command);
|
|
14094
|
+
results.push({ queueUrl, messageId: result2.MessageId });
|
|
14095
|
+
this.emit("replicated", {
|
|
14096
|
+
replicator: this.name,
|
|
14097
|
+
resource,
|
|
14098
|
+
operation,
|
|
14099
|
+
id,
|
|
14100
|
+
queueUrl,
|
|
14101
|
+
messageId: result2.MessageId,
|
|
14102
|
+
success: true
|
|
14103
|
+
});
|
|
14104
|
+
}
|
|
14105
|
+
return { success: true, results };
|
|
14106
|
+
});
|
|
14107
|
+
if (ok) return result;
|
|
14108
|
+
this.emit("replicator_error", {
|
|
14109
|
+
replicator: this.name,
|
|
14110
|
+
resource,
|
|
14111
|
+
operation,
|
|
14112
|
+
id,
|
|
14113
|
+
error: err.message
|
|
14114
|
+
});
|
|
14115
|
+
return { success: false, error: err.message };
|
|
14116
|
+
}
|
|
14117
|
+
async replicateBatch(resource, records) {
|
|
14118
|
+
if (!this.enabled || !this.shouldReplicateResource(resource)) {
|
|
14119
|
+
return { skipped: true, reason: "resource_not_included" };
|
|
14120
|
+
}
|
|
14121
|
+
const [ok, err, result] = await try_fn_default(async () => {
|
|
14122
|
+
const { SendMessageBatchCommand } = await import('@aws-sdk/client-sqs');
|
|
14123
|
+
const queueUrls = this.getQueueUrlsForResource(resource);
|
|
14124
|
+
const batchSize = 10;
|
|
14125
|
+
const batches = [];
|
|
14126
|
+
for (let i = 0; i < records.length; i += batchSize) {
|
|
14127
|
+
batches.push(records.slice(i, i + batchSize));
|
|
14128
|
+
}
|
|
14129
|
+
const results = [];
|
|
14130
|
+
const errors = [];
|
|
14131
|
+
for (const batch of batches) {
|
|
14132
|
+
const [okBatch, errBatch] = await try_fn_default(async () => {
|
|
14133
|
+
const entries = batch.map((record, index) => ({
|
|
14134
|
+
Id: `${record.id}-${index}`,
|
|
14135
|
+
MessageBody: JSON.stringify(this.createMessage(
|
|
14136
|
+
resource,
|
|
14137
|
+
record.operation,
|
|
14138
|
+
record.data,
|
|
14139
|
+
record.id,
|
|
14140
|
+
record.beforeData
|
|
14141
|
+
)),
|
|
14142
|
+
MessageGroupId: this.messageGroupId,
|
|
14143
|
+
MessageDeduplicationId: this.deduplicationId ? `${resource}:${record.operation}:${record.id}` : void 0
|
|
14144
|
+
}));
|
|
14145
|
+
const command = new SendMessageBatchCommand({
|
|
14146
|
+
QueueUrl: queueUrls[0],
|
|
14147
|
+
// Assuming all queueUrls in a batch are the same for batching
|
|
14148
|
+
Entries: entries
|
|
14149
|
+
});
|
|
14150
|
+
const result2 = await this.sqsClient.send(command);
|
|
14151
|
+
results.push(result2);
|
|
14152
|
+
});
|
|
14153
|
+
if (!okBatch) {
|
|
14154
|
+
errors.push({ batch: batch.length, error: errBatch.message });
|
|
14155
|
+
if (errBatch.message && (errBatch.message.includes("Batch error") || errBatch.message.includes("Connection") || errBatch.message.includes("Network"))) {
|
|
14156
|
+
throw errBatch;
|
|
14157
|
+
}
|
|
14158
|
+
}
|
|
14159
|
+
}
|
|
14160
|
+
this.emit("batch_replicated", {
|
|
14161
|
+
replicator: this.name,
|
|
14162
|
+
resource,
|
|
14163
|
+
queueUrl: queueUrls[0],
|
|
14164
|
+
// Assuming all queueUrls in a batch are the same for batching
|
|
14165
|
+
total: records.length,
|
|
14166
|
+
successful: results.length,
|
|
14167
|
+
errors: errors.length
|
|
14168
|
+
});
|
|
14169
|
+
return {
|
|
14170
|
+
success: errors.length === 0,
|
|
14171
|
+
results,
|
|
14172
|
+
errors,
|
|
14173
|
+
total: records.length,
|
|
14174
|
+
queueUrl: queueUrls[0]
|
|
14175
|
+
// Assuming all queueUrls in a batch are the same for batching
|
|
14176
|
+
};
|
|
14177
|
+
});
|
|
14178
|
+
if (ok) return result;
|
|
14179
|
+
const errorMessage = err?.message || err || "Unknown error";
|
|
14180
|
+
this.emit("batch_replicator_error", {
|
|
14181
|
+
replicator: this.name,
|
|
14182
|
+
resource,
|
|
14183
|
+
error: errorMessage
|
|
14184
|
+
});
|
|
14185
|
+
return { success: false, error: errorMessage };
|
|
14186
|
+
}
|
|
14187
|
+
async testConnection() {
|
|
14188
|
+
const [ok, err] = await try_fn_default(async () => {
|
|
14189
|
+
if (!this.sqsClient) {
|
|
14190
|
+
await this.initialize(this.database);
|
|
14191
|
+
}
|
|
14192
|
+
const { GetQueueAttributesCommand } = await import('@aws-sdk/client-sqs');
|
|
14193
|
+
const command = new GetQueueAttributesCommand({
|
|
14194
|
+
QueueUrl: this.queueUrl,
|
|
14195
|
+
AttributeNames: ["QueueArn"]
|
|
14196
|
+
});
|
|
14197
|
+
await this.sqsClient.send(command);
|
|
14198
|
+
return true;
|
|
14199
|
+
});
|
|
14200
|
+
if (ok) return true;
|
|
14201
|
+
this.emit("connection_error", {
|
|
14202
|
+
replicator: this.name,
|
|
14203
|
+
error: err.message
|
|
14204
|
+
});
|
|
14205
|
+
return false;
|
|
14206
|
+
}
|
|
14207
|
+
async getStatus() {
|
|
14208
|
+
const baseStatus = await super.getStatus();
|
|
14209
|
+
return {
|
|
14210
|
+
...baseStatus,
|
|
14211
|
+
connected: !!this.sqsClient,
|
|
14212
|
+
queueUrl: this.queueUrl,
|
|
14213
|
+
region: this.region,
|
|
14214
|
+
resources: Object.keys(this.resources || {}),
|
|
14215
|
+
totalreplicators: this.listenerCount("replicated"),
|
|
14216
|
+
totalErrors: this.listenerCount("replicator_error")
|
|
14217
|
+
};
|
|
14218
|
+
}
|
|
14219
|
+
async cleanup() {
|
|
14220
|
+
if (this.sqsClient) {
|
|
14221
|
+
this.sqsClient.destroy();
|
|
14222
|
+
}
|
|
14223
|
+
await super.cleanup();
|
|
14224
|
+
}
|
|
14225
|
+
shouldReplicateResource(resource) {
|
|
14226
|
+
const result = this.resourceQueueMap && Object.keys(this.resourceQueueMap).includes(resource) || this.queues && Object.keys(this.queues).includes(resource) || !!(this.defaultQueue || this.queueUrl) || this.resources && Object.keys(this.resources).includes(resource) || false;
|
|
14227
|
+
return result;
|
|
14228
|
+
}
|
|
14229
|
+
}
|
|
14230
|
+
var sqs_replicator_class_default = SqsReplicator;
|
|
14231
|
+
|
|
14232
|
+
const REPLICATOR_DRIVERS = {
|
|
14233
|
+
s3db: s3db_replicator_class_default,
|
|
14234
|
+
sqs: sqs_replicator_class_default,
|
|
14235
|
+
bigquery: bigquery_replicator_class_default,
|
|
14236
|
+
postgres: postgres_replicator_class_default
|
|
14237
|
+
};
|
|
14238
|
+
function createReplicator(driver, config = {}, resources = [], client = null) {
|
|
14239
|
+
const ReplicatorClass = REPLICATOR_DRIVERS[driver];
|
|
14240
|
+
if (!ReplicatorClass) {
|
|
14241
|
+
throw new Error(`Unknown replicator driver: ${driver}. Available drivers: ${Object.keys(REPLICATOR_DRIVERS).join(", ")}`);
|
|
14242
|
+
}
|
|
14243
|
+
return new ReplicatorClass(config, resources, client);
|
|
14244
|
+
}
|
|
14245
|
+
|
|
12108
14246
|
function normalizeResourceName(name) {
|
|
12109
14247
|
return typeof name === "string" ? name.trim().toLowerCase() : name;
|
|
12110
14248
|
}
|
|
@@ -12152,7 +14290,7 @@ class ReplicatorPlugin extends plugin_class_default {
|
|
|
12152
14290
|
return filtered;
|
|
12153
14291
|
}
|
|
12154
14292
|
installEventListeners(resource, database, plugin) {
|
|
12155
|
-
if (!resource || this.eventListenersInstalled.has(resource.name)) {
|
|
14293
|
+
if (!resource || this.eventListenersInstalled.has(resource.name) || resource.name === this.config.replicatorLogResource) {
|
|
12156
14294
|
return;
|
|
12157
14295
|
}
|
|
12158
14296
|
resource.on("insert", async (data) => {
|
|
@@ -12229,10 +14367,15 @@ class ReplicatorPlugin extends plugin_class_default {
|
|
|
12229
14367
|
this.installEventListeners(resource, database, this);
|
|
12230
14368
|
}
|
|
12231
14369
|
}
|
|
14370
|
+
createReplicator(driver, config, resources, client) {
|
|
14371
|
+
return createReplicator(driver, config, resources, client);
|
|
14372
|
+
}
|
|
12232
14373
|
async initializeReplicators(database) {
|
|
12233
14374
|
for (const replicatorConfig of this.config.replicators) {
|
|
12234
|
-
const { driver, config, resources } = replicatorConfig;
|
|
12235
|
-
const
|
|
14375
|
+
const { driver, config = {}, resources, client, ...otherConfig } = replicatorConfig;
|
|
14376
|
+
const replicatorResources = resources || config.resources || {};
|
|
14377
|
+
const mergedConfig = { ...config, ...otherConfig };
|
|
14378
|
+
const replicator = this.createReplicator(driver, mergedConfig, replicatorResources, client);
|
|
12236
14379
|
if (replicator) {
|
|
12237
14380
|
await replicator.initialize(database);
|
|
12238
14381
|
this.replicators.push(replicator);
|
|
@@ -12243,6 +14386,66 @@ class ReplicatorPlugin extends plugin_class_default {
|
|
|
12243
14386
|
}
|
|
12244
14387
|
async stop() {
|
|
12245
14388
|
}
|
|
14389
|
+
filterInternalFields(data) {
|
|
14390
|
+
if (!data || typeof data !== "object") return data;
|
|
14391
|
+
const filtered = {};
|
|
14392
|
+
for (const [key, value] of Object.entries(data)) {
|
|
14393
|
+
if (!key.startsWith("_") && !key.startsWith("$")) {
|
|
14394
|
+
filtered[key] = value;
|
|
14395
|
+
}
|
|
14396
|
+
}
|
|
14397
|
+
return filtered;
|
|
14398
|
+
}
|
|
14399
|
+
async uploadMetadataFile(database) {
|
|
14400
|
+
if (typeof database.uploadMetadataFile === "function") {
|
|
14401
|
+
await database.uploadMetadataFile();
|
|
14402
|
+
}
|
|
14403
|
+
}
|
|
14404
|
+
async getCompleteData(resource, data) {
|
|
14405
|
+
try {
|
|
14406
|
+
const [ok, err, record] = await try_fn_default(() => resource.get(data.id));
|
|
14407
|
+
if (ok && record) {
|
|
14408
|
+
return record;
|
|
14409
|
+
}
|
|
14410
|
+
} catch (error) {
|
|
14411
|
+
}
|
|
14412
|
+
return data;
|
|
14413
|
+
}
|
|
14414
|
+
async retryWithBackoff(operation, maxRetries = 3) {
|
|
14415
|
+
let lastError;
|
|
14416
|
+
for (let attempt = 1; attempt <= maxRetries; attempt++) {
|
|
14417
|
+
try {
|
|
14418
|
+
return await operation();
|
|
14419
|
+
} catch (error) {
|
|
14420
|
+
lastError = error;
|
|
14421
|
+
if (attempt === maxRetries) {
|
|
14422
|
+
throw error;
|
|
14423
|
+
}
|
|
14424
|
+
const delay = Math.pow(2, attempt - 1) * 1e3;
|
|
14425
|
+
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
14426
|
+
}
|
|
14427
|
+
}
|
|
14428
|
+
throw lastError;
|
|
14429
|
+
}
|
|
14430
|
+
async logError(replicator, resourceName, operation, recordId, data, error) {
|
|
14431
|
+
try {
|
|
14432
|
+
const logResourceName = this.config.replicatorLogResource;
|
|
14433
|
+
if (this.database && this.database.resources && this.database.resources[logResourceName]) {
|
|
14434
|
+
const logResource = this.database.resources[logResourceName];
|
|
14435
|
+
await logResource.insert({
|
|
14436
|
+
replicator: replicator.name || replicator.id,
|
|
14437
|
+
resourceName,
|
|
14438
|
+
operation,
|
|
14439
|
+
recordId,
|
|
14440
|
+
data: JSON.stringify(data),
|
|
14441
|
+
error: error.message,
|
|
14442
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
14443
|
+
status: "error"
|
|
14444
|
+
});
|
|
14445
|
+
}
|
|
14446
|
+
} catch (logError) {
|
|
14447
|
+
}
|
|
14448
|
+
}
|
|
12246
14449
|
async processReplicatorEvent(operation, resourceName, recordId, data, beforeData = null) {
|
|
12247
14450
|
if (!this.config.enabled) return;
|
|
12248
14451
|
const applicableReplicators = this.replicators.filter((replicator) => {
|
|
@@ -12373,7 +14576,7 @@ class ReplicatorPlugin extends plugin_class_default {
|
|
|
12373
14576
|
async getreplicatorStats() {
|
|
12374
14577
|
const replicatorStats = await Promise.all(
|
|
12375
14578
|
this.replicators.map(async (replicator) => {
|
|
12376
|
-
const status = await replicator.
|
|
14579
|
+
const status = await replicator.getStatus();
|
|
12377
14580
|
return {
|
|
12378
14581
|
id: replicator.id,
|
|
12379
14582
|
driver: replicator.driver,
|
|
@@ -12447,12 +14650,12 @@ class ReplicatorPlugin extends plugin_class_default {
|
|
|
12447
14650
|
this.stats.lastSync = (/* @__PURE__ */ new Date()).toISOString();
|
|
12448
14651
|
for (const resourceName in this.database.resources) {
|
|
12449
14652
|
if (normalizeResourceName(resourceName) === normalizeResourceName("replicator_logs")) continue;
|
|
12450
|
-
if (replicator.
|
|
14653
|
+
if (replicator.shouldReplicateResource(resourceName)) {
|
|
12451
14654
|
this.emit("replicator.sync.resource", { resourceName, replicatorId });
|
|
12452
14655
|
const resource = this.database.resources[resourceName];
|
|
12453
14656
|
const allRecords = await resource.getAll();
|
|
12454
14657
|
for (const record of allRecords) {
|
|
12455
|
-
await replicator.
|
|
14658
|
+
await replicator.replicate(resourceName, "insert", record, record.id);
|
|
12456
14659
|
}
|
|
12457
14660
|
}
|
|
12458
14661
|
}
|