s3db.js 7.2.1 → 7.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/s3db.cjs.js +1241 -378
- package/dist/s3db.cjs.min.js +1 -1
- package/dist/s3db.es.js +1242 -379
- package/dist/s3db.es.min.js +1 -1
- package/dist/s3db.iife.js +1240 -377
- package/dist/s3db.iife.min.js +1 -1
- package/package.json +30 -24
- package/src/behaviors/body-only.js +2 -2
- package/src/behaviors/truncate-data.js +2 -2
- package/src/client.class.js +1 -1
- package/src/database.class.js +1 -1
- package/src/errors.js +1 -1
- package/src/plugins/audit.plugin.js +5 -5
- package/src/plugins/cache/filesystem-cache.class.js +661 -0
- package/src/plugins/cache/index.js +4 -0
- package/src/plugins/cache/partition-aware-filesystem-cache.class.js +480 -0
- package/src/plugins/cache/s3-cache.class.js +1 -1
- package/src/plugins/cache.plugin.js +159 -9
- package/src/plugins/consumers/index.js +3 -3
- package/src/plugins/consumers/sqs-consumer.js +2 -2
- package/src/plugins/fulltext.plugin.js +5 -5
- package/src/plugins/metrics.plugin.js +2 -2
- package/src/plugins/queue-consumer.plugin.js +3 -3
- package/src/plugins/replicator.plugin.js +259 -362
- package/src/plugins/replicators/s3db-replicator.class.js +35 -19
- package/src/plugins/replicators/sqs-replicator.class.js +17 -5
- package/src/resource.class.js +14 -14
- package/src/schema.class.js +3 -3
package/dist/s3db.iife.js
CHANGED
|
@@ -1333,7 +1333,8 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
1333
1333
|
version: "2.0"
|
|
1334
1334
|
})
|
|
1335
1335
|
};
|
|
1336
|
-
this.logAudit(auditRecord).catch(
|
|
1336
|
+
this.logAudit(auditRecord).catch(() => {
|
|
1337
|
+
});
|
|
1337
1338
|
});
|
|
1338
1339
|
resource.on("update", async (data) => {
|
|
1339
1340
|
const recordId = data.id;
|
|
@@ -1359,7 +1360,8 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
1359
1360
|
version: "2.0"
|
|
1360
1361
|
})
|
|
1361
1362
|
};
|
|
1362
|
-
this.logAudit(auditRecord).catch(
|
|
1363
|
+
this.logAudit(auditRecord).catch(() => {
|
|
1364
|
+
});
|
|
1363
1365
|
});
|
|
1364
1366
|
resource.on("delete", async (data) => {
|
|
1365
1367
|
const recordId = data.id;
|
|
@@ -1385,7 +1387,8 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
1385
1387
|
version: "2.0"
|
|
1386
1388
|
})
|
|
1387
1389
|
};
|
|
1388
|
-
this.logAudit(auditRecord).catch(
|
|
1390
|
+
this.logAudit(auditRecord).catch(() => {
|
|
1391
|
+
});
|
|
1389
1392
|
});
|
|
1390
1393
|
resource.useMiddleware("deleteMany", async (ctx, next) => {
|
|
1391
1394
|
const ids = ctx.args[0];
|
|
@@ -1418,7 +1421,8 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
1418
1421
|
batchOperation: true
|
|
1419
1422
|
})
|
|
1420
1423
|
};
|
|
1421
|
-
this.logAudit(auditRecord).catch(
|
|
1424
|
+
this.logAudit(auditRecord).catch(() => {
|
|
1425
|
+
});
|
|
1422
1426
|
}
|
|
1423
1427
|
}
|
|
1424
1428
|
return result;
|
|
@@ -2065,6 +2069,16 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
2065
2069
|
// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
|
|
2066
2070
|
// USE OR OTHER DEALINGS IN THE SOFTWARE.
|
|
2067
2071
|
|
|
2072
|
+
var getOwnPropertyDescriptors = Object.getOwnPropertyDescriptors ||
|
|
2073
|
+
function getOwnPropertyDescriptors(obj) {
|
|
2074
|
+
var keys = Object.keys(obj);
|
|
2075
|
+
var descriptors = {};
|
|
2076
|
+
for (var i = 0; i < keys.length; i++) {
|
|
2077
|
+
descriptors[keys[i]] = Object.getOwnPropertyDescriptor(obj, keys[i]);
|
|
2078
|
+
}
|
|
2079
|
+
return descriptors;
|
|
2080
|
+
};
|
|
2081
|
+
|
|
2068
2082
|
var formatRegExp = /%[sdj%]/g;
|
|
2069
2083
|
function format(f) {
|
|
2070
2084
|
if (!isString(f)) {
|
|
@@ -2550,6 +2564,64 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
2550
2564
|
return Object.prototype.hasOwnProperty.call(obj, prop);
|
|
2551
2565
|
}
|
|
2552
2566
|
|
|
2567
|
+
var kCustomPromisifiedSymbol = typeof Symbol !== 'undefined' ? Symbol('util.promisify.custom') : undefined;
|
|
2568
|
+
|
|
2569
|
+
function promisify(original) {
|
|
2570
|
+
if (typeof original !== 'function')
|
|
2571
|
+
throw new TypeError('The "original" argument must be of type Function');
|
|
2572
|
+
|
|
2573
|
+
if (kCustomPromisifiedSymbol && original[kCustomPromisifiedSymbol]) {
|
|
2574
|
+
var fn = original[kCustomPromisifiedSymbol];
|
|
2575
|
+
if (typeof fn !== 'function') {
|
|
2576
|
+
throw new TypeError('The "util.promisify.custom" argument must be of type Function');
|
|
2577
|
+
}
|
|
2578
|
+
Object.defineProperty(fn, kCustomPromisifiedSymbol, {
|
|
2579
|
+
value: fn, enumerable: false, writable: false, configurable: true
|
|
2580
|
+
});
|
|
2581
|
+
return fn;
|
|
2582
|
+
}
|
|
2583
|
+
|
|
2584
|
+
function fn() {
|
|
2585
|
+
var promiseResolve, promiseReject;
|
|
2586
|
+
var promise = new Promise(function (resolve, reject) {
|
|
2587
|
+
promiseResolve = resolve;
|
|
2588
|
+
promiseReject = reject;
|
|
2589
|
+
});
|
|
2590
|
+
|
|
2591
|
+
var args = [];
|
|
2592
|
+
for (var i = 0; i < arguments.length; i++) {
|
|
2593
|
+
args.push(arguments[i]);
|
|
2594
|
+
}
|
|
2595
|
+
args.push(function (err, value) {
|
|
2596
|
+
if (err) {
|
|
2597
|
+
promiseReject(err);
|
|
2598
|
+
} else {
|
|
2599
|
+
promiseResolve(value);
|
|
2600
|
+
}
|
|
2601
|
+
});
|
|
2602
|
+
|
|
2603
|
+
try {
|
|
2604
|
+
original.apply(this, args);
|
|
2605
|
+
} catch (err) {
|
|
2606
|
+
promiseReject(err);
|
|
2607
|
+
}
|
|
2608
|
+
|
|
2609
|
+
return promise;
|
|
2610
|
+
}
|
|
2611
|
+
|
|
2612
|
+
Object.setPrototypeOf(fn, Object.getPrototypeOf(original));
|
|
2613
|
+
|
|
2614
|
+
if (kCustomPromisifiedSymbol) Object.defineProperty(fn, kCustomPromisifiedSymbol, {
|
|
2615
|
+
value: fn, enumerable: false, writable: false, configurable: true
|
|
2616
|
+
});
|
|
2617
|
+
return Object.defineProperties(
|
|
2618
|
+
fn,
|
|
2619
|
+
getOwnPropertyDescriptors(original)
|
|
2620
|
+
);
|
|
2621
|
+
}
|
|
2622
|
+
|
|
2623
|
+
promisify.custom = kCustomPromisifiedSymbol;
|
|
2624
|
+
|
|
2553
2625
|
var lookup = [];
|
|
2554
2626
|
var revLookup = [];
|
|
2555
2627
|
var Arr = typeof Uint8Array !== 'undefined' ? Uint8Array : Array;
|
|
@@ -6800,12 +6872,798 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
6800
6872
|
}
|
|
6801
6873
|
var memory_cache_class_default = MemoryCache;
|
|
6802
6874
|
|
|
6875
|
+
var fs = {};
|
|
6876
|
+
|
|
6877
|
+
const readFile$1 = promisify(fs.readFile);
|
|
6878
|
+
const writeFile$1 = promisify(fs.writeFile);
|
|
6879
|
+
const unlink = promisify(fs.unlink);
|
|
6880
|
+
const readdir$1 = promisify(fs.readdir);
|
|
6881
|
+
const stat$1 = promisify(fs.stat);
|
|
6882
|
+
const mkdir = promisify(fs.mkdir);
|
|
6883
|
+
class FilesystemCache extends Cache {
|
|
6884
|
+
constructor({
|
|
6885
|
+
directory,
|
|
6886
|
+
prefix = "cache",
|
|
6887
|
+
ttl = 36e5,
|
|
6888
|
+
enableCompression = true,
|
|
6889
|
+
compressionThreshold = 1024,
|
|
6890
|
+
createDirectory = true,
|
|
6891
|
+
fileExtension = ".cache",
|
|
6892
|
+
enableMetadata = true,
|
|
6893
|
+
maxFileSize = 10485760,
|
|
6894
|
+
// 10MB
|
|
6895
|
+
enableStats = false,
|
|
6896
|
+
enableCleanup = true,
|
|
6897
|
+
cleanupInterval = 3e5,
|
|
6898
|
+
// 5 minutes
|
|
6899
|
+
encoding = "utf8",
|
|
6900
|
+
fileMode = 420,
|
|
6901
|
+
enableBackup = false,
|
|
6902
|
+
backupSuffix = ".bak",
|
|
6903
|
+
enableLocking = false,
|
|
6904
|
+
lockTimeout = 5e3,
|
|
6905
|
+
enableJournal = false,
|
|
6906
|
+
journalFile = "cache.journal",
|
|
6907
|
+
...config
|
|
6908
|
+
}) {
|
|
6909
|
+
super(config);
|
|
6910
|
+
if (!directory) {
|
|
6911
|
+
throw new Error("FilesystemCache: directory parameter is required");
|
|
6912
|
+
}
|
|
6913
|
+
this.directory = path.resolve(directory);
|
|
6914
|
+
this.prefix = prefix;
|
|
6915
|
+
this.ttl = ttl;
|
|
6916
|
+
this.enableCompression = enableCompression;
|
|
6917
|
+
this.compressionThreshold = compressionThreshold;
|
|
6918
|
+
this.createDirectory = createDirectory;
|
|
6919
|
+
this.fileExtension = fileExtension;
|
|
6920
|
+
this.enableMetadata = enableMetadata;
|
|
6921
|
+
this.maxFileSize = maxFileSize;
|
|
6922
|
+
this.enableStats = enableStats;
|
|
6923
|
+
this.enableCleanup = enableCleanup;
|
|
6924
|
+
this.cleanupInterval = cleanupInterval;
|
|
6925
|
+
this.encoding = encoding;
|
|
6926
|
+
this.fileMode = fileMode;
|
|
6927
|
+
this.enableBackup = enableBackup;
|
|
6928
|
+
this.backupSuffix = backupSuffix;
|
|
6929
|
+
this.enableLocking = enableLocking;
|
|
6930
|
+
this.lockTimeout = lockTimeout;
|
|
6931
|
+
this.enableJournal = enableJournal;
|
|
6932
|
+
this.journalFile = path.join(this.directory, journalFile);
|
|
6933
|
+
this.stats = {
|
|
6934
|
+
hits: 0,
|
|
6935
|
+
misses: 0,
|
|
6936
|
+
sets: 0,
|
|
6937
|
+
deletes: 0,
|
|
6938
|
+
clears: 0,
|
|
6939
|
+
errors: 0
|
|
6940
|
+
};
|
|
6941
|
+
this.locks = /* @__PURE__ */ new Map();
|
|
6942
|
+
this.cleanupTimer = null;
|
|
6943
|
+
this._init();
|
|
6944
|
+
}
|
|
6945
|
+
async _init() {
|
|
6946
|
+
if (this.createDirectory) {
|
|
6947
|
+
await this._ensureDirectory(this.directory);
|
|
6948
|
+
}
|
|
6949
|
+
if (this.enableCleanup && this.cleanupInterval > 0) {
|
|
6950
|
+
this.cleanupTimer = setInterval(() => {
|
|
6951
|
+
this._cleanup().catch((err) => {
|
|
6952
|
+
console.warn("FilesystemCache cleanup error:", err.message);
|
|
6953
|
+
});
|
|
6954
|
+
}, this.cleanupInterval);
|
|
6955
|
+
}
|
|
6956
|
+
}
|
|
6957
|
+
async _ensureDirectory(dir) {
|
|
6958
|
+
const [ok, err] = await try_fn_default(async () => {
|
|
6959
|
+
await mkdir(dir, { recursive: true });
|
|
6960
|
+
});
|
|
6961
|
+
if (!ok && err.code !== "EEXIST") {
|
|
6962
|
+
throw new Error(`Failed to create cache directory: ${err.message}`);
|
|
6963
|
+
}
|
|
6964
|
+
}
|
|
6965
|
+
_getFilePath(key) {
|
|
6966
|
+
const sanitizedKey = key.replace(/[<>:"/\\|?*]/g, "_");
|
|
6967
|
+
const filename = `${this.prefix}_${sanitizedKey}${this.fileExtension}`;
|
|
6968
|
+
return path.join(this.directory, filename);
|
|
6969
|
+
}
|
|
6970
|
+
_getMetadataPath(filePath) {
|
|
6971
|
+
return filePath + ".meta";
|
|
6972
|
+
}
|
|
6973
|
+
async _set(key, data) {
|
|
6974
|
+
const filePath = this._getFilePath(key);
|
|
6975
|
+
try {
|
|
6976
|
+
let serialized = JSON.stringify(data);
|
|
6977
|
+
const originalSize = Buffer.byteLength(serialized, this.encoding);
|
|
6978
|
+
if (originalSize > this.maxFileSize) {
|
|
6979
|
+
throw new Error(`Cache data exceeds maximum file size: ${originalSize} > ${this.maxFileSize}`);
|
|
6980
|
+
}
|
|
6981
|
+
let compressed = false;
|
|
6982
|
+
let finalData = serialized;
|
|
6983
|
+
if (this.enableCompression && originalSize >= this.compressionThreshold) {
|
|
6984
|
+
const compressedBuffer = zlib.gzipSync(Buffer.from(serialized, this.encoding));
|
|
6985
|
+
finalData = compressedBuffer.toString("base64");
|
|
6986
|
+
compressed = true;
|
|
6987
|
+
}
|
|
6988
|
+
if (this.enableBackup && await this._fileExists(filePath)) {
|
|
6989
|
+
const backupPath = filePath + this.backupSuffix;
|
|
6990
|
+
await this._copyFile(filePath, backupPath);
|
|
6991
|
+
}
|
|
6992
|
+
if (this.enableLocking) {
|
|
6993
|
+
await this._acquireLock(filePath);
|
|
6994
|
+
}
|
|
6995
|
+
try {
|
|
6996
|
+
await writeFile$1(filePath, finalData, {
|
|
6997
|
+
encoding: compressed ? "utf8" : this.encoding,
|
|
6998
|
+
mode: this.fileMode
|
|
6999
|
+
});
|
|
7000
|
+
if (this.enableMetadata) {
|
|
7001
|
+
const metadata = {
|
|
7002
|
+
key,
|
|
7003
|
+
timestamp: Date.now(),
|
|
7004
|
+
ttl: this.ttl,
|
|
7005
|
+
compressed,
|
|
7006
|
+
originalSize,
|
|
7007
|
+
compressedSize: compressed ? Buffer.byteLength(finalData, "utf8") : originalSize,
|
|
7008
|
+
compressionRatio: compressed ? (Buffer.byteLength(finalData, "utf8") / originalSize).toFixed(2) : 1
|
|
7009
|
+
};
|
|
7010
|
+
await writeFile$1(this._getMetadataPath(filePath), JSON.stringify(metadata), {
|
|
7011
|
+
encoding: this.encoding,
|
|
7012
|
+
mode: this.fileMode
|
|
7013
|
+
});
|
|
7014
|
+
}
|
|
7015
|
+
if (this.enableStats) {
|
|
7016
|
+
this.stats.sets++;
|
|
7017
|
+
}
|
|
7018
|
+
if (this.enableJournal) {
|
|
7019
|
+
await this._journalOperation("set", key, { size: originalSize, compressed });
|
|
7020
|
+
}
|
|
7021
|
+
} finally {
|
|
7022
|
+
if (this.enableLocking) {
|
|
7023
|
+
this._releaseLock(filePath);
|
|
7024
|
+
}
|
|
7025
|
+
}
|
|
7026
|
+
return data;
|
|
7027
|
+
} catch (error) {
|
|
7028
|
+
if (this.enableStats) {
|
|
7029
|
+
this.stats.errors++;
|
|
7030
|
+
}
|
|
7031
|
+
throw new Error(`Failed to set cache key '${key}': ${error.message}`);
|
|
7032
|
+
}
|
|
7033
|
+
}
|
|
7034
|
+
async _get(key) {
|
|
7035
|
+
const filePath = this._getFilePath(key);
|
|
7036
|
+
try {
|
|
7037
|
+
if (!await this._fileExists(filePath)) {
|
|
7038
|
+
if (this.enableStats) {
|
|
7039
|
+
this.stats.misses++;
|
|
7040
|
+
}
|
|
7041
|
+
return null;
|
|
7042
|
+
}
|
|
7043
|
+
let isExpired = false;
|
|
7044
|
+
if (this.enableMetadata) {
|
|
7045
|
+
const metadataPath = this._getMetadataPath(filePath);
|
|
7046
|
+
if (await this._fileExists(metadataPath)) {
|
|
7047
|
+
const [ok, err, metadata] = await try_fn_default(async () => {
|
|
7048
|
+
const metaContent = await readFile$1(metadataPath, this.encoding);
|
|
7049
|
+
return JSON.parse(metaContent);
|
|
7050
|
+
});
|
|
7051
|
+
if (ok && metadata.ttl > 0) {
|
|
7052
|
+
const age = Date.now() - metadata.timestamp;
|
|
7053
|
+
isExpired = age > metadata.ttl;
|
|
7054
|
+
}
|
|
7055
|
+
}
|
|
7056
|
+
} else if (this.ttl > 0) {
|
|
7057
|
+
const stats = await stat$1(filePath);
|
|
7058
|
+
const age = Date.now() - stats.mtime.getTime();
|
|
7059
|
+
isExpired = age > this.ttl;
|
|
7060
|
+
}
|
|
7061
|
+
if (isExpired) {
|
|
7062
|
+
await this._del(key);
|
|
7063
|
+
if (this.enableStats) {
|
|
7064
|
+
this.stats.misses++;
|
|
7065
|
+
}
|
|
7066
|
+
return null;
|
|
7067
|
+
}
|
|
7068
|
+
if (this.enableLocking) {
|
|
7069
|
+
await this._acquireLock(filePath);
|
|
7070
|
+
}
|
|
7071
|
+
try {
|
|
7072
|
+
const content = await readFile$1(filePath, this.encoding);
|
|
7073
|
+
let isCompressed = false;
|
|
7074
|
+
if (this.enableMetadata) {
|
|
7075
|
+
const metadataPath = this._getMetadataPath(filePath);
|
|
7076
|
+
if (await this._fileExists(metadataPath)) {
|
|
7077
|
+
const [ok, err, metadata] = await try_fn_default(async () => {
|
|
7078
|
+
const metaContent = await readFile$1(metadataPath, this.encoding);
|
|
7079
|
+
return JSON.parse(metaContent);
|
|
7080
|
+
});
|
|
7081
|
+
if (ok) {
|
|
7082
|
+
isCompressed = metadata.compressed;
|
|
7083
|
+
}
|
|
7084
|
+
}
|
|
7085
|
+
}
|
|
7086
|
+
let finalContent = content;
|
|
7087
|
+
if (isCompressed || this.enableCompression && content.match(/^[A-Za-z0-9+/=]+$/)) {
|
|
7088
|
+
try {
|
|
7089
|
+
const compressedBuffer = Buffer.from(content, "base64");
|
|
7090
|
+
finalContent = zlib.gunzipSync(compressedBuffer).toString(this.encoding);
|
|
7091
|
+
} catch (decompressError) {
|
|
7092
|
+
finalContent = content;
|
|
7093
|
+
}
|
|
7094
|
+
}
|
|
7095
|
+
const data = JSON.parse(finalContent);
|
|
7096
|
+
if (this.enableStats) {
|
|
7097
|
+
this.stats.hits++;
|
|
7098
|
+
}
|
|
7099
|
+
return data;
|
|
7100
|
+
} finally {
|
|
7101
|
+
if (this.enableLocking) {
|
|
7102
|
+
this._releaseLock(filePath);
|
|
7103
|
+
}
|
|
7104
|
+
}
|
|
7105
|
+
} catch (error) {
|
|
7106
|
+
if (this.enableStats) {
|
|
7107
|
+
this.stats.errors++;
|
|
7108
|
+
}
|
|
7109
|
+
await this._del(key);
|
|
7110
|
+
return null;
|
|
7111
|
+
}
|
|
7112
|
+
}
|
|
7113
|
+
async _del(key) {
|
|
7114
|
+
const filePath = this._getFilePath(key);
|
|
7115
|
+
try {
|
|
7116
|
+
if (await this._fileExists(filePath)) {
|
|
7117
|
+
await unlink(filePath);
|
|
7118
|
+
}
|
|
7119
|
+
if (this.enableMetadata) {
|
|
7120
|
+
const metadataPath = this._getMetadataPath(filePath);
|
|
7121
|
+
if (await this._fileExists(metadataPath)) {
|
|
7122
|
+
await unlink(metadataPath);
|
|
7123
|
+
}
|
|
7124
|
+
}
|
|
7125
|
+
if (this.enableBackup) {
|
|
7126
|
+
const backupPath = filePath + this.backupSuffix;
|
|
7127
|
+
if (await this._fileExists(backupPath)) {
|
|
7128
|
+
await unlink(backupPath);
|
|
7129
|
+
}
|
|
7130
|
+
}
|
|
7131
|
+
if (this.enableStats) {
|
|
7132
|
+
this.stats.deletes++;
|
|
7133
|
+
}
|
|
7134
|
+
if (this.enableJournal) {
|
|
7135
|
+
await this._journalOperation("delete", key);
|
|
7136
|
+
}
|
|
7137
|
+
return true;
|
|
7138
|
+
} catch (error) {
|
|
7139
|
+
if (this.enableStats) {
|
|
7140
|
+
this.stats.errors++;
|
|
7141
|
+
}
|
|
7142
|
+
throw new Error(`Failed to delete cache key '${key}': ${error.message}`);
|
|
7143
|
+
}
|
|
7144
|
+
}
|
|
7145
|
+
async _clear(prefix) {
|
|
7146
|
+
try {
|
|
7147
|
+
const files = await readdir$1(this.directory);
|
|
7148
|
+
const cacheFiles = files.filter((file) => {
|
|
7149
|
+
if (!file.startsWith(this.prefix)) return false;
|
|
7150
|
+
if (!file.endsWith(this.fileExtension)) return false;
|
|
7151
|
+
if (prefix) {
|
|
7152
|
+
const keyPart = file.slice(this.prefix.length + 1, -this.fileExtension.length);
|
|
7153
|
+
return keyPart.startsWith(prefix);
|
|
7154
|
+
}
|
|
7155
|
+
return true;
|
|
7156
|
+
});
|
|
7157
|
+
for (const file of cacheFiles) {
|
|
7158
|
+
const filePath = path.join(this.directory, file);
|
|
7159
|
+
if (await this._fileExists(filePath)) {
|
|
7160
|
+
await unlink(filePath);
|
|
7161
|
+
}
|
|
7162
|
+
if (this.enableMetadata) {
|
|
7163
|
+
const metadataPath = this._getMetadataPath(filePath);
|
|
7164
|
+
if (await this._fileExists(metadataPath)) {
|
|
7165
|
+
await unlink(metadataPath);
|
|
7166
|
+
}
|
|
7167
|
+
}
|
|
7168
|
+
if (this.enableBackup) {
|
|
7169
|
+
const backupPath = filePath + this.backupSuffix;
|
|
7170
|
+
if (await this._fileExists(backupPath)) {
|
|
7171
|
+
await unlink(backupPath);
|
|
7172
|
+
}
|
|
7173
|
+
}
|
|
7174
|
+
}
|
|
7175
|
+
if (this.enableStats) {
|
|
7176
|
+
this.stats.clears++;
|
|
7177
|
+
}
|
|
7178
|
+
if (this.enableJournal) {
|
|
7179
|
+
await this._journalOperation("clear", prefix || "all", { count: cacheFiles.length });
|
|
7180
|
+
}
|
|
7181
|
+
return true;
|
|
7182
|
+
} catch (error) {
|
|
7183
|
+
if (this.enableStats) {
|
|
7184
|
+
this.stats.errors++;
|
|
7185
|
+
}
|
|
7186
|
+
throw new Error(`Failed to clear cache: ${error.message}`);
|
|
7187
|
+
}
|
|
7188
|
+
}
|
|
7189
|
+
async size() {
|
|
7190
|
+
const keys = await this.keys();
|
|
7191
|
+
return keys.length;
|
|
7192
|
+
}
|
|
7193
|
+
async keys() {
|
|
7194
|
+
try {
|
|
7195
|
+
const files = await readdir$1(this.directory);
|
|
7196
|
+
const cacheFiles = files.filter(
|
|
7197
|
+
(file) => file.startsWith(this.prefix) && file.endsWith(this.fileExtension)
|
|
7198
|
+
);
|
|
7199
|
+
const keys = cacheFiles.map((file) => {
|
|
7200
|
+
const keyPart = file.slice(this.prefix.length + 1, -this.fileExtension.length);
|
|
7201
|
+
return keyPart;
|
|
7202
|
+
});
|
|
7203
|
+
return keys;
|
|
7204
|
+
} catch (error) {
|
|
7205
|
+
console.warn("FilesystemCache: Failed to list keys:", error.message);
|
|
7206
|
+
return [];
|
|
7207
|
+
}
|
|
7208
|
+
}
|
|
7209
|
+
// Helper methods
|
|
7210
|
+
async _fileExists(filePath) {
|
|
7211
|
+
const [ok] = await try_fn_default(async () => {
|
|
7212
|
+
await stat$1(filePath);
|
|
7213
|
+
});
|
|
7214
|
+
return ok;
|
|
7215
|
+
}
|
|
7216
|
+
async _copyFile(src, dest) {
|
|
7217
|
+
const [ok, err] = await try_fn_default(async () => {
|
|
7218
|
+
const content = await readFile$1(src);
|
|
7219
|
+
await writeFile$1(dest, content);
|
|
7220
|
+
});
|
|
7221
|
+
if (!ok) {
|
|
7222
|
+
console.warn("FilesystemCache: Failed to create backup:", err.message);
|
|
7223
|
+
}
|
|
7224
|
+
}
|
|
7225
|
+
async _cleanup() {
|
|
7226
|
+
if (!this.ttl || this.ttl <= 0) return;
|
|
7227
|
+
try {
|
|
7228
|
+
const files = await readdir$1(this.directory);
|
|
7229
|
+
const now = Date.now();
|
|
7230
|
+
for (const file of files) {
|
|
7231
|
+
if (!file.startsWith(this.prefix) || !file.endsWith(this.fileExtension)) {
|
|
7232
|
+
continue;
|
|
7233
|
+
}
|
|
7234
|
+
const filePath = path.join(this.directory, file);
|
|
7235
|
+
let shouldDelete = false;
|
|
7236
|
+
if (this.enableMetadata) {
|
|
7237
|
+
const metadataPath = this._getMetadataPath(filePath);
|
|
7238
|
+
if (await this._fileExists(metadataPath)) {
|
|
7239
|
+
const [ok, err, metadata] = await try_fn_default(async () => {
|
|
7240
|
+
const metaContent = await readFile$1(metadataPath, this.encoding);
|
|
7241
|
+
return JSON.parse(metaContent);
|
|
7242
|
+
});
|
|
7243
|
+
if (ok && metadata.ttl > 0) {
|
|
7244
|
+
const age = now - metadata.timestamp;
|
|
7245
|
+
shouldDelete = age > metadata.ttl;
|
|
7246
|
+
}
|
|
7247
|
+
}
|
|
7248
|
+
} else {
|
|
7249
|
+
const [ok, err, stats] = await try_fn_default(async () => {
|
|
7250
|
+
return await stat$1(filePath);
|
|
7251
|
+
});
|
|
7252
|
+
if (ok) {
|
|
7253
|
+
const age = now - stats.mtime.getTime();
|
|
7254
|
+
shouldDelete = age > this.ttl;
|
|
7255
|
+
}
|
|
7256
|
+
}
|
|
7257
|
+
if (shouldDelete) {
|
|
7258
|
+
const keyPart = file.slice(this.prefix.length + 1, -this.fileExtension.length);
|
|
7259
|
+
await this._del(keyPart);
|
|
7260
|
+
}
|
|
7261
|
+
}
|
|
7262
|
+
} catch (error) {
|
|
7263
|
+
console.warn("FilesystemCache cleanup error:", error.message);
|
|
7264
|
+
}
|
|
7265
|
+
}
|
|
7266
|
+
async _acquireLock(filePath) {
|
|
7267
|
+
if (!this.enableLocking) return;
|
|
7268
|
+
const lockKey = filePath;
|
|
7269
|
+
const startTime = Date.now();
|
|
7270
|
+
while (this.locks.has(lockKey)) {
|
|
7271
|
+
if (Date.now() - startTime > this.lockTimeout) {
|
|
7272
|
+
throw new Error(`Lock timeout for file: ${filePath}`);
|
|
7273
|
+
}
|
|
7274
|
+
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
7275
|
+
}
|
|
7276
|
+
this.locks.set(lockKey, Date.now());
|
|
7277
|
+
}
|
|
7278
|
+
_releaseLock(filePath) {
|
|
7279
|
+
if (!this.enableLocking) return;
|
|
7280
|
+
this.locks.delete(filePath);
|
|
7281
|
+
}
|
|
7282
|
+
async _journalOperation(operation, key, metadata = {}) {
|
|
7283
|
+
if (!this.enableJournal) return;
|
|
7284
|
+
const entry = {
|
|
7285
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
7286
|
+
operation,
|
|
7287
|
+
key,
|
|
7288
|
+
metadata
|
|
7289
|
+
};
|
|
7290
|
+
const [ok, err] = await try_fn_default(async () => {
|
|
7291
|
+
const line = JSON.stringify(entry) + "\n";
|
|
7292
|
+
await fs.promises.appendFile(this.journalFile, line, this.encoding);
|
|
7293
|
+
});
|
|
7294
|
+
if (!ok) {
|
|
7295
|
+
console.warn("FilesystemCache journal error:", err.message);
|
|
7296
|
+
}
|
|
7297
|
+
}
|
|
7298
|
+
// Cleanup on process exit
|
|
7299
|
+
destroy() {
|
|
7300
|
+
if (this.cleanupTimer) {
|
|
7301
|
+
clearInterval(this.cleanupTimer);
|
|
7302
|
+
this.cleanupTimer = null;
|
|
7303
|
+
}
|
|
7304
|
+
}
|
|
7305
|
+
// Get cache statistics
|
|
7306
|
+
getStats() {
|
|
7307
|
+
return {
|
|
7308
|
+
...this.stats,
|
|
7309
|
+
directory: this.directory,
|
|
7310
|
+
ttl: this.ttl,
|
|
7311
|
+
compression: this.enableCompression,
|
|
7312
|
+
metadata: this.enableMetadata,
|
|
7313
|
+
cleanup: this.enableCleanup,
|
|
7314
|
+
locking: this.enableLocking,
|
|
7315
|
+
journal: this.enableJournal
|
|
7316
|
+
};
|
|
7317
|
+
}
|
|
7318
|
+
}
|
|
7319
|
+
|
|
7320
|
+
promisify(fs.mkdir);
|
|
7321
|
+
const rmdir = promisify(fs.rmdir);
|
|
7322
|
+
const readdir = promisify(fs.readdir);
|
|
7323
|
+
const stat = promisify(fs.stat);
|
|
7324
|
+
const writeFile = promisify(fs.writeFile);
|
|
7325
|
+
const readFile = promisify(fs.readFile);
|
|
7326
|
+
class PartitionAwareFilesystemCache extends FilesystemCache {
|
|
7327
|
+
constructor({
|
|
7328
|
+
partitionStrategy = "hierarchical",
|
|
7329
|
+
// 'hierarchical', 'flat', 'temporal'
|
|
7330
|
+
trackUsage = true,
|
|
7331
|
+
preloadRelated = false,
|
|
7332
|
+
preloadThreshold = 10,
|
|
7333
|
+
maxCacheSize = null,
|
|
7334
|
+
usageStatsFile = "partition-usage.json",
|
|
7335
|
+
...config
|
|
7336
|
+
}) {
|
|
7337
|
+
super(config);
|
|
7338
|
+
this.partitionStrategy = partitionStrategy;
|
|
7339
|
+
this.trackUsage = trackUsage;
|
|
7340
|
+
this.preloadRelated = preloadRelated;
|
|
7341
|
+
this.preloadThreshold = preloadThreshold;
|
|
7342
|
+
this.maxCacheSize = maxCacheSize;
|
|
7343
|
+
this.usageStatsFile = path.join(this.directory, usageStatsFile);
|
|
7344
|
+
this.partitionUsage = /* @__PURE__ */ new Map();
|
|
7345
|
+
this.loadUsageStats();
|
|
7346
|
+
}
|
|
7347
|
+
/**
|
|
7348
|
+
* Generate partition-aware cache key
|
|
7349
|
+
*/
|
|
7350
|
+
_getPartitionCacheKey(resource, action, partition, partitionValues = {}, params = {}) {
|
|
7351
|
+
const keyParts = [`resource=${resource}`, `action=${action}`];
|
|
7352
|
+
if (partition && Object.keys(partitionValues).length > 0) {
|
|
7353
|
+
keyParts.push(`partition=${partition}`);
|
|
7354
|
+
const sortedFields = Object.entries(partitionValues).sort(([a], [b]) => a.localeCompare(b));
|
|
7355
|
+
for (const [field, value] of sortedFields) {
|
|
7356
|
+
if (value !== null && value !== void 0) {
|
|
7357
|
+
keyParts.push(`${field}=${value}`);
|
|
7358
|
+
}
|
|
7359
|
+
}
|
|
7360
|
+
}
|
|
7361
|
+
if (Object.keys(params).length > 0) {
|
|
7362
|
+
const paramsStr = Object.entries(params).sort(([a], [b]) => a.localeCompare(b)).map(([k, v]) => `${k}=${v}`).join("|");
|
|
7363
|
+
keyParts.push(`params=${Buffer.from(paramsStr).toString("base64")}`);
|
|
7364
|
+
}
|
|
7365
|
+
return keyParts.join("/") + this.fileExtension;
|
|
7366
|
+
}
|
|
7367
|
+
/**
|
|
7368
|
+
* Get directory path for partition cache
|
|
7369
|
+
*/
|
|
7370
|
+
_getPartitionDirectory(resource, partition, partitionValues = {}) {
|
|
7371
|
+
const basePath = path.join(this.directory, `resource=${resource}`);
|
|
7372
|
+
if (!partition) {
|
|
7373
|
+
return basePath;
|
|
7374
|
+
}
|
|
7375
|
+
if (this.partitionStrategy === "flat") {
|
|
7376
|
+
return path.join(basePath, "partitions");
|
|
7377
|
+
}
|
|
7378
|
+
if (this.partitionStrategy === "temporal" && this._isTemporalPartition(partition, partitionValues)) {
|
|
7379
|
+
return this._getTemporalDirectory(basePath, partition, partitionValues);
|
|
7380
|
+
}
|
|
7381
|
+
const pathParts = [basePath, `partition=${partition}`];
|
|
7382
|
+
const sortedFields = Object.entries(partitionValues).sort(([a], [b]) => a.localeCompare(b));
|
|
7383
|
+
for (const [field, value] of sortedFields) {
|
|
7384
|
+
if (value !== null && value !== void 0) {
|
|
7385
|
+
pathParts.push(`${field}=${this._sanitizePathValue(value)}`);
|
|
7386
|
+
}
|
|
7387
|
+
}
|
|
7388
|
+
return path.join(...pathParts);
|
|
7389
|
+
}
|
|
7390
|
+
/**
|
|
7391
|
+
* Enhanced set method with partition awareness
|
|
7392
|
+
*/
|
|
7393
|
+
async _set(key, data, options = {}) {
|
|
7394
|
+
const { resource, action, partition, partitionValues, params } = options;
|
|
7395
|
+
if (resource && partition) {
|
|
7396
|
+
const partitionKey = this._getPartitionCacheKey(resource, action, partition, partitionValues, params);
|
|
7397
|
+
const partitionDir = this._getPartitionDirectory(resource, partition, partitionValues);
|
|
7398
|
+
await this._ensureDirectory(partitionDir);
|
|
7399
|
+
const filePath = path.join(partitionDir, this._sanitizeFileName(partitionKey));
|
|
7400
|
+
if (this.trackUsage) {
|
|
7401
|
+
await this._trackPartitionUsage(resource, partition, partitionValues);
|
|
7402
|
+
}
|
|
7403
|
+
const partitionData = {
|
|
7404
|
+
data,
|
|
7405
|
+
metadata: {
|
|
7406
|
+
resource,
|
|
7407
|
+
partition,
|
|
7408
|
+
partitionValues,
|
|
7409
|
+
timestamp: Date.now(),
|
|
7410
|
+
ttl: this.ttl
|
|
7411
|
+
}
|
|
7412
|
+
};
|
|
7413
|
+
return this._writeFileWithMetadata(filePath, partitionData);
|
|
7414
|
+
}
|
|
7415
|
+
return super._set(key, data);
|
|
7416
|
+
}
|
|
7417
|
+
/**
|
|
7418
|
+
* Enhanced get method with partition awareness
|
|
7419
|
+
*/
|
|
7420
|
+
async _get(key, options = {}) {
|
|
7421
|
+
const { resource, action, partition, partitionValues, params } = options;
|
|
7422
|
+
if (resource && partition) {
|
|
7423
|
+
const partitionKey = this._getPartitionCacheKey(resource, action, partition, partitionValues, params);
|
|
7424
|
+
const partitionDir = this._getPartitionDirectory(resource, partition, partitionValues);
|
|
7425
|
+
const filePath = path.join(partitionDir, this._sanitizeFileName(partitionKey));
|
|
7426
|
+
if (!await this._fileExists(filePath)) {
|
|
7427
|
+
if (this.preloadRelated) {
|
|
7428
|
+
await this._preloadRelatedPartitions(resource, partition, partitionValues);
|
|
7429
|
+
}
|
|
7430
|
+
return null;
|
|
7431
|
+
}
|
|
7432
|
+
const result = await this._readFileWithMetadata(filePath);
|
|
7433
|
+
if (result && this.trackUsage) {
|
|
7434
|
+
await this._trackPartitionUsage(resource, partition, partitionValues);
|
|
7435
|
+
}
|
|
7436
|
+
return result?.data || null;
|
|
7437
|
+
}
|
|
7438
|
+
return super._get(key);
|
|
7439
|
+
}
|
|
7440
|
+
/**
|
|
7441
|
+
* Clear cache for specific partition
|
|
7442
|
+
*/
|
|
7443
|
+
async clearPartition(resource, partition, partitionValues = {}) {
|
|
7444
|
+
const partitionDir = this._getPartitionDirectory(resource, partition, partitionValues);
|
|
7445
|
+
const [ok, err] = await try_fn_default(async () => {
|
|
7446
|
+
if (await this._fileExists(partitionDir)) {
|
|
7447
|
+
await rmdir(partitionDir, { recursive: true });
|
|
7448
|
+
}
|
|
7449
|
+
});
|
|
7450
|
+
if (!ok) {
|
|
7451
|
+
console.warn(`Failed to clear partition cache: ${err.message}`);
|
|
7452
|
+
}
|
|
7453
|
+
const usageKey = this._getUsageKey(resource, partition, partitionValues);
|
|
7454
|
+
this.partitionUsage.delete(usageKey);
|
|
7455
|
+
await this._saveUsageStats();
|
|
7456
|
+
return ok;
|
|
7457
|
+
}
|
|
7458
|
+
/**
|
|
7459
|
+
* Clear all partitions for a resource
|
|
7460
|
+
*/
|
|
7461
|
+
async clearResourcePartitions(resource) {
|
|
7462
|
+
const resourceDir = path.join(this.directory, `resource=${resource}`);
|
|
7463
|
+
const [ok, err] = await try_fn_default(async () => {
|
|
7464
|
+
if (await this._fileExists(resourceDir)) {
|
|
7465
|
+
await rmdir(resourceDir, { recursive: true });
|
|
7466
|
+
}
|
|
7467
|
+
});
|
|
7468
|
+
for (const [key] of this.partitionUsage.entries()) {
|
|
7469
|
+
if (key.startsWith(`${resource}/`)) {
|
|
7470
|
+
this.partitionUsage.delete(key);
|
|
7471
|
+
}
|
|
7472
|
+
}
|
|
7473
|
+
await this._saveUsageStats();
|
|
7474
|
+
return ok;
|
|
7475
|
+
}
|
|
7476
|
+
/**
|
|
7477
|
+
* Get partition cache statistics
|
|
7478
|
+
*/
|
|
7479
|
+
async getPartitionStats(resource, partition = null) {
|
|
7480
|
+
const stats = {
|
|
7481
|
+
totalFiles: 0,
|
|
7482
|
+
totalSize: 0,
|
|
7483
|
+
partitions: {},
|
|
7484
|
+
usage: {}
|
|
7485
|
+
};
|
|
7486
|
+
const resourceDir = path.join(this.directory, `resource=${resource}`);
|
|
7487
|
+
if (!await this._fileExists(resourceDir)) {
|
|
7488
|
+
return stats;
|
|
7489
|
+
}
|
|
7490
|
+
await this._calculateDirectoryStats(resourceDir, stats);
|
|
7491
|
+
for (const [key, usage] of this.partitionUsage.entries()) {
|
|
7492
|
+
if (key.startsWith(`${resource}/`)) {
|
|
7493
|
+
const partitionName = key.split("/")[1];
|
|
7494
|
+
if (!partition || partitionName === partition) {
|
|
7495
|
+
stats.usage[partitionName] = usage;
|
|
7496
|
+
}
|
|
7497
|
+
}
|
|
7498
|
+
}
|
|
7499
|
+
return stats;
|
|
7500
|
+
}
|
|
7501
|
+
/**
|
|
7502
|
+
* Get cache recommendations based on usage patterns
|
|
7503
|
+
*/
|
|
7504
|
+
async getCacheRecommendations(resource) {
|
|
7505
|
+
const recommendations = [];
|
|
7506
|
+
const now = Date.now();
|
|
7507
|
+
const dayMs = 24 * 60 * 60 * 1e3;
|
|
7508
|
+
for (const [key, usage] of this.partitionUsage.entries()) {
|
|
7509
|
+
if (key.startsWith(`${resource}/`)) {
|
|
7510
|
+
const [, partition] = key.split("/");
|
|
7511
|
+
const daysSinceLastAccess = (now - usage.lastAccess) / dayMs;
|
|
7512
|
+
const accessesPerDay = usage.count / Math.max(1, daysSinceLastAccess);
|
|
7513
|
+
let recommendation = "keep";
|
|
7514
|
+
let priority = usage.count;
|
|
7515
|
+
if (daysSinceLastAccess > 30) {
|
|
7516
|
+
recommendation = "archive";
|
|
7517
|
+
priority = 0;
|
|
7518
|
+
} else if (accessesPerDay < 0.1) {
|
|
7519
|
+
recommendation = "reduce_ttl";
|
|
7520
|
+
priority = 1;
|
|
7521
|
+
} else if (accessesPerDay > 10) {
|
|
7522
|
+
recommendation = "preload";
|
|
7523
|
+
priority = 100;
|
|
7524
|
+
}
|
|
7525
|
+
recommendations.push({
|
|
7526
|
+
partition,
|
|
7527
|
+
recommendation,
|
|
7528
|
+
priority,
|
|
7529
|
+
usage: accessesPerDay,
|
|
7530
|
+
lastAccess: new Date(usage.lastAccess).toISOString()
|
|
7531
|
+
});
|
|
7532
|
+
}
|
|
7533
|
+
}
|
|
7534
|
+
return recommendations.sort((a, b) => b.priority - a.priority);
|
|
7535
|
+
}
|
|
7536
|
+
/**
|
|
7537
|
+
* Preload frequently accessed partitions
|
|
7538
|
+
*/
|
|
7539
|
+
async warmPartitionCache(resource, options = {}) {
|
|
7540
|
+
const { partitions = [], maxFiles = 1e3 } = options;
|
|
7541
|
+
let warmedCount = 0;
|
|
7542
|
+
for (const partition of partitions) {
|
|
7543
|
+
const usageKey = `${resource}/${partition}`;
|
|
7544
|
+
const usage = this.partitionUsage.get(usageKey);
|
|
7545
|
+
if (usage && usage.count >= this.preloadThreshold) {
|
|
7546
|
+
console.log(`\u{1F525} Warming cache for ${resource}/${partition} (${usage.count} accesses)`);
|
|
7547
|
+
warmedCount++;
|
|
7548
|
+
}
|
|
7549
|
+
if (warmedCount >= maxFiles) break;
|
|
7550
|
+
}
|
|
7551
|
+
return warmedCount;
|
|
7552
|
+
}
|
|
7553
|
+
// Private helper methods
|
|
7554
|
+
async _trackPartitionUsage(resource, partition, partitionValues) {
|
|
7555
|
+
const usageKey = this._getUsageKey(resource, partition, partitionValues);
|
|
7556
|
+
const current = this.partitionUsage.get(usageKey) || {
|
|
7557
|
+
count: 0,
|
|
7558
|
+
firstAccess: Date.now(),
|
|
7559
|
+
lastAccess: Date.now()
|
|
7560
|
+
};
|
|
7561
|
+
current.count++;
|
|
7562
|
+
current.lastAccess = Date.now();
|
|
7563
|
+
this.partitionUsage.set(usageKey, current);
|
|
7564
|
+
if (current.count % 10 === 0) {
|
|
7565
|
+
await this._saveUsageStats();
|
|
7566
|
+
}
|
|
7567
|
+
}
|
|
7568
|
+
_getUsageKey(resource, partition, partitionValues) {
|
|
7569
|
+
const valuePart = Object.entries(partitionValues).sort(([a], [b]) => a.localeCompare(b)).map(([k, v]) => `${k}=${v}`).join("|");
|
|
7570
|
+
return `${resource}/${partition}/${valuePart}`;
|
|
7571
|
+
}
|
|
7572
|
+
async _preloadRelatedPartitions(resource, partition, partitionValues) {
|
|
7573
|
+
console.log(`\u{1F3AF} Preloading related partitions for ${resource}/${partition}`);
|
|
7574
|
+
if (partitionValues.timestamp || partitionValues.date) ;
|
|
7575
|
+
}
|
|
7576
|
+
_isTemporalPartition(partition, partitionValues) {
|
|
7577
|
+
const temporalFields = ["date", "timestamp", "createdAt", "updatedAt"];
|
|
7578
|
+
return Object.keys(partitionValues).some(
|
|
7579
|
+
(field) => temporalFields.some((tf) => field.toLowerCase().includes(tf))
|
|
7580
|
+
);
|
|
7581
|
+
}
|
|
7582
|
+
_getTemporalDirectory(basePath, partition, partitionValues) {
|
|
7583
|
+
const dateValue = Object.values(partitionValues)[0];
|
|
7584
|
+
if (typeof dateValue === "string" && dateValue.match(/^\d{4}-\d{2}-\d{2}/)) {
|
|
7585
|
+
const [year, month, day] = dateValue.split("-");
|
|
7586
|
+
return path.join(basePath, "temporal", year, month, day);
|
|
7587
|
+
}
|
|
7588
|
+
return path.join(basePath, `partition=${partition}`);
|
|
7589
|
+
}
|
|
7590
|
+
_sanitizePathValue(value) {
|
|
7591
|
+
return String(value).replace(/[<>:"/\\|?*]/g, "_");
|
|
7592
|
+
}
|
|
7593
|
+
_sanitizeFileName(filename) {
|
|
7594
|
+
return filename.replace(/[<>:"/\\|?*]/g, "_");
|
|
7595
|
+
}
|
|
7596
|
+
async _calculateDirectoryStats(dir, stats) {
|
|
7597
|
+
const [ok, err, files] = await try_fn_default(() => readdir(dir));
|
|
7598
|
+
if (!ok) return;
|
|
7599
|
+
for (const file of files) {
|
|
7600
|
+
const filePath = path.join(dir, file);
|
|
7601
|
+
const [statOk, statErr, fileStat] = await try_fn_default(() => stat(filePath));
|
|
7602
|
+
if (statOk) {
|
|
7603
|
+
if (fileStat.isDirectory()) {
|
|
7604
|
+
await this._calculateDirectoryStats(filePath, stats);
|
|
7605
|
+
} else {
|
|
7606
|
+
stats.totalFiles++;
|
|
7607
|
+
stats.totalSize += fileStat.size;
|
|
7608
|
+
}
|
|
7609
|
+
}
|
|
7610
|
+
}
|
|
7611
|
+
}
|
|
7612
|
+
async loadUsageStats() {
|
|
7613
|
+
const [ok, err, content] = await try_fn_default(async () => {
|
|
7614
|
+
const data = await readFile(this.usageStatsFile, "utf8");
|
|
7615
|
+
return JSON.parse(data);
|
|
7616
|
+
});
|
|
7617
|
+
if (ok && content) {
|
|
7618
|
+
this.partitionUsage = new Map(Object.entries(content));
|
|
7619
|
+
}
|
|
7620
|
+
}
|
|
7621
|
+
async _saveUsageStats() {
|
|
7622
|
+
const statsObject = Object.fromEntries(this.partitionUsage);
|
|
7623
|
+
await try_fn_default(async () => {
|
|
7624
|
+
await writeFile(
|
|
7625
|
+
this.usageStatsFile,
|
|
7626
|
+
JSON.stringify(statsObject, null, 2),
|
|
7627
|
+
"utf8"
|
|
7628
|
+
);
|
|
7629
|
+
});
|
|
7630
|
+
}
|
|
7631
|
+
async _writeFileWithMetadata(filePath, data) {
|
|
7632
|
+
const content = JSON.stringify(data);
|
|
7633
|
+
const [ok, err] = await try_fn_default(async () => {
|
|
7634
|
+
await writeFile(filePath, content, {
|
|
7635
|
+
encoding: this.encoding,
|
|
7636
|
+
mode: this.fileMode
|
|
7637
|
+
});
|
|
7638
|
+
});
|
|
7639
|
+
if (!ok) {
|
|
7640
|
+
throw new Error(`Failed to write cache file: ${err.message}`);
|
|
7641
|
+
}
|
|
7642
|
+
return true;
|
|
7643
|
+
}
|
|
7644
|
+
async _readFileWithMetadata(filePath) {
|
|
7645
|
+
const [ok, err, content] = await try_fn_default(async () => {
|
|
7646
|
+
return await readFile(filePath, this.encoding);
|
|
7647
|
+
});
|
|
7648
|
+
if (!ok || !content) return null;
|
|
7649
|
+
try {
|
|
7650
|
+
return JSON.parse(content);
|
|
7651
|
+
} catch (error) {
|
|
7652
|
+
return { data: content };
|
|
7653
|
+
}
|
|
7654
|
+
}
|
|
7655
|
+
}
|
|
7656
|
+
|
|
6803
7657
|
class CachePlugin extends plugin_class_default {
|
|
6804
7658
|
constructor(options = {}) {
|
|
6805
7659
|
super(options);
|
|
6806
7660
|
this.driver = options.driver;
|
|
6807
7661
|
this.config = {
|
|
6808
7662
|
includePartitions: options.includePartitions !== false,
|
|
7663
|
+
partitionStrategy: options.partitionStrategy || "hierarchical",
|
|
7664
|
+
partitionAware: options.partitionAware !== false,
|
|
7665
|
+
trackUsage: options.trackUsage !== false,
|
|
7666
|
+
preloadRelated: options.preloadRelated !== false,
|
|
6809
7667
|
...options
|
|
6810
7668
|
};
|
|
6811
7669
|
}
|
|
@@ -6817,6 +7675,17 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
6817
7675
|
this.driver = this.config.driver;
|
|
6818
7676
|
} else if (this.config.driverType === "memory") {
|
|
6819
7677
|
this.driver = new memory_cache_class_default(this.config.memoryOptions || {});
|
|
7678
|
+
} else if (this.config.driverType === "filesystem") {
|
|
7679
|
+
if (this.config.partitionAware) {
|
|
7680
|
+
this.driver = new PartitionAwareFilesystemCache({
|
|
7681
|
+
partitionStrategy: this.config.partitionStrategy,
|
|
7682
|
+
trackUsage: this.config.trackUsage,
|
|
7683
|
+
preloadRelated: this.config.preloadRelated,
|
|
7684
|
+
...this.config.filesystemOptions
|
|
7685
|
+
});
|
|
7686
|
+
} else {
|
|
7687
|
+
this.driver = new FilesystemCache(this.config.filesystemOptions || {});
|
|
7688
|
+
}
|
|
6820
7689
|
} else {
|
|
6821
7690
|
this.driver = new s3_cache_class_default({ client: this.database.client, ...this.config.s3Options || {} });
|
|
6822
7691
|
}
|
|
@@ -6857,6 +7726,20 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
6857
7726
|
const { action, params = {}, partition, partitionValues } = options;
|
|
6858
7727
|
return this.generateCacheKey(resource, action, params, partition, partitionValues);
|
|
6859
7728
|
};
|
|
7729
|
+
if (this.driver instanceof PartitionAwareFilesystemCache) {
|
|
7730
|
+
resource.clearPartitionCache = async (partition, partitionValues = {}) => {
|
|
7731
|
+
return await this.driver.clearPartition(resource.name, partition, partitionValues);
|
|
7732
|
+
};
|
|
7733
|
+
resource.getPartitionCacheStats = async (partition = null) => {
|
|
7734
|
+
return await this.driver.getPartitionStats(resource.name, partition);
|
|
7735
|
+
};
|
|
7736
|
+
resource.getCacheRecommendations = async () => {
|
|
7737
|
+
return await this.driver.getCacheRecommendations(resource.name);
|
|
7738
|
+
};
|
|
7739
|
+
resource.warmPartitionCache = async (partitions = [], options = {}) => {
|
|
7740
|
+
return await this.driver.warmPartitionCache(resource.name, { partitions, ...options });
|
|
7741
|
+
};
|
|
7742
|
+
}
|
|
6860
7743
|
const cacheMethods = [
|
|
6861
7744
|
"count",
|
|
6862
7745
|
"listIds",
|
|
@@ -6882,12 +7765,37 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
6882
7765
|
} else if (method === "get") {
|
|
6883
7766
|
key = await resource.cacheKeyFor({ action: method, params: { id: ctx.args[0] } });
|
|
6884
7767
|
}
|
|
6885
|
-
|
|
6886
|
-
|
|
6887
|
-
|
|
6888
|
-
|
|
6889
|
-
|
|
6890
|
-
|
|
7768
|
+
if (this.driver instanceof PartitionAwareFilesystemCache) {
|
|
7769
|
+
let partition, partitionValues;
|
|
7770
|
+
if (method === "list" || method === "listIds" || method === "count" || method === "page") {
|
|
7771
|
+
const args = ctx.args[0] || {};
|
|
7772
|
+
partition = args.partition;
|
|
7773
|
+
partitionValues = args.partitionValues;
|
|
7774
|
+
}
|
|
7775
|
+
const [ok, err, result] = await try_fn_default(() => resource.cache._get(key, {
|
|
7776
|
+
resource: resource.name,
|
|
7777
|
+
action: method,
|
|
7778
|
+
partition,
|
|
7779
|
+
partitionValues
|
|
7780
|
+
}));
|
|
7781
|
+
if (ok && result !== null && result !== void 0) return result;
|
|
7782
|
+
if (!ok && err.name !== "NoSuchKey") throw err;
|
|
7783
|
+
const freshResult = await next();
|
|
7784
|
+
await resource.cache._set(key, freshResult, {
|
|
7785
|
+
resource: resource.name,
|
|
7786
|
+
action: method,
|
|
7787
|
+
partition,
|
|
7788
|
+
partitionValues
|
|
7789
|
+
});
|
|
7790
|
+
return freshResult;
|
|
7791
|
+
} else {
|
|
7792
|
+
const [ok, err, result] = await try_fn_default(() => resource.cache.get(key));
|
|
7793
|
+
if (ok && result !== null && result !== void 0) return result;
|
|
7794
|
+
if (!ok && err.name !== "NoSuchKey") throw err;
|
|
7795
|
+
const freshResult = await next();
|
|
7796
|
+
await resource.cache.set(key, freshResult);
|
|
7797
|
+
return freshResult;
|
|
7798
|
+
}
|
|
6891
7799
|
});
|
|
6892
7800
|
}
|
|
6893
7801
|
const writeMethods = ["insert", "update", "delete", "deleteMany"];
|
|
@@ -6980,6 +7888,10 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
6980
7888
|
throw new Error(`Resource '${resourceName}' not found`);
|
|
6981
7889
|
}
|
|
6982
7890
|
const { includePartitions = true } = options;
|
|
7891
|
+
if (this.driver instanceof PartitionAwareFilesystemCache && resource.warmPartitionCache) {
|
|
7892
|
+
const partitionNames = resource.config.partitions ? Object.keys(resource.config.partitions) : [];
|
|
7893
|
+
return await resource.warmPartitionCache(partitionNames, options);
|
|
7894
|
+
}
|
|
6983
7895
|
await resource.getAll();
|
|
6984
7896
|
if (includePartitions && resource.config.partitions) {
|
|
6985
7897
|
for (const [partitionName, partitionDef] of Object.entries(resource.config.partitions)) {
|
|
@@ -7001,6 +7913,57 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
7001
7913
|
}
|
|
7002
7914
|
}
|
|
7003
7915
|
}
|
|
7916
|
+
// Partition-specific methods
|
|
7917
|
+
async getPartitionCacheStats(resourceName, partition = null) {
|
|
7918
|
+
if (!(this.driver instanceof PartitionAwareFilesystemCache)) {
|
|
7919
|
+
throw new Error("Partition cache statistics are only available with PartitionAwareFilesystemCache");
|
|
7920
|
+
}
|
|
7921
|
+
return await this.driver.getPartitionStats(resourceName, partition);
|
|
7922
|
+
}
|
|
7923
|
+
async getCacheRecommendations(resourceName) {
|
|
7924
|
+
if (!(this.driver instanceof PartitionAwareFilesystemCache)) {
|
|
7925
|
+
throw new Error("Cache recommendations are only available with PartitionAwareFilesystemCache");
|
|
7926
|
+
}
|
|
7927
|
+
return await this.driver.getCacheRecommendations(resourceName);
|
|
7928
|
+
}
|
|
7929
|
+
async clearPartitionCache(resourceName, partition, partitionValues = {}) {
|
|
7930
|
+
if (!(this.driver instanceof PartitionAwareFilesystemCache)) {
|
|
7931
|
+
throw new Error("Partition cache clearing is only available with PartitionAwareFilesystemCache");
|
|
7932
|
+
}
|
|
7933
|
+
return await this.driver.clearPartition(resourceName, partition, partitionValues);
|
|
7934
|
+
}
|
|
7935
|
+
async analyzeCacheUsage() {
|
|
7936
|
+
if (!(this.driver instanceof PartitionAwareFilesystemCache)) {
|
|
7937
|
+
return { message: "Cache usage analysis is only available with PartitionAwareFilesystemCache" };
|
|
7938
|
+
}
|
|
7939
|
+
const analysis = {
|
|
7940
|
+
totalResources: Object.keys(this.database.resources).length,
|
|
7941
|
+
resourceStats: {},
|
|
7942
|
+
recommendations: {},
|
|
7943
|
+
summary: {
|
|
7944
|
+
mostUsedPartitions: [],
|
|
7945
|
+
leastUsedPartitions: [],
|
|
7946
|
+
suggestedOptimizations: []
|
|
7947
|
+
}
|
|
7948
|
+
};
|
|
7949
|
+
for (const [resourceName, resource] of Object.entries(this.database.resources)) {
|
|
7950
|
+
try {
|
|
7951
|
+
analysis.resourceStats[resourceName] = await this.driver.getPartitionStats(resourceName);
|
|
7952
|
+
analysis.recommendations[resourceName] = await this.driver.getCacheRecommendations(resourceName);
|
|
7953
|
+
} catch (error) {
|
|
7954
|
+
analysis.resourceStats[resourceName] = { error: error.message };
|
|
7955
|
+
}
|
|
7956
|
+
}
|
|
7957
|
+
const allRecommendations = Object.values(analysis.recommendations).flat();
|
|
7958
|
+
analysis.summary.mostUsedPartitions = allRecommendations.filter((r) => r.recommendation === "preload").sort((a, b) => b.priority - a.priority).slice(0, 5);
|
|
7959
|
+
analysis.summary.leastUsedPartitions = allRecommendations.filter((r) => r.recommendation === "archive").slice(0, 5);
|
|
7960
|
+
analysis.summary.suggestedOptimizations = [
|
|
7961
|
+
`Consider preloading ${analysis.summary.mostUsedPartitions.length} high-usage partitions`,
|
|
7962
|
+
`Archive ${analysis.summary.leastUsedPartitions.length} unused partitions`,
|
|
7963
|
+
`Monitor cache hit rates for partition efficiency`
|
|
7964
|
+
];
|
|
7965
|
+
return analysis;
|
|
7966
|
+
}
|
|
7004
7967
|
}
|
|
7005
7968
|
|
|
7006
7969
|
const CostsPlugin = {
|
|
@@ -7177,24 +8140,29 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
7177
8140
|
resource._deleteMany = resource.deleteMany;
|
|
7178
8141
|
this.wrapResourceMethod(resource, "insert", async (result, args, methodName) => {
|
|
7179
8142
|
const [data] = args;
|
|
7180
|
-
this.indexRecord(resource.name, result.id, data).catch(
|
|
8143
|
+
this.indexRecord(resource.name, result.id, data).catch(() => {
|
|
8144
|
+
});
|
|
7181
8145
|
return result;
|
|
7182
8146
|
});
|
|
7183
8147
|
this.wrapResourceMethod(resource, "update", async (result, args, methodName) => {
|
|
7184
8148
|
const [id, data] = args;
|
|
7185
|
-
this.removeRecordFromIndex(resource.name, id).catch(
|
|
7186
|
-
|
|
8149
|
+
this.removeRecordFromIndex(resource.name, id).catch(() => {
|
|
8150
|
+
});
|
|
8151
|
+
this.indexRecord(resource.name, id, result).catch(() => {
|
|
8152
|
+
});
|
|
7187
8153
|
return result;
|
|
7188
8154
|
});
|
|
7189
8155
|
this.wrapResourceMethod(resource, "delete", async (result, args, methodName) => {
|
|
7190
8156
|
const [id] = args;
|
|
7191
|
-
this.removeRecordFromIndex(resource.name, id).catch(
|
|
8157
|
+
this.removeRecordFromIndex(resource.name, id).catch(() => {
|
|
8158
|
+
});
|
|
7192
8159
|
return result;
|
|
7193
8160
|
});
|
|
7194
8161
|
this.wrapResourceMethod(resource, "deleteMany", async (result, args, methodName) => {
|
|
7195
8162
|
const [ids] = args;
|
|
7196
8163
|
for (const id of ids) {
|
|
7197
|
-
this.removeRecordFromIndex(resource.name, id).catch(
|
|
8164
|
+
this.removeRecordFromIndex(resource.name, id).catch(() => {
|
|
8165
|
+
});
|
|
7198
8166
|
}
|
|
7199
8167
|
return result;
|
|
7200
8168
|
});
|
|
@@ -7684,7 +8652,8 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
7684
8652
|
}
|
|
7685
8653
|
if (this.config.flushInterval > 0) {
|
|
7686
8654
|
this.flushTimer = setInterval(() => {
|
|
7687
|
-
this.flushMetrics().catch(
|
|
8655
|
+
this.flushMetrics().catch(() => {
|
|
8656
|
+
});
|
|
7688
8657
|
}, this.config.flushInterval);
|
|
7689
8658
|
}
|
|
7690
8659
|
}
|
|
@@ -7756,9 +8725,6 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
7756
8725
|
}
|
|
7757
8726
|
this.resetMetrics();
|
|
7758
8727
|
});
|
|
7759
|
-
if (!ok) {
|
|
7760
|
-
console.error("Failed to flush metrics:", err);
|
|
7761
|
-
}
|
|
7762
8728
|
}
|
|
7763
8729
|
resetMetrics() {
|
|
7764
8730
|
for (const operation of Object.keys(this.metrics.operations)) {
|
|
@@ -10595,7 +11561,7 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
10595
11561
|
if (okParse) contentType = "application/json";
|
|
10596
11562
|
}
|
|
10597
11563
|
if (this.behavior === "body-only" && (!body || body === "")) {
|
|
10598
|
-
throw new Error(`[Resource.insert]
|
|
11564
|
+
throw new Error(`[Resource.insert] Attempt to save object without body! Data: id=${finalId}, resource=${this.name}`);
|
|
10599
11565
|
}
|
|
10600
11566
|
const [okPut, errPut, putResult] = await try_fn_default(() => this.client.putObject({
|
|
10601
11567
|
key,
|
|
@@ -11942,8 +12908,8 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
11942
12908
|
return mappedData;
|
|
11943
12909
|
}
|
|
11944
12910
|
/**
|
|
11945
|
-
* Compose the full object (metadata + body) as
|
|
11946
|
-
*
|
|
12911
|
+
* Compose the full object (metadata + body) as returned by .get(),
|
|
12912
|
+
* using in-memory data after insert/update, according to behavior
|
|
11947
12913
|
*/
|
|
11948
12914
|
async composeFullObjectFromWrite({ id, metadata, body, behavior }) {
|
|
11949
12915
|
const behaviorFlags = {};
|
|
@@ -12098,7 +13064,7 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
12098
13064
|
if (!this._middlewares.has(method)) throw new ResourceError(`No such method for middleware: ${method}`, { operation: "useMiddleware", method });
|
|
12099
13065
|
this._middlewares.get(method).push(fn);
|
|
12100
13066
|
}
|
|
12101
|
-
//
|
|
13067
|
+
// Utility to apply schema default values
|
|
12102
13068
|
applyDefaults(data) {
|
|
12103
13069
|
const out = { ...data };
|
|
12104
13070
|
for (const [key, def] of Object.entries(this.attributes)) {
|
|
@@ -12230,7 +13196,7 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
12230
13196
|
super();
|
|
12231
13197
|
this.version = "1";
|
|
12232
13198
|
this.s3dbVersion = (() => {
|
|
12233
|
-
const [ok, err, version] = try_fn_default(() => true ? "7.2
|
|
13199
|
+
const [ok, err, version] = try_fn_default(() => true ? "7.3.2" : "latest");
|
|
12234
13200
|
return ok ? version : "latest";
|
|
12235
13201
|
})();
|
|
12236
13202
|
this.resources = {};
|
|
@@ -12303,7 +13269,7 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
12303
13269
|
name,
|
|
12304
13270
|
client: this.client,
|
|
12305
13271
|
database: this,
|
|
12306
|
-
//
|
|
13272
|
+
// ensure reference
|
|
12307
13273
|
version: currentVersion,
|
|
12308
13274
|
attributes: versionData.attributes,
|
|
12309
13275
|
behavior: versionData.behavior || "user-managed",
|
|
@@ -12804,21 +13770,33 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
12804
13770
|
throw err;
|
|
12805
13771
|
}
|
|
12806
13772
|
}
|
|
12807
|
-
//
|
|
12808
|
-
async replicate(
|
|
13773
|
+
// Support both object and parameter signatures for flexibility
|
|
13774
|
+
async replicate(resourceOrObj, operation, data, recordId, beforeData) {
|
|
13775
|
+
let resource, op, payload, id;
|
|
13776
|
+
if (typeof resourceOrObj === "object" && resourceOrObj.resource) {
|
|
13777
|
+
resource = resourceOrObj.resource;
|
|
13778
|
+
op = resourceOrObj.operation;
|
|
13779
|
+
payload = resourceOrObj.data;
|
|
13780
|
+
id = resourceOrObj.id;
|
|
13781
|
+
} else {
|
|
13782
|
+
resource = resourceOrObj;
|
|
13783
|
+
op = operation;
|
|
13784
|
+
payload = data;
|
|
13785
|
+
id = recordId;
|
|
13786
|
+
}
|
|
12809
13787
|
const normResource = normalizeResourceName$1(resource);
|
|
12810
|
-
const destResource = this._resolveDestResource(normResource,
|
|
13788
|
+
const destResource = this._resolveDestResource(normResource, payload);
|
|
12811
13789
|
const destResourceObj = this._getDestResourceObj(destResource);
|
|
12812
|
-
const transformedData = this._applyTransformer(normResource,
|
|
13790
|
+
const transformedData = this._applyTransformer(normResource, payload);
|
|
12813
13791
|
let result;
|
|
12814
|
-
if (
|
|
13792
|
+
if (op === "insert") {
|
|
12815
13793
|
result = await destResourceObj.insert(transformedData);
|
|
12816
|
-
} else if (
|
|
12817
|
-
result = await destResourceObj.update(
|
|
12818
|
-
} else if (
|
|
12819
|
-
result = await destResourceObj.delete(
|
|
13794
|
+
} else if (op === "update") {
|
|
13795
|
+
result = await destResourceObj.update(id, transformedData);
|
|
13796
|
+
} else if (op === "delete") {
|
|
13797
|
+
result = await destResourceObj.delete(id);
|
|
12820
13798
|
} else {
|
|
12821
|
-
throw new Error(`Invalid operation: ${
|
|
13799
|
+
throw new Error(`Invalid operation: ${op}. Supported operations are: insert, update, delete`);
|
|
12822
13800
|
}
|
|
12823
13801
|
return result;
|
|
12824
13802
|
}
|
|
@@ -12851,7 +13829,7 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
12851
13829
|
if (typeof entry[0] === "function") return resource;
|
|
12852
13830
|
}
|
|
12853
13831
|
if (typeof entry === "string") return entry;
|
|
12854
|
-
if (
|
|
13832
|
+
if (resource && !targetResourceName) targetResourceName = resource;
|
|
12855
13833
|
if (typeof entry === "object" && entry.resource) return entry.resource;
|
|
12856
13834
|
return resource;
|
|
12857
13835
|
}
|
|
@@ -12964,7 +13942,6 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
12964
13942
|
class SqsReplicator extends base_replicator_class_default {
|
|
12965
13943
|
constructor(config = {}, resources = [], client = null) {
|
|
12966
13944
|
super(config);
|
|
12967
|
-
this.resources = resources;
|
|
12968
13945
|
this.client = client;
|
|
12969
13946
|
this.queueUrl = config.queueUrl;
|
|
12970
13947
|
this.queues = config.queues || {};
|
|
@@ -12973,12 +13950,24 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
12973
13950
|
this.sqsClient = client || null;
|
|
12974
13951
|
this.messageGroupId = config.messageGroupId;
|
|
12975
13952
|
this.deduplicationId = config.deduplicationId;
|
|
12976
|
-
if (resources
|
|
13953
|
+
if (Array.isArray(resources)) {
|
|
13954
|
+
this.resources = {};
|
|
13955
|
+
for (const resource of resources) {
|
|
13956
|
+
if (typeof resource === "string") {
|
|
13957
|
+
this.resources[resource] = true;
|
|
13958
|
+
} else if (typeof resource === "object" && resource.name) {
|
|
13959
|
+
this.resources[resource.name] = resource;
|
|
13960
|
+
}
|
|
13961
|
+
}
|
|
13962
|
+
} else if (typeof resources === "object") {
|
|
13963
|
+
this.resources = resources;
|
|
12977
13964
|
for (const [resourceName, resourceConfig] of Object.entries(resources)) {
|
|
12978
|
-
if (resourceConfig.queueUrl) {
|
|
13965
|
+
if (resourceConfig && resourceConfig.queueUrl) {
|
|
12979
13966
|
this.queues[resourceName] = resourceConfig.queueUrl;
|
|
12980
13967
|
}
|
|
12981
13968
|
}
|
|
13969
|
+
} else {
|
|
13970
|
+
this.resources = {};
|
|
12982
13971
|
}
|
|
12983
13972
|
}
|
|
12984
13973
|
validateConfig() {
|
|
@@ -13214,7 +14203,7 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
13214
14203
|
connected: !!this.sqsClient,
|
|
13215
14204
|
queueUrl: this.queueUrl,
|
|
13216
14205
|
region: this.region,
|
|
13217
|
-
resources: this.resources,
|
|
14206
|
+
resources: Object.keys(this.resources || {}),
|
|
13218
14207
|
totalreplicators: this.listenerCount("replicated"),
|
|
13219
14208
|
totalErrors: this.listenerCount("replicator_error")
|
|
13220
14209
|
};
|
|
@@ -13252,33 +14241,28 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
13252
14241
|
class ReplicatorPlugin extends plugin_class_default {
|
|
13253
14242
|
constructor(options = {}) {
|
|
13254
14243
|
super();
|
|
13255
|
-
if (options.verbose) {
|
|
13256
|
-
console.log("[PLUGIN][CONSTRUCTOR] ReplicatorPlugin constructor called");
|
|
13257
|
-
}
|
|
13258
|
-
if (options.verbose) {
|
|
13259
|
-
console.log("[PLUGIN][constructor] New ReplicatorPlugin instance created with config:", options);
|
|
13260
|
-
}
|
|
13261
14244
|
if (!options.replicators || !Array.isArray(options.replicators)) {
|
|
13262
14245
|
throw new Error("ReplicatorPlugin: replicators array is required");
|
|
13263
14246
|
}
|
|
13264
14247
|
for (const rep of options.replicators) {
|
|
13265
14248
|
if (!rep.driver) throw new Error("ReplicatorPlugin: each replicator must have a driver");
|
|
14249
|
+
if (!rep.resources || typeof rep.resources !== "object") throw new Error("ReplicatorPlugin: each replicator must have resources config");
|
|
14250
|
+
if (Object.keys(rep.resources).length === 0) throw new Error("ReplicatorPlugin: each replicator must have at least one resource configured");
|
|
13266
14251
|
}
|
|
13267
14252
|
this.config = {
|
|
13268
|
-
|
|
13269
|
-
|
|
13270
|
-
replicatorLogResource: options.replicatorLogResource
|
|
13271
|
-
|
|
14253
|
+
replicators: options.replicators || [],
|
|
14254
|
+
logErrors: options.logErrors !== false,
|
|
14255
|
+
replicatorLogResource: options.replicatorLogResource || "replicator_log",
|
|
14256
|
+
enabled: options.enabled !== false,
|
|
14257
|
+
batchSize: options.batchSize || 100,
|
|
14258
|
+
maxRetries: options.maxRetries || 3,
|
|
14259
|
+
timeout: options.timeout || 3e4,
|
|
14260
|
+
verbose: options.verbose || false,
|
|
14261
|
+
...options
|
|
13272
14262
|
};
|
|
13273
14263
|
this.replicators = [];
|
|
13274
|
-
this.
|
|
13275
|
-
this.
|
|
13276
|
-
this.stats = {
|
|
13277
|
-
totalOperations: 0,
|
|
13278
|
-
totalErrors: 0,
|
|
13279
|
-
lastError: null
|
|
13280
|
-
};
|
|
13281
|
-
this._installedListeners = [];
|
|
14264
|
+
this.database = null;
|
|
14265
|
+
this.eventListenersInstalled = /* @__PURE__ */ new Set();
|
|
13282
14266
|
}
|
|
13283
14267
|
/**
|
|
13284
14268
|
* Decompress data if it was compressed
|
|
@@ -13297,79 +14281,34 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
13297
14281
|
}
|
|
13298
14282
|
return filtered;
|
|
13299
14283
|
}
|
|
13300
|
-
installEventListeners(resource) {
|
|
13301
|
-
|
|
13302
|
-
|
|
13303
|
-
console.log("[PLUGIN] installEventListeners called for:", resource && resource.name, {
|
|
13304
|
-
hasDatabase: !!resource.database,
|
|
13305
|
-
sameDatabase: resource.database === plugin.database,
|
|
13306
|
-
alreadyInstalled: resource._replicatorListenersInstalled,
|
|
13307
|
-
resourceObj: resource,
|
|
13308
|
-
resourceObjId: resource && resource.id,
|
|
13309
|
-
resourceObjType: typeof resource,
|
|
13310
|
-
resourceObjIs: resource && Object.is(resource, plugin.database.resources && plugin.database.resources[resource.name]),
|
|
13311
|
-
resourceObjEq: resource === (plugin.database.resources && plugin.database.resources[resource.name])
|
|
13312
|
-
});
|
|
13313
|
-
}
|
|
13314
|
-
if (!resource || resource.name === plugin.config.replicatorLogResource || !resource.database || resource.database !== plugin.database) return;
|
|
13315
|
-
if (resource._replicatorListenersInstalled) return;
|
|
13316
|
-
resource._replicatorListenersInstalled = true;
|
|
13317
|
-
this._installedListeners.push(resource);
|
|
13318
|
-
if (plugin.config.verbose) {
|
|
13319
|
-
console.log(`[PLUGIN] installEventListeners INSTALLED for resource: ${resource && resource.name}`);
|
|
14284
|
+
installEventListeners(resource, database, plugin) {
|
|
14285
|
+
if (!resource || this.eventListenersInstalled.has(resource.name) || resource.name === this.config.replicatorLogResource) {
|
|
14286
|
+
return;
|
|
13320
14287
|
}
|
|
13321
14288
|
resource.on("insert", async (data) => {
|
|
13322
|
-
if (plugin.config.verbose) {
|
|
13323
|
-
console.log("[PLUGIN] Listener INSERT on", resource.name, "plugin.replicators.length:", plugin.replicators.length, plugin.replicators.map((r) => ({ id: r.id, driver: r.driver })));
|
|
13324
|
-
}
|
|
13325
14289
|
try {
|
|
13326
|
-
const completeData =
|
|
13327
|
-
|
|
13328
|
-
|
|
13329
|
-
}
|
|
13330
|
-
await plugin.processReplicatorEvent(resource.name, "insert", data.id, completeData, null);
|
|
13331
|
-
} catch (err) {
|
|
13332
|
-
if (plugin.config.verbose) {
|
|
13333
|
-
console.error(`[PLUGIN] Listener INSERT error on ${resource.name} id=${data && data.id}:`, err);
|
|
13334
|
-
}
|
|
14290
|
+
const completeData = { ...data, createdAt: (/* @__PURE__ */ new Date()).toISOString() };
|
|
14291
|
+
await plugin.processReplicatorEvent("insert", resource.name, completeData.id, completeData);
|
|
14292
|
+
} catch (error) {
|
|
14293
|
+
this.emit("error", { operation: "insert", error: error.message, resource: resource.name });
|
|
13335
14294
|
}
|
|
13336
14295
|
});
|
|
13337
|
-
resource.on("update", async (data) => {
|
|
13338
|
-
console.log("[PLUGIN][Listener][UPDATE][START] triggered for resource:", resource.name, "data:", data);
|
|
13339
|
-
const beforeData = data && data.$before;
|
|
13340
|
-
if (plugin.config.verbose) {
|
|
13341
|
-
console.log("[PLUGIN] Listener UPDATE on", resource.name, "plugin.replicators.length:", plugin.replicators.length, plugin.replicators.map((r) => ({ id: r.id, driver: r.driver })), "data:", data, "beforeData:", beforeData);
|
|
13342
|
-
}
|
|
14296
|
+
resource.on("update", async (data, beforeData) => {
|
|
13343
14297
|
try {
|
|
13344
|
-
|
|
13345
|
-
|
|
13346
|
-
|
|
13347
|
-
|
|
13348
|
-
} else {
|
|
13349
|
-
completeData = data;
|
|
13350
|
-
}
|
|
13351
|
-
await plugin.processReplicatorEvent(resource.name, "update", data.id, completeData, beforeData);
|
|
13352
|
-
} catch (err) {
|
|
13353
|
-
if (plugin.config.verbose) {
|
|
13354
|
-
console.error(`[PLUGIN] Listener UPDATE erro em ${resource.name} id=${data && data.id}:`, err);
|
|
13355
|
-
}
|
|
14298
|
+
const completeData = { ...data, updatedAt: (/* @__PURE__ */ new Date()).toISOString() };
|
|
14299
|
+
await plugin.processReplicatorEvent("update", resource.name, completeData.id, completeData, beforeData);
|
|
14300
|
+
} catch (error) {
|
|
14301
|
+
this.emit("error", { operation: "update", error: error.message, resource: resource.name });
|
|
13356
14302
|
}
|
|
13357
14303
|
});
|
|
13358
|
-
resource.on("delete", async (data
|
|
13359
|
-
if (plugin.config.verbose) {
|
|
13360
|
-
console.log("[PLUGIN] Listener DELETE on", resource.name, "plugin.replicators.length:", plugin.replicators.length, plugin.replicators.map((r) => ({ id: r.id, driver: r.driver })));
|
|
13361
|
-
}
|
|
14304
|
+
resource.on("delete", async (data) => {
|
|
13362
14305
|
try {
|
|
13363
|
-
await plugin.processReplicatorEvent(resource.name,
|
|
13364
|
-
} catch (
|
|
13365
|
-
|
|
13366
|
-
console.error(`[PLUGIN] Listener DELETE erro em ${resource.name} id=${data && data.id}:`, err);
|
|
13367
|
-
}
|
|
14306
|
+
await plugin.processReplicatorEvent("delete", resource.name, data.id, data);
|
|
14307
|
+
} catch (error) {
|
|
14308
|
+
this.emit("error", { operation: "delete", error: error.message, resource: resource.name });
|
|
13368
14309
|
}
|
|
13369
14310
|
});
|
|
13370
|
-
|
|
13371
|
-
console.log(`[PLUGIN] Listeners instalados para resource: ${resource && resource.name} (insert: ${resource.listenerCount("insert")}, update: ${resource.listenerCount("update")}, delete: ${resource.listenerCount("delete")})`);
|
|
13372
|
-
}
|
|
14311
|
+
this.eventListenersInstalled.add(resource.name);
|
|
13373
14312
|
}
|
|
13374
14313
|
/**
|
|
13375
14314
|
* Get complete data by always fetching the full record from the resource
|
|
@@ -13380,112 +14319,58 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
13380
14319
|
return ok ? completeRecord : data;
|
|
13381
14320
|
}
|
|
13382
14321
|
async setup(database) {
|
|
13383
|
-
console.log("[PLUGIN][SETUP] setup called");
|
|
13384
|
-
if (this.config.verbose) {
|
|
13385
|
-
console.log("[PLUGIN][setup] called with database:", database && database.name);
|
|
13386
|
-
}
|
|
13387
14322
|
this.database = database;
|
|
13388
|
-
|
|
13389
|
-
|
|
13390
|
-
|
|
13391
|
-
|
|
14323
|
+
try {
|
|
14324
|
+
await this.initializeReplicators(database);
|
|
14325
|
+
} catch (error) {
|
|
14326
|
+
this.emit("error", { operation: "setup", error: error.message });
|
|
14327
|
+
throw error;
|
|
14328
|
+
}
|
|
14329
|
+
try {
|
|
14330
|
+
if (this.config.replicatorLogResource) {
|
|
14331
|
+
const logRes = await database.createResource({
|
|
13392
14332
|
name: this.config.replicatorLogResource,
|
|
13393
|
-
behavior: "
|
|
14333
|
+
behavior: "body-overflow",
|
|
13394
14334
|
attributes: {
|
|
13395
|
-
|
|
13396
|
-
|
|
13397
|
-
|
|
13398
|
-
data: "
|
|
13399
|
-
|
|
13400
|
-
|
|
13401
|
-
|
|
13402
|
-
|
|
13403
|
-
byDate: { fields: { "createdAt": "string|maxlength:10" } }
|
|
14335
|
+
operation: "string",
|
|
14336
|
+
resourceName: "string",
|
|
14337
|
+
recordId: "string",
|
|
14338
|
+
data: "string",
|
|
14339
|
+
error: "string|optional",
|
|
14340
|
+
replicator: "string",
|
|
14341
|
+
timestamp: "string",
|
|
14342
|
+
status: "string"
|
|
13404
14343
|
}
|
|
13405
14344
|
});
|
|
13406
|
-
if (this.config.verbose) {
|
|
13407
|
-
console.log("[PLUGIN] Log resource created:", this.config.replicatorLogResource, !!logRes);
|
|
13408
|
-
}
|
|
13409
|
-
}
|
|
13410
|
-
database.resources[normalizeResourceName(this.config.replicatorLogResource)] = logRes;
|
|
13411
|
-
this.replicatorLog = logRes;
|
|
13412
|
-
if (this.config.verbose) {
|
|
13413
|
-
console.log("[PLUGIN] Log resource created and registered:", this.config.replicatorLogResource, !!database.resources[normalizeResourceName(this.config.replicatorLogResource)]);
|
|
13414
|
-
}
|
|
13415
|
-
if (typeof database.uploadMetadataFile === "function") {
|
|
13416
|
-
await database.uploadMetadataFile();
|
|
13417
|
-
if (this.config.verbose) {
|
|
13418
|
-
console.log("[PLUGIN] uploadMetadataFile called. database.resources keys:", Object.keys(database.resources));
|
|
13419
|
-
}
|
|
13420
|
-
}
|
|
13421
|
-
}
|
|
13422
|
-
if (this.config.replicators && this.config.replicators.length > 0 && this.replicators.length === 0) {
|
|
13423
|
-
await this.initializeReplicators();
|
|
13424
|
-
console.log("[PLUGIN][SETUP] after initializeReplicators, replicators.length:", this.replicators.length);
|
|
13425
|
-
if (this.config.verbose) {
|
|
13426
|
-
console.log("[PLUGIN][setup] After initializeReplicators, replicators.length:", this.replicators.length, this.replicators.map((r) => ({ id: r.id, driver: r.driver })));
|
|
13427
|
-
}
|
|
13428
|
-
}
|
|
13429
|
-
for (const resourceName in database.resources) {
|
|
13430
|
-
if (normalizeResourceName(resourceName) !== normalizeResourceName(this.config.replicatorLogResource)) {
|
|
13431
|
-
this.installEventListeners(database.resources[resourceName]);
|
|
13432
14345
|
}
|
|
14346
|
+
} catch (error) {
|
|
13433
14347
|
}
|
|
13434
|
-
|
|
13435
|
-
for (const resourceName in database.resources) {
|
|
13436
|
-
if (normalizeResourceName(resourceName) !== normalizeResourceName(this.config.replicatorLogResource)) {
|
|
13437
|
-
this.installEventListeners(database.resources[resourceName]);
|
|
13438
|
-
}
|
|
13439
|
-
}
|
|
13440
|
-
});
|
|
14348
|
+
await this.uploadMetadataFile(database);
|
|
13441
14349
|
const originalCreateResource = database.createResource.bind(database);
|
|
13442
14350
|
database.createResource = async (config) => {
|
|
13443
|
-
if (this.config.verbose) {
|
|
13444
|
-
console.log("[PLUGIN] createResource proxy called for:", config && config.name);
|
|
13445
|
-
}
|
|
13446
14351
|
const resource = await originalCreateResource(config);
|
|
13447
|
-
if (resource
|
|
13448
|
-
this.installEventListeners(resource);
|
|
14352
|
+
if (resource) {
|
|
14353
|
+
this.installEventListeners(resource, database, this);
|
|
13449
14354
|
}
|
|
13450
14355
|
return resource;
|
|
13451
14356
|
};
|
|
13452
|
-
|
|
13453
|
-
const resource = database.resources[resourceName];
|
|
13454
|
-
if (resource && resource.name !== this.config.replicatorLogResource) {
|
|
13455
|
-
this.installEventListeners(resource);
|
|
13456
|
-
}
|
|
13457
|
-
});
|
|
13458
|
-
database.on("s3db.resourceUpdated", (resourceName) => {
|
|
14357
|
+
for (const resourceName in database.resources) {
|
|
13459
14358
|
const resource = database.resources[resourceName];
|
|
13460
|
-
|
|
13461
|
-
|
|
13462
|
-
|
|
13463
|
-
|
|
14359
|
+
this.installEventListeners(resource, database, this);
|
|
14360
|
+
}
|
|
14361
|
+
}
|
|
14362
|
+
createReplicator(driver, config, resources, client) {
|
|
14363
|
+
return createReplicator(driver, config, resources, client);
|
|
13464
14364
|
}
|
|
13465
|
-
async initializeReplicators() {
|
|
13466
|
-
console.log("[PLUGIN][INIT] initializeReplicators called");
|
|
14365
|
+
async initializeReplicators(database) {
|
|
13467
14366
|
for (const replicatorConfig of this.config.replicators) {
|
|
13468
|
-
|
|
13469
|
-
|
|
13470
|
-
|
|
13471
|
-
|
|
13472
|
-
|
|
13473
|
-
|
|
13474
|
-
|
|
13475
|
-
await replicator.initialize(this.database);
|
|
13476
|
-
this.replicators.push({
|
|
13477
|
-
id: Math.random().toString(36).slice(2),
|
|
13478
|
-
driver,
|
|
13479
|
-
config: replicatorConfig,
|
|
13480
|
-
resources,
|
|
13481
|
-
instance: replicator
|
|
13482
|
-
});
|
|
13483
|
-
console.log("[PLUGIN][INIT] pushed replicator:", driver, resources);
|
|
13484
|
-
} else {
|
|
13485
|
-
console.log("[PLUGIN][INIT] createReplicator returned null/undefined for driver:", driver);
|
|
13486
|
-
}
|
|
13487
|
-
} catch (err) {
|
|
13488
|
-
console.error("[PLUGIN][INIT] Error creating replicator:", err);
|
|
14367
|
+
const { driver, config = {}, resources, client, ...otherConfig } = replicatorConfig;
|
|
14368
|
+
const replicatorResources = resources || config.resources || {};
|
|
14369
|
+
const mergedConfig = { ...config, ...otherConfig };
|
|
14370
|
+
const replicator = this.createReplicator(driver, mergedConfig, replicatorResources, client);
|
|
14371
|
+
if (replicator) {
|
|
14372
|
+
await replicator.initialize(database);
|
|
14373
|
+
this.replicators.push(replicator);
|
|
13489
14374
|
}
|
|
13490
14375
|
}
|
|
13491
14376
|
}
|
|
@@ -13493,160 +14378,162 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
13493
14378
|
}
|
|
13494
14379
|
async stop() {
|
|
13495
14380
|
}
|
|
13496
|
-
|
|
13497
|
-
if (
|
|
13498
|
-
|
|
13499
|
-
|
|
14381
|
+
filterInternalFields(data) {
|
|
14382
|
+
if (!data || typeof data !== "object") return data;
|
|
14383
|
+
const filtered = {};
|
|
14384
|
+
for (const [key, value] of Object.entries(data)) {
|
|
14385
|
+
if (!key.startsWith("_") && !key.startsWith("$")) {
|
|
14386
|
+
filtered[key] = value;
|
|
14387
|
+
}
|
|
13500
14388
|
}
|
|
13501
|
-
|
|
13502
|
-
|
|
14389
|
+
return filtered;
|
|
14390
|
+
}
|
|
14391
|
+
async uploadMetadataFile(database) {
|
|
14392
|
+
if (typeof database.uploadMetadataFile === "function") {
|
|
14393
|
+
await database.uploadMetadataFile();
|
|
13503
14394
|
}
|
|
13504
|
-
|
|
13505
|
-
|
|
14395
|
+
}
|
|
14396
|
+
async getCompleteData(resource, data) {
|
|
14397
|
+
try {
|
|
14398
|
+
const [ok, err, record] = await try_fn_default(() => resource.get(data.id));
|
|
14399
|
+
if (ok && record) {
|
|
14400
|
+
return record;
|
|
14401
|
+
}
|
|
14402
|
+
} catch (error) {
|
|
13506
14403
|
}
|
|
13507
|
-
|
|
13508
|
-
|
|
13509
|
-
|
|
14404
|
+
return data;
|
|
14405
|
+
}
|
|
14406
|
+
async retryWithBackoff(operation, maxRetries = 3) {
|
|
14407
|
+
let lastError;
|
|
14408
|
+
for (let attempt = 1; attempt <= maxRetries; attempt++) {
|
|
14409
|
+
try {
|
|
14410
|
+
return await operation();
|
|
14411
|
+
} catch (error) {
|
|
14412
|
+
lastError = error;
|
|
14413
|
+
if (attempt === maxRetries) {
|
|
14414
|
+
throw error;
|
|
14415
|
+
}
|
|
14416
|
+
const delay = Math.pow(2, attempt - 1) * 1e3;
|
|
14417
|
+
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
13510
14418
|
}
|
|
13511
|
-
return;
|
|
13512
14419
|
}
|
|
13513
|
-
|
|
13514
|
-
|
|
13515
|
-
|
|
13516
|
-
|
|
14420
|
+
throw lastError;
|
|
14421
|
+
}
|
|
14422
|
+
async logError(replicator, resourceName, operation, recordId, data, error) {
|
|
14423
|
+
try {
|
|
14424
|
+
const logResourceName = this.config.replicatorLogResource;
|
|
14425
|
+
if (this.database && this.database.resources && this.database.resources[logResourceName]) {
|
|
14426
|
+
const logResource = this.database.resources[logResourceName];
|
|
14427
|
+
await logResource.insert({
|
|
14428
|
+
replicator: replicator.name || replicator.id,
|
|
14429
|
+
resourceName,
|
|
14430
|
+
operation,
|
|
14431
|
+
recordId,
|
|
14432
|
+
data: JSON.stringify(data),
|
|
14433
|
+
error: error.message,
|
|
14434
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
14435
|
+
status: "error"
|
|
14436
|
+
});
|
|
13517
14437
|
}
|
|
14438
|
+
} catch (logError) {
|
|
14439
|
+
}
|
|
14440
|
+
}
|
|
14441
|
+
async processReplicatorEvent(operation, resourceName, recordId, data, beforeData = null) {
|
|
14442
|
+
if (!this.config.enabled) return;
|
|
14443
|
+
const applicableReplicators = this.replicators.filter((replicator) => {
|
|
14444
|
+
const should = replicator.shouldReplicateResource && replicator.shouldReplicateResource(resourceName, operation);
|
|
13518
14445
|
return should;
|
|
13519
14446
|
});
|
|
13520
|
-
if (this.config.verbose) {
|
|
13521
|
-
console.log(`[PLUGIN] processReplicatorEvent: applicableReplicators for resource=${resourceName}:`, applicableReplicators.map((r) => r.driver));
|
|
13522
|
-
}
|
|
13523
14447
|
if (applicableReplicators.length === 0) {
|
|
13524
|
-
if (this.config.verbose) {
|
|
13525
|
-
console.log("[PLUGIN] No applicable replicators for resource", resourceName);
|
|
13526
|
-
}
|
|
13527
14448
|
return;
|
|
13528
14449
|
}
|
|
13529
|
-
const
|
|
13530
|
-
|
|
13531
|
-
|
|
13532
|
-
|
|
13533
|
-
|
|
13534
|
-
|
|
13535
|
-
|
|
13536
|
-
|
|
13537
|
-
|
|
13538
|
-
|
|
13539
|
-
|
|
13540
|
-
|
|
13541
|
-
|
|
13542
|
-
const [ok, err, result] = await try_fn_default(async () => this.processreplicatorItem(item));
|
|
13543
|
-
if (ok) {
|
|
13544
|
-
if (logId) {
|
|
13545
|
-
await this.updatereplicatorLog(logId, {
|
|
13546
|
-
status: result.success ? "success" : "failed",
|
|
13547
|
-
attempts: 1,
|
|
13548
|
-
error: result.success ? "" : JSON.stringify(result.results)
|
|
14450
|
+
const promises = applicableReplicators.map(async (replicator) => {
|
|
14451
|
+
try {
|
|
14452
|
+
const result = await this.retryWithBackoff(
|
|
14453
|
+
() => replicator.replicate(resourceName, operation, data, recordId, beforeData),
|
|
14454
|
+
this.config.maxRetries
|
|
14455
|
+
);
|
|
14456
|
+
this.emit("replicated", {
|
|
14457
|
+
replicator: replicator.name || replicator.id,
|
|
14458
|
+
resourceName,
|
|
14459
|
+
operation,
|
|
14460
|
+
recordId,
|
|
14461
|
+
result,
|
|
14462
|
+
success: true
|
|
13549
14463
|
});
|
|
13550
|
-
|
|
13551
|
-
|
|
13552
|
-
|
|
13553
|
-
|
|
13554
|
-
|
|
13555
|
-
|
|
13556
|
-
|
|
13557
|
-
|
|
13558
|
-
if (logId) {
|
|
13559
|
-
await this.updatereplicatorLog(logId, {
|
|
13560
|
-
status: "failed",
|
|
13561
|
-
attempts: 1,
|
|
13562
|
-
error: err.message
|
|
14464
|
+
return result;
|
|
14465
|
+
} catch (error) {
|
|
14466
|
+
this.emit("replicator_error", {
|
|
14467
|
+
replicator: replicator.name || replicator.id,
|
|
14468
|
+
resourceName,
|
|
14469
|
+
operation,
|
|
14470
|
+
recordId,
|
|
14471
|
+
error: error.message
|
|
13563
14472
|
});
|
|
14473
|
+
if (this.config.logErrors && this.database) {
|
|
14474
|
+
await this.logError(replicator, resourceName, operation, recordId, data, error);
|
|
14475
|
+
}
|
|
14476
|
+
throw error;
|
|
13564
14477
|
}
|
|
13565
|
-
|
|
13566
|
-
|
|
14478
|
+
});
|
|
14479
|
+
return Promise.allSettled(promises);
|
|
13567
14480
|
}
|
|
13568
14481
|
async processreplicatorItem(item) {
|
|
13569
|
-
if (this.config.verbose) {
|
|
13570
|
-
console.log("[PLUGIN][processreplicatorItem] called with item:", item);
|
|
13571
|
-
}
|
|
13572
14482
|
const applicableReplicators = this.replicators.filter((replicator) => {
|
|
13573
|
-
const should = replicator.
|
|
13574
|
-
if (this.config.verbose) {
|
|
13575
|
-
console.log(`[PLUGIN] processreplicatorItem: Replicator ${replicator.driver} shouldReplicateResource(${item.resourceName}, ${item.operation}):`, should);
|
|
13576
|
-
}
|
|
14483
|
+
const should = replicator.shouldReplicateResource && replicator.shouldReplicateResource(item.resourceName, item.operation);
|
|
13577
14484
|
return should;
|
|
13578
14485
|
});
|
|
13579
|
-
if (this.config.verbose) {
|
|
13580
|
-
console.log(`[PLUGIN] processreplicatorItem: applicableReplicators for resource=${item.resourceName}:`, applicableReplicators.map((r) => r.driver));
|
|
13581
|
-
}
|
|
13582
14486
|
if (applicableReplicators.length === 0) {
|
|
13583
|
-
|
|
13584
|
-
console.log("[PLUGIN] processreplicatorItem: No applicable replicators for resource", item.resourceName);
|
|
13585
|
-
}
|
|
13586
|
-
return { success: true, skipped: true, reason: "no_applicable_replicators" };
|
|
14487
|
+
return;
|
|
13587
14488
|
}
|
|
13588
|
-
const
|
|
13589
|
-
|
|
13590
|
-
|
|
13591
|
-
|
|
13592
|
-
|
|
13593
|
-
|
|
13594
|
-
|
|
14489
|
+
const promises = applicableReplicators.map(async (replicator) => {
|
|
14490
|
+
try {
|
|
14491
|
+
const [ok, err, result] = await try_fn_default(
|
|
14492
|
+
() => replicator.replicate(item.resourceName, item.operation, item.data, item.recordId, item.beforeData)
|
|
14493
|
+
);
|
|
14494
|
+
if (!ok) {
|
|
14495
|
+
this.emit("replicator_error", {
|
|
14496
|
+
replicator: replicator.name || replicator.id,
|
|
14497
|
+
resourceName: item.resourceName,
|
|
14498
|
+
operation: item.operation,
|
|
14499
|
+
recordId: item.recordId,
|
|
14500
|
+
error: err.message
|
|
14501
|
+
});
|
|
14502
|
+
if (this.config.logErrors && this.database) {
|
|
14503
|
+
await this.logError(replicator, item.resourceName, item.operation, item.recordId, item.data, err);
|
|
14504
|
+
}
|
|
14505
|
+
return { success: false, error: err.message };
|
|
14506
|
+
}
|
|
14507
|
+
this.emit("replicated", {
|
|
14508
|
+
replicator: replicator.name || replicator.id,
|
|
14509
|
+
resourceName: item.resourceName,
|
|
13595
14510
|
operation: item.operation,
|
|
13596
|
-
|
|
13597
|
-
|
|
13598
|
-
|
|
14511
|
+
recordId: item.recordId,
|
|
14512
|
+
result,
|
|
14513
|
+
success: true
|
|
13599
14514
|
});
|
|
14515
|
+
return { success: true, result };
|
|
14516
|
+
} catch (error) {
|
|
14517
|
+
this.emit("replicator_error", {
|
|
14518
|
+
replicator: replicator.name || replicator.id,
|
|
14519
|
+
resourceName: item.resourceName,
|
|
14520
|
+
operation: item.operation,
|
|
14521
|
+
recordId: item.recordId,
|
|
14522
|
+
error: error.message
|
|
14523
|
+
});
|
|
14524
|
+
if (this.config.logErrors && this.database) {
|
|
14525
|
+
await this.logError(replicator, item.resourceName, item.operation, item.recordId, item.data, error);
|
|
14526
|
+
}
|
|
14527
|
+
return { success: false, error: error.message };
|
|
13600
14528
|
}
|
|
13601
|
-
|
|
13602
|
-
|
|
13603
|
-
() => replicator.instance.replicate({
|
|
13604
|
-
resource: item.resourceName,
|
|
13605
|
-
operation: item.operation,
|
|
13606
|
-
data: item.data,
|
|
13607
|
-
id: item.recordId,
|
|
13608
|
-
beforeData: item.beforeData
|
|
13609
|
-
})
|
|
13610
|
-
);
|
|
13611
|
-
} else {
|
|
13612
|
-
[ok, err, result] = await try_fn_default(
|
|
13613
|
-
() => replicator.instance.replicate(
|
|
13614
|
-
item.resourceName,
|
|
13615
|
-
item.operation,
|
|
13616
|
-
item.data,
|
|
13617
|
-
item.recordId,
|
|
13618
|
-
item.beforeData
|
|
13619
|
-
)
|
|
13620
|
-
);
|
|
13621
|
-
}
|
|
13622
|
-
results.push({
|
|
13623
|
-
replicatorId: replicator.id,
|
|
13624
|
-
driver: replicator.driver,
|
|
13625
|
-
success: result && result.success,
|
|
13626
|
-
error: result && result.error,
|
|
13627
|
-
skipped: result && result.skipped
|
|
13628
|
-
});
|
|
13629
|
-
}
|
|
13630
|
-
return {
|
|
13631
|
-
success: results.every((r) => r.success || r.skipped),
|
|
13632
|
-
results
|
|
13633
|
-
};
|
|
14529
|
+
});
|
|
14530
|
+
return Promise.allSettled(promises);
|
|
13634
14531
|
}
|
|
13635
14532
|
async logreplicator(item) {
|
|
13636
14533
|
const logRes = this.replicatorLog || this.database.resources[normalizeResourceName(this.config.replicatorLogResource)];
|
|
13637
14534
|
if (!logRes) {
|
|
13638
|
-
if (this.config.verbose) {
|
|
13639
|
-
console.error("[PLUGIN] replicator log resource not found!");
|
|
13640
|
-
}
|
|
13641
14535
|
if (this.database) {
|
|
13642
|
-
if (this.
|
|
13643
|
-
console.warn("[PLUGIN] database.resources keys:", Object.keys(this.database.resources));
|
|
13644
|
-
}
|
|
13645
|
-
if (this.database.options && this.database.options.connectionString) {
|
|
13646
|
-
if (this.config.verbose) {
|
|
13647
|
-
console.warn("[PLUGIN] database connectionString:", this.database.options.connectionString);
|
|
13648
|
-
}
|
|
13649
|
-
}
|
|
14536
|
+
if (this.database.options && this.database.options.connectionString) ;
|
|
13650
14537
|
}
|
|
13651
14538
|
this.emit("replicator.log.failed", { error: "replicator log resource not found", item });
|
|
13652
14539
|
return;
|
|
@@ -13662,9 +14549,6 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
13662
14549
|
try {
|
|
13663
14550
|
await logRes.insert(logItem);
|
|
13664
14551
|
} catch (err) {
|
|
13665
|
-
if (this.config.verbose) {
|
|
13666
|
-
console.error("[PLUGIN] Error writing to replicator log:", err);
|
|
13667
|
-
}
|
|
13668
14552
|
this.emit("replicator.log.failed", { error: err, item });
|
|
13669
14553
|
}
|
|
13670
14554
|
}
|
|
@@ -13684,7 +14568,7 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
13684
14568
|
async getreplicatorStats() {
|
|
13685
14569
|
const replicatorStats = await Promise.all(
|
|
13686
14570
|
this.replicators.map(async (replicator) => {
|
|
13687
|
-
const status = await replicator.
|
|
14571
|
+
const status = await replicator.getStatus();
|
|
13688
14572
|
return {
|
|
13689
14573
|
id: replicator.id,
|
|
13690
14574
|
driver: replicator.driver,
|
|
@@ -13746,10 +14630,6 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
13746
14630
|
});
|
|
13747
14631
|
if (ok) {
|
|
13748
14632
|
retried++;
|
|
13749
|
-
} else {
|
|
13750
|
-
if (this.config.verbose) {
|
|
13751
|
-
console.error("Failed to retry replicator:", err);
|
|
13752
|
-
}
|
|
13753
14633
|
}
|
|
13754
14634
|
}
|
|
13755
14635
|
return { retried };
|
|
@@ -13762,52 +14642,35 @@ ${JSON.stringify(validation, null, 2)}`,
|
|
|
13762
14642
|
this.stats.lastSync = (/* @__PURE__ */ new Date()).toISOString();
|
|
13763
14643
|
for (const resourceName in this.database.resources) {
|
|
13764
14644
|
if (normalizeResourceName(resourceName) === normalizeResourceName("replicator_logs")) continue;
|
|
13765
|
-
if (replicator.
|
|
14645
|
+
if (replicator.shouldReplicateResource(resourceName)) {
|
|
13766
14646
|
this.emit("replicator.sync.resource", { resourceName, replicatorId });
|
|
13767
14647
|
const resource = this.database.resources[resourceName];
|
|
13768
14648
|
const allRecords = await resource.getAll();
|
|
13769
14649
|
for (const record of allRecords) {
|
|
13770
|
-
await replicator.
|
|
14650
|
+
await replicator.replicate(resourceName, "insert", record, record.id);
|
|
13771
14651
|
}
|
|
13772
14652
|
}
|
|
13773
14653
|
}
|
|
13774
14654
|
this.emit("replicator.sync.completed", { replicatorId, stats: this.stats });
|
|
13775
14655
|
}
|
|
13776
14656
|
async cleanup() {
|
|
13777
|
-
|
|
13778
|
-
|
|
13779
|
-
|
|
13780
|
-
|
|
13781
|
-
|
|
13782
|
-
|
|
13783
|
-
|
|
13784
|
-
|
|
13785
|
-
|
|
13786
|
-
}
|
|
13787
|
-
|
|
13788
|
-
}
|
|
13789
|
-
this._installedListeners = [];
|
|
13790
|
-
}
|
|
13791
|
-
if (this.database && typeof this.database.removeAllListeners === "function") {
|
|
13792
|
-
this.database.removeAllListeners();
|
|
13793
|
-
}
|
|
13794
|
-
if (this.replicators && Array.isArray(this.replicators)) {
|
|
13795
|
-
for (const rep of this.replicators) {
|
|
13796
|
-
if (rep.instance && typeof rep.instance.cleanup === "function") {
|
|
13797
|
-
await rep.instance.cleanup();
|
|
13798
|
-
}
|
|
14657
|
+
try {
|
|
14658
|
+
if (this.replicators && this.replicators.length > 0) {
|
|
14659
|
+
const cleanupPromises = this.replicators.map(async (replicator) => {
|
|
14660
|
+
try {
|
|
14661
|
+
if (replicator && typeof replicator.cleanup === "function") {
|
|
14662
|
+
await replicator.cleanup();
|
|
14663
|
+
}
|
|
14664
|
+
} catch (error) {
|
|
14665
|
+
}
|
|
14666
|
+
});
|
|
14667
|
+
await Promise.allSettled(cleanupPromises);
|
|
13799
14668
|
}
|
|
13800
14669
|
this.replicators = [];
|
|
13801
|
-
|
|
13802
|
-
|
|
13803
|
-
|
|
13804
|
-
|
|
13805
|
-
totalOperations: 0,
|
|
13806
|
-
totalErrors: 0,
|
|
13807
|
-
lastError: null
|
|
13808
|
-
};
|
|
13809
|
-
if (this.config.verbose) {
|
|
13810
|
-
console.log("[PLUGIN][CLEANUP] ReplicatorPlugin cleanup complete");
|
|
14670
|
+
this.database = null;
|
|
14671
|
+
this.eventListenersInstalled.clear();
|
|
14672
|
+
this.removeAllListeners();
|
|
14673
|
+
} catch (error) {
|
|
13811
14674
|
}
|
|
13812
14675
|
}
|
|
13813
14676
|
}
|