s3db.js 7.2.1 → 7.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/s3db.cjs.js +1241 -378
- package/dist/s3db.cjs.min.js +1 -1
- package/dist/s3db.es.js +1242 -379
- package/dist/s3db.es.min.js +1 -1
- package/dist/s3db.iife.js +1240 -377
- package/dist/s3db.iife.min.js +1 -1
- package/package.json +30 -24
- package/src/behaviors/body-only.js +2 -2
- package/src/behaviors/truncate-data.js +2 -2
- package/src/client.class.js +1 -1
- package/src/database.class.js +1 -1
- package/src/errors.js +1 -1
- package/src/plugins/audit.plugin.js +5 -5
- package/src/plugins/cache/filesystem-cache.class.js +661 -0
- package/src/plugins/cache/index.js +4 -0
- package/src/plugins/cache/partition-aware-filesystem-cache.class.js +480 -0
- package/src/plugins/cache/s3-cache.class.js +1 -1
- package/src/plugins/cache.plugin.js +159 -9
- package/src/plugins/consumers/index.js +3 -3
- package/src/plugins/consumers/sqs-consumer.js +2 -2
- package/src/plugins/fulltext.plugin.js +5 -5
- package/src/plugins/metrics.plugin.js +2 -2
- package/src/plugins/queue-consumer.plugin.js +3 -3
- package/src/plugins/replicator.plugin.js +259 -362
- package/src/plugins/replicators/s3db-replicator.class.js +35 -19
- package/src/plugins/replicators/sqs-replicator.class.js +17 -5
- package/src/resource.class.js +14 -14
- package/src/schema.class.js +3 -3
package/dist/s3db.cjs.js
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
Object.defineProperty(exports, '__esModule', { value: true });
|
|
4
4
|
|
|
5
5
|
var nanoid = require('nanoid');
|
|
6
|
-
var zlib = require('zlib');
|
|
6
|
+
var zlib = require('node:zlib');
|
|
7
7
|
var promisePool = require('@supercharge/promise-pool');
|
|
8
8
|
var web = require('node:stream/web');
|
|
9
9
|
var lodashEs = require('lodash-es');
|
|
@@ -1345,7 +1345,8 @@ class AuditPlugin extends plugin_class_default {
|
|
|
1345
1345
|
version: "2.0"
|
|
1346
1346
|
})
|
|
1347
1347
|
};
|
|
1348
|
-
this.logAudit(auditRecord).catch(
|
|
1348
|
+
this.logAudit(auditRecord).catch(() => {
|
|
1349
|
+
});
|
|
1349
1350
|
});
|
|
1350
1351
|
resource.on("update", async (data) => {
|
|
1351
1352
|
const recordId = data.id;
|
|
@@ -1371,7 +1372,8 @@ class AuditPlugin extends plugin_class_default {
|
|
|
1371
1372
|
version: "2.0"
|
|
1372
1373
|
})
|
|
1373
1374
|
};
|
|
1374
|
-
this.logAudit(auditRecord).catch(
|
|
1375
|
+
this.logAudit(auditRecord).catch(() => {
|
|
1376
|
+
});
|
|
1375
1377
|
});
|
|
1376
1378
|
resource.on("delete", async (data) => {
|
|
1377
1379
|
const recordId = data.id;
|
|
@@ -1397,7 +1399,8 @@ class AuditPlugin extends plugin_class_default {
|
|
|
1397
1399
|
version: "2.0"
|
|
1398
1400
|
})
|
|
1399
1401
|
};
|
|
1400
|
-
this.logAudit(auditRecord).catch(
|
|
1402
|
+
this.logAudit(auditRecord).catch(() => {
|
|
1403
|
+
});
|
|
1401
1404
|
});
|
|
1402
1405
|
resource.useMiddleware("deleteMany", async (ctx, next) => {
|
|
1403
1406
|
const ids = ctx.args[0];
|
|
@@ -1430,7 +1433,8 @@ class AuditPlugin extends plugin_class_default {
|
|
|
1430
1433
|
batchOperation: true
|
|
1431
1434
|
})
|
|
1432
1435
|
};
|
|
1433
|
-
this.logAudit(auditRecord).catch(
|
|
1436
|
+
this.logAudit(auditRecord).catch(() => {
|
|
1437
|
+
});
|
|
1434
1438
|
}
|
|
1435
1439
|
}
|
|
1436
1440
|
return result;
|
|
@@ -2077,6 +2081,16 @@ if (typeof Object.create === 'function'){
|
|
|
2077
2081
|
// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
|
|
2078
2082
|
// USE OR OTHER DEALINGS IN THE SOFTWARE.
|
|
2079
2083
|
|
|
2084
|
+
var getOwnPropertyDescriptors = Object.getOwnPropertyDescriptors ||
|
|
2085
|
+
function getOwnPropertyDescriptors(obj) {
|
|
2086
|
+
var keys = Object.keys(obj);
|
|
2087
|
+
var descriptors = {};
|
|
2088
|
+
for (var i = 0; i < keys.length; i++) {
|
|
2089
|
+
descriptors[keys[i]] = Object.getOwnPropertyDescriptor(obj, keys[i]);
|
|
2090
|
+
}
|
|
2091
|
+
return descriptors;
|
|
2092
|
+
};
|
|
2093
|
+
|
|
2080
2094
|
var formatRegExp = /%[sdj%]/g;
|
|
2081
2095
|
function format(f) {
|
|
2082
2096
|
if (!isString(f)) {
|
|
@@ -2562,6 +2576,64 @@ function hasOwnProperty(obj, prop) {
|
|
|
2562
2576
|
return Object.prototype.hasOwnProperty.call(obj, prop);
|
|
2563
2577
|
}
|
|
2564
2578
|
|
|
2579
|
+
var kCustomPromisifiedSymbol = typeof Symbol !== 'undefined' ? Symbol('util.promisify.custom') : undefined;
|
|
2580
|
+
|
|
2581
|
+
function promisify(original) {
|
|
2582
|
+
if (typeof original !== 'function')
|
|
2583
|
+
throw new TypeError('The "original" argument must be of type Function');
|
|
2584
|
+
|
|
2585
|
+
if (kCustomPromisifiedSymbol && original[kCustomPromisifiedSymbol]) {
|
|
2586
|
+
var fn = original[kCustomPromisifiedSymbol];
|
|
2587
|
+
if (typeof fn !== 'function') {
|
|
2588
|
+
throw new TypeError('The "util.promisify.custom" argument must be of type Function');
|
|
2589
|
+
}
|
|
2590
|
+
Object.defineProperty(fn, kCustomPromisifiedSymbol, {
|
|
2591
|
+
value: fn, enumerable: false, writable: false, configurable: true
|
|
2592
|
+
});
|
|
2593
|
+
return fn;
|
|
2594
|
+
}
|
|
2595
|
+
|
|
2596
|
+
function fn() {
|
|
2597
|
+
var promiseResolve, promiseReject;
|
|
2598
|
+
var promise = new Promise(function (resolve, reject) {
|
|
2599
|
+
promiseResolve = resolve;
|
|
2600
|
+
promiseReject = reject;
|
|
2601
|
+
});
|
|
2602
|
+
|
|
2603
|
+
var args = [];
|
|
2604
|
+
for (var i = 0; i < arguments.length; i++) {
|
|
2605
|
+
args.push(arguments[i]);
|
|
2606
|
+
}
|
|
2607
|
+
args.push(function (err, value) {
|
|
2608
|
+
if (err) {
|
|
2609
|
+
promiseReject(err);
|
|
2610
|
+
} else {
|
|
2611
|
+
promiseResolve(value);
|
|
2612
|
+
}
|
|
2613
|
+
});
|
|
2614
|
+
|
|
2615
|
+
try {
|
|
2616
|
+
original.apply(this, args);
|
|
2617
|
+
} catch (err) {
|
|
2618
|
+
promiseReject(err);
|
|
2619
|
+
}
|
|
2620
|
+
|
|
2621
|
+
return promise;
|
|
2622
|
+
}
|
|
2623
|
+
|
|
2624
|
+
Object.setPrototypeOf(fn, Object.getPrototypeOf(original));
|
|
2625
|
+
|
|
2626
|
+
if (kCustomPromisifiedSymbol) Object.defineProperty(fn, kCustomPromisifiedSymbol, {
|
|
2627
|
+
value: fn, enumerable: false, writable: false, configurable: true
|
|
2628
|
+
});
|
|
2629
|
+
return Object.defineProperties(
|
|
2630
|
+
fn,
|
|
2631
|
+
getOwnPropertyDescriptors(original)
|
|
2632
|
+
);
|
|
2633
|
+
}
|
|
2634
|
+
|
|
2635
|
+
promisify.custom = kCustomPromisifiedSymbol;
|
|
2636
|
+
|
|
2565
2637
|
var lookup = [];
|
|
2566
2638
|
var revLookup = [];
|
|
2567
2639
|
var Arr = typeof Uint8Array !== 'undefined' ? Uint8Array : Array;
|
|
@@ -6812,12 +6884,798 @@ class MemoryCache extends Cache {
|
|
|
6812
6884
|
}
|
|
6813
6885
|
var memory_cache_class_default = MemoryCache;
|
|
6814
6886
|
|
|
6887
|
+
var fs = {};
|
|
6888
|
+
|
|
6889
|
+
const readFile$1 = promisify(fs.readFile);
|
|
6890
|
+
const writeFile$1 = promisify(fs.writeFile);
|
|
6891
|
+
const unlink = promisify(fs.unlink);
|
|
6892
|
+
const readdir$1 = promisify(fs.readdir);
|
|
6893
|
+
const stat$1 = promisify(fs.stat);
|
|
6894
|
+
const mkdir = promisify(fs.mkdir);
|
|
6895
|
+
class FilesystemCache extends Cache {
|
|
6896
|
+
constructor({
|
|
6897
|
+
directory,
|
|
6898
|
+
prefix = "cache",
|
|
6899
|
+
ttl = 36e5,
|
|
6900
|
+
enableCompression = true,
|
|
6901
|
+
compressionThreshold = 1024,
|
|
6902
|
+
createDirectory = true,
|
|
6903
|
+
fileExtension = ".cache",
|
|
6904
|
+
enableMetadata = true,
|
|
6905
|
+
maxFileSize = 10485760,
|
|
6906
|
+
// 10MB
|
|
6907
|
+
enableStats = false,
|
|
6908
|
+
enableCleanup = true,
|
|
6909
|
+
cleanupInterval = 3e5,
|
|
6910
|
+
// 5 minutes
|
|
6911
|
+
encoding = "utf8",
|
|
6912
|
+
fileMode = 420,
|
|
6913
|
+
enableBackup = false,
|
|
6914
|
+
backupSuffix = ".bak",
|
|
6915
|
+
enableLocking = false,
|
|
6916
|
+
lockTimeout = 5e3,
|
|
6917
|
+
enableJournal = false,
|
|
6918
|
+
journalFile = "cache.journal",
|
|
6919
|
+
...config
|
|
6920
|
+
}) {
|
|
6921
|
+
super(config);
|
|
6922
|
+
if (!directory) {
|
|
6923
|
+
throw new Error("FilesystemCache: directory parameter is required");
|
|
6924
|
+
}
|
|
6925
|
+
this.directory = path.resolve(directory);
|
|
6926
|
+
this.prefix = prefix;
|
|
6927
|
+
this.ttl = ttl;
|
|
6928
|
+
this.enableCompression = enableCompression;
|
|
6929
|
+
this.compressionThreshold = compressionThreshold;
|
|
6930
|
+
this.createDirectory = createDirectory;
|
|
6931
|
+
this.fileExtension = fileExtension;
|
|
6932
|
+
this.enableMetadata = enableMetadata;
|
|
6933
|
+
this.maxFileSize = maxFileSize;
|
|
6934
|
+
this.enableStats = enableStats;
|
|
6935
|
+
this.enableCleanup = enableCleanup;
|
|
6936
|
+
this.cleanupInterval = cleanupInterval;
|
|
6937
|
+
this.encoding = encoding;
|
|
6938
|
+
this.fileMode = fileMode;
|
|
6939
|
+
this.enableBackup = enableBackup;
|
|
6940
|
+
this.backupSuffix = backupSuffix;
|
|
6941
|
+
this.enableLocking = enableLocking;
|
|
6942
|
+
this.lockTimeout = lockTimeout;
|
|
6943
|
+
this.enableJournal = enableJournal;
|
|
6944
|
+
this.journalFile = path.join(this.directory, journalFile);
|
|
6945
|
+
this.stats = {
|
|
6946
|
+
hits: 0,
|
|
6947
|
+
misses: 0,
|
|
6948
|
+
sets: 0,
|
|
6949
|
+
deletes: 0,
|
|
6950
|
+
clears: 0,
|
|
6951
|
+
errors: 0
|
|
6952
|
+
};
|
|
6953
|
+
this.locks = /* @__PURE__ */ new Map();
|
|
6954
|
+
this.cleanupTimer = null;
|
|
6955
|
+
this._init();
|
|
6956
|
+
}
|
|
6957
|
+
async _init() {
|
|
6958
|
+
if (this.createDirectory) {
|
|
6959
|
+
await this._ensureDirectory(this.directory);
|
|
6960
|
+
}
|
|
6961
|
+
if (this.enableCleanup && this.cleanupInterval > 0) {
|
|
6962
|
+
this.cleanupTimer = setInterval(() => {
|
|
6963
|
+
this._cleanup().catch((err) => {
|
|
6964
|
+
console.warn("FilesystemCache cleanup error:", err.message);
|
|
6965
|
+
});
|
|
6966
|
+
}, this.cleanupInterval);
|
|
6967
|
+
}
|
|
6968
|
+
}
|
|
6969
|
+
async _ensureDirectory(dir) {
|
|
6970
|
+
const [ok, err] = await try_fn_default(async () => {
|
|
6971
|
+
await mkdir(dir, { recursive: true });
|
|
6972
|
+
});
|
|
6973
|
+
if (!ok && err.code !== "EEXIST") {
|
|
6974
|
+
throw new Error(`Failed to create cache directory: ${err.message}`);
|
|
6975
|
+
}
|
|
6976
|
+
}
|
|
6977
|
+
_getFilePath(key) {
|
|
6978
|
+
const sanitizedKey = key.replace(/[<>:"/\\|?*]/g, "_");
|
|
6979
|
+
const filename = `${this.prefix}_${sanitizedKey}${this.fileExtension}`;
|
|
6980
|
+
return path.join(this.directory, filename);
|
|
6981
|
+
}
|
|
6982
|
+
_getMetadataPath(filePath) {
|
|
6983
|
+
return filePath + ".meta";
|
|
6984
|
+
}
|
|
6985
|
+
async _set(key, data) {
|
|
6986
|
+
const filePath = this._getFilePath(key);
|
|
6987
|
+
try {
|
|
6988
|
+
let serialized = JSON.stringify(data);
|
|
6989
|
+
const originalSize = Buffer.byteLength(serialized, this.encoding);
|
|
6990
|
+
if (originalSize > this.maxFileSize) {
|
|
6991
|
+
throw new Error(`Cache data exceeds maximum file size: ${originalSize} > ${this.maxFileSize}`);
|
|
6992
|
+
}
|
|
6993
|
+
let compressed = false;
|
|
6994
|
+
let finalData = serialized;
|
|
6995
|
+
if (this.enableCompression && originalSize >= this.compressionThreshold) {
|
|
6996
|
+
const compressedBuffer = zlib.gzipSync(Buffer.from(serialized, this.encoding));
|
|
6997
|
+
finalData = compressedBuffer.toString("base64");
|
|
6998
|
+
compressed = true;
|
|
6999
|
+
}
|
|
7000
|
+
if (this.enableBackup && await this._fileExists(filePath)) {
|
|
7001
|
+
const backupPath = filePath + this.backupSuffix;
|
|
7002
|
+
await this._copyFile(filePath, backupPath);
|
|
7003
|
+
}
|
|
7004
|
+
if (this.enableLocking) {
|
|
7005
|
+
await this._acquireLock(filePath);
|
|
7006
|
+
}
|
|
7007
|
+
try {
|
|
7008
|
+
await writeFile$1(filePath, finalData, {
|
|
7009
|
+
encoding: compressed ? "utf8" : this.encoding,
|
|
7010
|
+
mode: this.fileMode
|
|
7011
|
+
});
|
|
7012
|
+
if (this.enableMetadata) {
|
|
7013
|
+
const metadata = {
|
|
7014
|
+
key,
|
|
7015
|
+
timestamp: Date.now(),
|
|
7016
|
+
ttl: this.ttl,
|
|
7017
|
+
compressed,
|
|
7018
|
+
originalSize,
|
|
7019
|
+
compressedSize: compressed ? Buffer.byteLength(finalData, "utf8") : originalSize,
|
|
7020
|
+
compressionRatio: compressed ? (Buffer.byteLength(finalData, "utf8") / originalSize).toFixed(2) : 1
|
|
7021
|
+
};
|
|
7022
|
+
await writeFile$1(this._getMetadataPath(filePath), JSON.stringify(metadata), {
|
|
7023
|
+
encoding: this.encoding,
|
|
7024
|
+
mode: this.fileMode
|
|
7025
|
+
});
|
|
7026
|
+
}
|
|
7027
|
+
if (this.enableStats) {
|
|
7028
|
+
this.stats.sets++;
|
|
7029
|
+
}
|
|
7030
|
+
if (this.enableJournal) {
|
|
7031
|
+
await this._journalOperation("set", key, { size: originalSize, compressed });
|
|
7032
|
+
}
|
|
7033
|
+
} finally {
|
|
7034
|
+
if (this.enableLocking) {
|
|
7035
|
+
this._releaseLock(filePath);
|
|
7036
|
+
}
|
|
7037
|
+
}
|
|
7038
|
+
return data;
|
|
7039
|
+
} catch (error) {
|
|
7040
|
+
if (this.enableStats) {
|
|
7041
|
+
this.stats.errors++;
|
|
7042
|
+
}
|
|
7043
|
+
throw new Error(`Failed to set cache key '${key}': ${error.message}`);
|
|
7044
|
+
}
|
|
7045
|
+
}
|
|
7046
|
+
async _get(key) {
|
|
7047
|
+
const filePath = this._getFilePath(key);
|
|
7048
|
+
try {
|
|
7049
|
+
if (!await this._fileExists(filePath)) {
|
|
7050
|
+
if (this.enableStats) {
|
|
7051
|
+
this.stats.misses++;
|
|
7052
|
+
}
|
|
7053
|
+
return null;
|
|
7054
|
+
}
|
|
7055
|
+
let isExpired = false;
|
|
7056
|
+
if (this.enableMetadata) {
|
|
7057
|
+
const metadataPath = this._getMetadataPath(filePath);
|
|
7058
|
+
if (await this._fileExists(metadataPath)) {
|
|
7059
|
+
const [ok, err, metadata] = await try_fn_default(async () => {
|
|
7060
|
+
const metaContent = await readFile$1(metadataPath, this.encoding);
|
|
7061
|
+
return JSON.parse(metaContent);
|
|
7062
|
+
});
|
|
7063
|
+
if (ok && metadata.ttl > 0) {
|
|
7064
|
+
const age = Date.now() - metadata.timestamp;
|
|
7065
|
+
isExpired = age > metadata.ttl;
|
|
7066
|
+
}
|
|
7067
|
+
}
|
|
7068
|
+
} else if (this.ttl > 0) {
|
|
7069
|
+
const stats = await stat$1(filePath);
|
|
7070
|
+
const age = Date.now() - stats.mtime.getTime();
|
|
7071
|
+
isExpired = age > this.ttl;
|
|
7072
|
+
}
|
|
7073
|
+
if (isExpired) {
|
|
7074
|
+
await this._del(key);
|
|
7075
|
+
if (this.enableStats) {
|
|
7076
|
+
this.stats.misses++;
|
|
7077
|
+
}
|
|
7078
|
+
return null;
|
|
7079
|
+
}
|
|
7080
|
+
if (this.enableLocking) {
|
|
7081
|
+
await this._acquireLock(filePath);
|
|
7082
|
+
}
|
|
7083
|
+
try {
|
|
7084
|
+
const content = await readFile$1(filePath, this.encoding);
|
|
7085
|
+
let isCompressed = false;
|
|
7086
|
+
if (this.enableMetadata) {
|
|
7087
|
+
const metadataPath = this._getMetadataPath(filePath);
|
|
7088
|
+
if (await this._fileExists(metadataPath)) {
|
|
7089
|
+
const [ok, err, metadata] = await try_fn_default(async () => {
|
|
7090
|
+
const metaContent = await readFile$1(metadataPath, this.encoding);
|
|
7091
|
+
return JSON.parse(metaContent);
|
|
7092
|
+
});
|
|
7093
|
+
if (ok) {
|
|
7094
|
+
isCompressed = metadata.compressed;
|
|
7095
|
+
}
|
|
7096
|
+
}
|
|
7097
|
+
}
|
|
7098
|
+
let finalContent = content;
|
|
7099
|
+
if (isCompressed || this.enableCompression && content.match(/^[A-Za-z0-9+/=]+$/)) {
|
|
7100
|
+
try {
|
|
7101
|
+
const compressedBuffer = Buffer.from(content, "base64");
|
|
7102
|
+
finalContent = zlib.gunzipSync(compressedBuffer).toString(this.encoding);
|
|
7103
|
+
} catch (decompressError) {
|
|
7104
|
+
finalContent = content;
|
|
7105
|
+
}
|
|
7106
|
+
}
|
|
7107
|
+
const data = JSON.parse(finalContent);
|
|
7108
|
+
if (this.enableStats) {
|
|
7109
|
+
this.stats.hits++;
|
|
7110
|
+
}
|
|
7111
|
+
return data;
|
|
7112
|
+
} finally {
|
|
7113
|
+
if (this.enableLocking) {
|
|
7114
|
+
this._releaseLock(filePath);
|
|
7115
|
+
}
|
|
7116
|
+
}
|
|
7117
|
+
} catch (error) {
|
|
7118
|
+
if (this.enableStats) {
|
|
7119
|
+
this.stats.errors++;
|
|
7120
|
+
}
|
|
7121
|
+
await this._del(key);
|
|
7122
|
+
return null;
|
|
7123
|
+
}
|
|
7124
|
+
}
|
|
7125
|
+
async _del(key) {
|
|
7126
|
+
const filePath = this._getFilePath(key);
|
|
7127
|
+
try {
|
|
7128
|
+
if (await this._fileExists(filePath)) {
|
|
7129
|
+
await unlink(filePath);
|
|
7130
|
+
}
|
|
7131
|
+
if (this.enableMetadata) {
|
|
7132
|
+
const metadataPath = this._getMetadataPath(filePath);
|
|
7133
|
+
if (await this._fileExists(metadataPath)) {
|
|
7134
|
+
await unlink(metadataPath);
|
|
7135
|
+
}
|
|
7136
|
+
}
|
|
7137
|
+
if (this.enableBackup) {
|
|
7138
|
+
const backupPath = filePath + this.backupSuffix;
|
|
7139
|
+
if (await this._fileExists(backupPath)) {
|
|
7140
|
+
await unlink(backupPath);
|
|
7141
|
+
}
|
|
7142
|
+
}
|
|
7143
|
+
if (this.enableStats) {
|
|
7144
|
+
this.stats.deletes++;
|
|
7145
|
+
}
|
|
7146
|
+
if (this.enableJournal) {
|
|
7147
|
+
await this._journalOperation("delete", key);
|
|
7148
|
+
}
|
|
7149
|
+
return true;
|
|
7150
|
+
} catch (error) {
|
|
7151
|
+
if (this.enableStats) {
|
|
7152
|
+
this.stats.errors++;
|
|
7153
|
+
}
|
|
7154
|
+
throw new Error(`Failed to delete cache key '${key}': ${error.message}`);
|
|
7155
|
+
}
|
|
7156
|
+
}
|
|
7157
|
+
async _clear(prefix) {
|
|
7158
|
+
try {
|
|
7159
|
+
const files = await readdir$1(this.directory);
|
|
7160
|
+
const cacheFiles = files.filter((file) => {
|
|
7161
|
+
if (!file.startsWith(this.prefix)) return false;
|
|
7162
|
+
if (!file.endsWith(this.fileExtension)) return false;
|
|
7163
|
+
if (prefix) {
|
|
7164
|
+
const keyPart = file.slice(this.prefix.length + 1, -this.fileExtension.length);
|
|
7165
|
+
return keyPart.startsWith(prefix);
|
|
7166
|
+
}
|
|
7167
|
+
return true;
|
|
7168
|
+
});
|
|
7169
|
+
for (const file of cacheFiles) {
|
|
7170
|
+
const filePath = path.join(this.directory, file);
|
|
7171
|
+
if (await this._fileExists(filePath)) {
|
|
7172
|
+
await unlink(filePath);
|
|
7173
|
+
}
|
|
7174
|
+
if (this.enableMetadata) {
|
|
7175
|
+
const metadataPath = this._getMetadataPath(filePath);
|
|
7176
|
+
if (await this._fileExists(metadataPath)) {
|
|
7177
|
+
await unlink(metadataPath);
|
|
7178
|
+
}
|
|
7179
|
+
}
|
|
7180
|
+
if (this.enableBackup) {
|
|
7181
|
+
const backupPath = filePath + this.backupSuffix;
|
|
7182
|
+
if (await this._fileExists(backupPath)) {
|
|
7183
|
+
await unlink(backupPath);
|
|
7184
|
+
}
|
|
7185
|
+
}
|
|
7186
|
+
}
|
|
7187
|
+
if (this.enableStats) {
|
|
7188
|
+
this.stats.clears++;
|
|
7189
|
+
}
|
|
7190
|
+
if (this.enableJournal) {
|
|
7191
|
+
await this._journalOperation("clear", prefix || "all", { count: cacheFiles.length });
|
|
7192
|
+
}
|
|
7193
|
+
return true;
|
|
7194
|
+
} catch (error) {
|
|
7195
|
+
if (this.enableStats) {
|
|
7196
|
+
this.stats.errors++;
|
|
7197
|
+
}
|
|
7198
|
+
throw new Error(`Failed to clear cache: ${error.message}`);
|
|
7199
|
+
}
|
|
7200
|
+
}
|
|
7201
|
+
async size() {
|
|
7202
|
+
const keys = await this.keys();
|
|
7203
|
+
return keys.length;
|
|
7204
|
+
}
|
|
7205
|
+
async keys() {
|
|
7206
|
+
try {
|
|
7207
|
+
const files = await readdir$1(this.directory);
|
|
7208
|
+
const cacheFiles = files.filter(
|
|
7209
|
+
(file) => file.startsWith(this.prefix) && file.endsWith(this.fileExtension)
|
|
7210
|
+
);
|
|
7211
|
+
const keys = cacheFiles.map((file) => {
|
|
7212
|
+
const keyPart = file.slice(this.prefix.length + 1, -this.fileExtension.length);
|
|
7213
|
+
return keyPart;
|
|
7214
|
+
});
|
|
7215
|
+
return keys;
|
|
7216
|
+
} catch (error) {
|
|
7217
|
+
console.warn("FilesystemCache: Failed to list keys:", error.message);
|
|
7218
|
+
return [];
|
|
7219
|
+
}
|
|
7220
|
+
}
|
|
7221
|
+
// Helper methods
|
|
7222
|
+
async _fileExists(filePath) {
|
|
7223
|
+
const [ok] = await try_fn_default(async () => {
|
|
7224
|
+
await stat$1(filePath);
|
|
7225
|
+
});
|
|
7226
|
+
return ok;
|
|
7227
|
+
}
|
|
7228
|
+
async _copyFile(src, dest) {
|
|
7229
|
+
const [ok, err] = await try_fn_default(async () => {
|
|
7230
|
+
const content = await readFile$1(src);
|
|
7231
|
+
await writeFile$1(dest, content);
|
|
7232
|
+
});
|
|
7233
|
+
if (!ok) {
|
|
7234
|
+
console.warn("FilesystemCache: Failed to create backup:", err.message);
|
|
7235
|
+
}
|
|
7236
|
+
}
|
|
7237
|
+
async _cleanup() {
|
|
7238
|
+
if (!this.ttl || this.ttl <= 0) return;
|
|
7239
|
+
try {
|
|
7240
|
+
const files = await readdir$1(this.directory);
|
|
7241
|
+
const now = Date.now();
|
|
7242
|
+
for (const file of files) {
|
|
7243
|
+
if (!file.startsWith(this.prefix) || !file.endsWith(this.fileExtension)) {
|
|
7244
|
+
continue;
|
|
7245
|
+
}
|
|
7246
|
+
const filePath = path.join(this.directory, file);
|
|
7247
|
+
let shouldDelete = false;
|
|
7248
|
+
if (this.enableMetadata) {
|
|
7249
|
+
const metadataPath = this._getMetadataPath(filePath);
|
|
7250
|
+
if (await this._fileExists(metadataPath)) {
|
|
7251
|
+
const [ok, err, metadata] = await try_fn_default(async () => {
|
|
7252
|
+
const metaContent = await readFile$1(metadataPath, this.encoding);
|
|
7253
|
+
return JSON.parse(metaContent);
|
|
7254
|
+
});
|
|
7255
|
+
if (ok && metadata.ttl > 0) {
|
|
7256
|
+
const age = now - metadata.timestamp;
|
|
7257
|
+
shouldDelete = age > metadata.ttl;
|
|
7258
|
+
}
|
|
7259
|
+
}
|
|
7260
|
+
} else {
|
|
7261
|
+
const [ok, err, stats] = await try_fn_default(async () => {
|
|
7262
|
+
return await stat$1(filePath);
|
|
7263
|
+
});
|
|
7264
|
+
if (ok) {
|
|
7265
|
+
const age = now - stats.mtime.getTime();
|
|
7266
|
+
shouldDelete = age > this.ttl;
|
|
7267
|
+
}
|
|
7268
|
+
}
|
|
7269
|
+
if (shouldDelete) {
|
|
7270
|
+
const keyPart = file.slice(this.prefix.length + 1, -this.fileExtension.length);
|
|
7271
|
+
await this._del(keyPart);
|
|
7272
|
+
}
|
|
7273
|
+
}
|
|
7274
|
+
} catch (error) {
|
|
7275
|
+
console.warn("FilesystemCache cleanup error:", error.message);
|
|
7276
|
+
}
|
|
7277
|
+
}
|
|
7278
|
+
async _acquireLock(filePath) {
|
|
7279
|
+
if (!this.enableLocking) return;
|
|
7280
|
+
const lockKey = filePath;
|
|
7281
|
+
const startTime = Date.now();
|
|
7282
|
+
while (this.locks.has(lockKey)) {
|
|
7283
|
+
if (Date.now() - startTime > this.lockTimeout) {
|
|
7284
|
+
throw new Error(`Lock timeout for file: ${filePath}`);
|
|
7285
|
+
}
|
|
7286
|
+
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
7287
|
+
}
|
|
7288
|
+
this.locks.set(lockKey, Date.now());
|
|
7289
|
+
}
|
|
7290
|
+
_releaseLock(filePath) {
|
|
7291
|
+
if (!this.enableLocking) return;
|
|
7292
|
+
this.locks.delete(filePath);
|
|
7293
|
+
}
|
|
7294
|
+
async _journalOperation(operation, key, metadata = {}) {
|
|
7295
|
+
if (!this.enableJournal) return;
|
|
7296
|
+
const entry = {
|
|
7297
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
7298
|
+
operation,
|
|
7299
|
+
key,
|
|
7300
|
+
metadata
|
|
7301
|
+
};
|
|
7302
|
+
const [ok, err] = await try_fn_default(async () => {
|
|
7303
|
+
const line = JSON.stringify(entry) + "\n";
|
|
7304
|
+
await fs.promises.appendFile(this.journalFile, line, this.encoding);
|
|
7305
|
+
});
|
|
7306
|
+
if (!ok) {
|
|
7307
|
+
console.warn("FilesystemCache journal error:", err.message);
|
|
7308
|
+
}
|
|
7309
|
+
}
|
|
7310
|
+
// Cleanup on process exit
|
|
7311
|
+
destroy() {
|
|
7312
|
+
if (this.cleanupTimer) {
|
|
7313
|
+
clearInterval(this.cleanupTimer);
|
|
7314
|
+
this.cleanupTimer = null;
|
|
7315
|
+
}
|
|
7316
|
+
}
|
|
7317
|
+
// Get cache statistics
|
|
7318
|
+
getStats() {
|
|
7319
|
+
return {
|
|
7320
|
+
...this.stats,
|
|
7321
|
+
directory: this.directory,
|
|
7322
|
+
ttl: this.ttl,
|
|
7323
|
+
compression: this.enableCompression,
|
|
7324
|
+
metadata: this.enableMetadata,
|
|
7325
|
+
cleanup: this.enableCleanup,
|
|
7326
|
+
locking: this.enableLocking,
|
|
7327
|
+
journal: this.enableJournal
|
|
7328
|
+
};
|
|
7329
|
+
}
|
|
7330
|
+
}
|
|
7331
|
+
|
|
7332
|
+
promisify(fs.mkdir);
|
|
7333
|
+
const rmdir = promisify(fs.rmdir);
|
|
7334
|
+
const readdir = promisify(fs.readdir);
|
|
7335
|
+
const stat = promisify(fs.stat);
|
|
7336
|
+
const writeFile = promisify(fs.writeFile);
|
|
7337
|
+
const readFile = promisify(fs.readFile);
|
|
7338
|
+
class PartitionAwareFilesystemCache extends FilesystemCache {
|
|
7339
|
+
constructor({
|
|
7340
|
+
partitionStrategy = "hierarchical",
|
|
7341
|
+
// 'hierarchical', 'flat', 'temporal'
|
|
7342
|
+
trackUsage = true,
|
|
7343
|
+
preloadRelated = false,
|
|
7344
|
+
preloadThreshold = 10,
|
|
7345
|
+
maxCacheSize = null,
|
|
7346
|
+
usageStatsFile = "partition-usage.json",
|
|
7347
|
+
...config
|
|
7348
|
+
}) {
|
|
7349
|
+
super(config);
|
|
7350
|
+
this.partitionStrategy = partitionStrategy;
|
|
7351
|
+
this.trackUsage = trackUsage;
|
|
7352
|
+
this.preloadRelated = preloadRelated;
|
|
7353
|
+
this.preloadThreshold = preloadThreshold;
|
|
7354
|
+
this.maxCacheSize = maxCacheSize;
|
|
7355
|
+
this.usageStatsFile = path.join(this.directory, usageStatsFile);
|
|
7356
|
+
this.partitionUsage = /* @__PURE__ */ new Map();
|
|
7357
|
+
this.loadUsageStats();
|
|
7358
|
+
}
|
|
7359
|
+
/**
|
|
7360
|
+
* Generate partition-aware cache key
|
|
7361
|
+
*/
|
|
7362
|
+
_getPartitionCacheKey(resource, action, partition, partitionValues = {}, params = {}) {
|
|
7363
|
+
const keyParts = [`resource=${resource}`, `action=${action}`];
|
|
7364
|
+
if (partition && Object.keys(partitionValues).length > 0) {
|
|
7365
|
+
keyParts.push(`partition=${partition}`);
|
|
7366
|
+
const sortedFields = Object.entries(partitionValues).sort(([a], [b]) => a.localeCompare(b));
|
|
7367
|
+
for (const [field, value] of sortedFields) {
|
|
7368
|
+
if (value !== null && value !== void 0) {
|
|
7369
|
+
keyParts.push(`${field}=${value}`);
|
|
7370
|
+
}
|
|
7371
|
+
}
|
|
7372
|
+
}
|
|
7373
|
+
if (Object.keys(params).length > 0) {
|
|
7374
|
+
const paramsStr = Object.entries(params).sort(([a], [b]) => a.localeCompare(b)).map(([k, v]) => `${k}=${v}`).join("|");
|
|
7375
|
+
keyParts.push(`params=${Buffer.from(paramsStr).toString("base64")}`);
|
|
7376
|
+
}
|
|
7377
|
+
return keyParts.join("/") + this.fileExtension;
|
|
7378
|
+
}
|
|
7379
|
+
/**
|
|
7380
|
+
* Get directory path for partition cache
|
|
7381
|
+
*/
|
|
7382
|
+
_getPartitionDirectory(resource, partition, partitionValues = {}) {
|
|
7383
|
+
const basePath = path.join(this.directory, `resource=${resource}`);
|
|
7384
|
+
if (!partition) {
|
|
7385
|
+
return basePath;
|
|
7386
|
+
}
|
|
7387
|
+
if (this.partitionStrategy === "flat") {
|
|
7388
|
+
return path.join(basePath, "partitions");
|
|
7389
|
+
}
|
|
7390
|
+
if (this.partitionStrategy === "temporal" && this._isTemporalPartition(partition, partitionValues)) {
|
|
7391
|
+
return this._getTemporalDirectory(basePath, partition, partitionValues);
|
|
7392
|
+
}
|
|
7393
|
+
const pathParts = [basePath, `partition=${partition}`];
|
|
7394
|
+
const sortedFields = Object.entries(partitionValues).sort(([a], [b]) => a.localeCompare(b));
|
|
7395
|
+
for (const [field, value] of sortedFields) {
|
|
7396
|
+
if (value !== null && value !== void 0) {
|
|
7397
|
+
pathParts.push(`${field}=${this._sanitizePathValue(value)}`);
|
|
7398
|
+
}
|
|
7399
|
+
}
|
|
7400
|
+
return path.join(...pathParts);
|
|
7401
|
+
}
|
|
7402
|
+
/**
|
|
7403
|
+
* Enhanced set method with partition awareness
|
|
7404
|
+
*/
|
|
7405
|
+
async _set(key, data, options = {}) {
|
|
7406
|
+
const { resource, action, partition, partitionValues, params } = options;
|
|
7407
|
+
if (resource && partition) {
|
|
7408
|
+
const partitionKey = this._getPartitionCacheKey(resource, action, partition, partitionValues, params);
|
|
7409
|
+
const partitionDir = this._getPartitionDirectory(resource, partition, partitionValues);
|
|
7410
|
+
await this._ensureDirectory(partitionDir);
|
|
7411
|
+
const filePath = path.join(partitionDir, this._sanitizeFileName(partitionKey));
|
|
7412
|
+
if (this.trackUsage) {
|
|
7413
|
+
await this._trackPartitionUsage(resource, partition, partitionValues);
|
|
7414
|
+
}
|
|
7415
|
+
const partitionData = {
|
|
7416
|
+
data,
|
|
7417
|
+
metadata: {
|
|
7418
|
+
resource,
|
|
7419
|
+
partition,
|
|
7420
|
+
partitionValues,
|
|
7421
|
+
timestamp: Date.now(),
|
|
7422
|
+
ttl: this.ttl
|
|
7423
|
+
}
|
|
7424
|
+
};
|
|
7425
|
+
return this._writeFileWithMetadata(filePath, partitionData);
|
|
7426
|
+
}
|
|
7427
|
+
return super._set(key, data);
|
|
7428
|
+
}
|
|
7429
|
+
/**
|
|
7430
|
+
* Enhanced get method with partition awareness
|
|
7431
|
+
*/
|
|
7432
|
+
async _get(key, options = {}) {
|
|
7433
|
+
const { resource, action, partition, partitionValues, params } = options;
|
|
7434
|
+
if (resource && partition) {
|
|
7435
|
+
const partitionKey = this._getPartitionCacheKey(resource, action, partition, partitionValues, params);
|
|
7436
|
+
const partitionDir = this._getPartitionDirectory(resource, partition, partitionValues);
|
|
7437
|
+
const filePath = path.join(partitionDir, this._sanitizeFileName(partitionKey));
|
|
7438
|
+
if (!await this._fileExists(filePath)) {
|
|
7439
|
+
if (this.preloadRelated) {
|
|
7440
|
+
await this._preloadRelatedPartitions(resource, partition, partitionValues);
|
|
7441
|
+
}
|
|
7442
|
+
return null;
|
|
7443
|
+
}
|
|
7444
|
+
const result = await this._readFileWithMetadata(filePath);
|
|
7445
|
+
if (result && this.trackUsage) {
|
|
7446
|
+
await this._trackPartitionUsage(resource, partition, partitionValues);
|
|
7447
|
+
}
|
|
7448
|
+
return result?.data || null;
|
|
7449
|
+
}
|
|
7450
|
+
return super._get(key);
|
|
7451
|
+
}
|
|
7452
|
+
/**
|
|
7453
|
+
* Clear cache for specific partition
|
|
7454
|
+
*/
|
|
7455
|
+
async clearPartition(resource, partition, partitionValues = {}) {
|
|
7456
|
+
const partitionDir = this._getPartitionDirectory(resource, partition, partitionValues);
|
|
7457
|
+
const [ok, err] = await try_fn_default(async () => {
|
|
7458
|
+
if (await this._fileExists(partitionDir)) {
|
|
7459
|
+
await rmdir(partitionDir, { recursive: true });
|
|
7460
|
+
}
|
|
7461
|
+
});
|
|
7462
|
+
if (!ok) {
|
|
7463
|
+
console.warn(`Failed to clear partition cache: ${err.message}`);
|
|
7464
|
+
}
|
|
7465
|
+
const usageKey = this._getUsageKey(resource, partition, partitionValues);
|
|
7466
|
+
this.partitionUsage.delete(usageKey);
|
|
7467
|
+
await this._saveUsageStats();
|
|
7468
|
+
return ok;
|
|
7469
|
+
}
|
|
7470
|
+
/**
|
|
7471
|
+
* Clear all partitions for a resource
|
|
7472
|
+
*/
|
|
7473
|
+
async clearResourcePartitions(resource) {
|
|
7474
|
+
const resourceDir = path.join(this.directory, `resource=${resource}`);
|
|
7475
|
+
const [ok, err] = await try_fn_default(async () => {
|
|
7476
|
+
if (await this._fileExists(resourceDir)) {
|
|
7477
|
+
await rmdir(resourceDir, { recursive: true });
|
|
7478
|
+
}
|
|
7479
|
+
});
|
|
7480
|
+
for (const [key] of this.partitionUsage.entries()) {
|
|
7481
|
+
if (key.startsWith(`${resource}/`)) {
|
|
7482
|
+
this.partitionUsage.delete(key);
|
|
7483
|
+
}
|
|
7484
|
+
}
|
|
7485
|
+
await this._saveUsageStats();
|
|
7486
|
+
return ok;
|
|
7487
|
+
}
|
|
7488
|
+
/**
|
|
7489
|
+
* Get partition cache statistics
|
|
7490
|
+
*/
|
|
7491
|
+
async getPartitionStats(resource, partition = null) {
|
|
7492
|
+
const stats = {
|
|
7493
|
+
totalFiles: 0,
|
|
7494
|
+
totalSize: 0,
|
|
7495
|
+
partitions: {},
|
|
7496
|
+
usage: {}
|
|
7497
|
+
};
|
|
7498
|
+
const resourceDir = path.join(this.directory, `resource=${resource}`);
|
|
7499
|
+
if (!await this._fileExists(resourceDir)) {
|
|
7500
|
+
return stats;
|
|
7501
|
+
}
|
|
7502
|
+
await this._calculateDirectoryStats(resourceDir, stats);
|
|
7503
|
+
for (const [key, usage] of this.partitionUsage.entries()) {
|
|
7504
|
+
if (key.startsWith(`${resource}/`)) {
|
|
7505
|
+
const partitionName = key.split("/")[1];
|
|
7506
|
+
if (!partition || partitionName === partition) {
|
|
7507
|
+
stats.usage[partitionName] = usage;
|
|
7508
|
+
}
|
|
7509
|
+
}
|
|
7510
|
+
}
|
|
7511
|
+
return stats;
|
|
7512
|
+
}
|
|
7513
|
+
/**
|
|
7514
|
+
* Get cache recommendations based on usage patterns
|
|
7515
|
+
*/
|
|
7516
|
+
async getCacheRecommendations(resource) {
|
|
7517
|
+
const recommendations = [];
|
|
7518
|
+
const now = Date.now();
|
|
7519
|
+
const dayMs = 24 * 60 * 60 * 1e3;
|
|
7520
|
+
for (const [key, usage] of this.partitionUsage.entries()) {
|
|
7521
|
+
if (key.startsWith(`${resource}/`)) {
|
|
7522
|
+
const [, partition] = key.split("/");
|
|
7523
|
+
const daysSinceLastAccess = (now - usage.lastAccess) / dayMs;
|
|
7524
|
+
const accessesPerDay = usage.count / Math.max(1, daysSinceLastAccess);
|
|
7525
|
+
let recommendation = "keep";
|
|
7526
|
+
let priority = usage.count;
|
|
7527
|
+
if (daysSinceLastAccess > 30) {
|
|
7528
|
+
recommendation = "archive";
|
|
7529
|
+
priority = 0;
|
|
7530
|
+
} else if (accessesPerDay < 0.1) {
|
|
7531
|
+
recommendation = "reduce_ttl";
|
|
7532
|
+
priority = 1;
|
|
7533
|
+
} else if (accessesPerDay > 10) {
|
|
7534
|
+
recommendation = "preload";
|
|
7535
|
+
priority = 100;
|
|
7536
|
+
}
|
|
7537
|
+
recommendations.push({
|
|
7538
|
+
partition,
|
|
7539
|
+
recommendation,
|
|
7540
|
+
priority,
|
|
7541
|
+
usage: accessesPerDay,
|
|
7542
|
+
lastAccess: new Date(usage.lastAccess).toISOString()
|
|
7543
|
+
});
|
|
7544
|
+
}
|
|
7545
|
+
}
|
|
7546
|
+
return recommendations.sort((a, b) => b.priority - a.priority);
|
|
7547
|
+
}
|
|
7548
|
+
/**
|
|
7549
|
+
* Preload frequently accessed partitions
|
|
7550
|
+
*/
|
|
7551
|
+
async warmPartitionCache(resource, options = {}) {
|
|
7552
|
+
const { partitions = [], maxFiles = 1e3 } = options;
|
|
7553
|
+
let warmedCount = 0;
|
|
7554
|
+
for (const partition of partitions) {
|
|
7555
|
+
const usageKey = `${resource}/${partition}`;
|
|
7556
|
+
const usage = this.partitionUsage.get(usageKey);
|
|
7557
|
+
if (usage && usage.count >= this.preloadThreshold) {
|
|
7558
|
+
console.log(`\u{1F525} Warming cache for ${resource}/${partition} (${usage.count} accesses)`);
|
|
7559
|
+
warmedCount++;
|
|
7560
|
+
}
|
|
7561
|
+
if (warmedCount >= maxFiles) break;
|
|
7562
|
+
}
|
|
7563
|
+
return warmedCount;
|
|
7564
|
+
}
|
|
7565
|
+
// Private helper methods
|
|
7566
|
+
async _trackPartitionUsage(resource, partition, partitionValues) {
|
|
7567
|
+
const usageKey = this._getUsageKey(resource, partition, partitionValues);
|
|
7568
|
+
const current = this.partitionUsage.get(usageKey) || {
|
|
7569
|
+
count: 0,
|
|
7570
|
+
firstAccess: Date.now(),
|
|
7571
|
+
lastAccess: Date.now()
|
|
7572
|
+
};
|
|
7573
|
+
current.count++;
|
|
7574
|
+
current.lastAccess = Date.now();
|
|
7575
|
+
this.partitionUsage.set(usageKey, current);
|
|
7576
|
+
if (current.count % 10 === 0) {
|
|
7577
|
+
await this._saveUsageStats();
|
|
7578
|
+
}
|
|
7579
|
+
}
|
|
7580
|
+
_getUsageKey(resource, partition, partitionValues) {
|
|
7581
|
+
const valuePart = Object.entries(partitionValues).sort(([a], [b]) => a.localeCompare(b)).map(([k, v]) => `${k}=${v}`).join("|");
|
|
7582
|
+
return `${resource}/${partition}/${valuePart}`;
|
|
7583
|
+
}
|
|
7584
|
+
async _preloadRelatedPartitions(resource, partition, partitionValues) {
|
|
7585
|
+
console.log(`\u{1F3AF} Preloading related partitions for ${resource}/${partition}`);
|
|
7586
|
+
if (partitionValues.timestamp || partitionValues.date) ;
|
|
7587
|
+
}
|
|
7588
|
+
_isTemporalPartition(partition, partitionValues) {
|
|
7589
|
+
const temporalFields = ["date", "timestamp", "createdAt", "updatedAt"];
|
|
7590
|
+
return Object.keys(partitionValues).some(
|
|
7591
|
+
(field) => temporalFields.some((tf) => field.toLowerCase().includes(tf))
|
|
7592
|
+
);
|
|
7593
|
+
}
|
|
7594
|
+
_getTemporalDirectory(basePath, partition, partitionValues) {
|
|
7595
|
+
const dateValue = Object.values(partitionValues)[0];
|
|
7596
|
+
if (typeof dateValue === "string" && dateValue.match(/^\d{4}-\d{2}-\d{2}/)) {
|
|
7597
|
+
const [year, month, day] = dateValue.split("-");
|
|
7598
|
+
return path.join(basePath, "temporal", year, month, day);
|
|
7599
|
+
}
|
|
7600
|
+
return path.join(basePath, `partition=${partition}`);
|
|
7601
|
+
}
|
|
7602
|
+
_sanitizePathValue(value) {
|
|
7603
|
+
return String(value).replace(/[<>:"/\\|?*]/g, "_");
|
|
7604
|
+
}
|
|
7605
|
+
_sanitizeFileName(filename) {
|
|
7606
|
+
return filename.replace(/[<>:"/\\|?*]/g, "_");
|
|
7607
|
+
}
|
|
7608
|
+
async _calculateDirectoryStats(dir, stats) {
|
|
7609
|
+
const [ok, err, files] = await try_fn_default(() => readdir(dir));
|
|
7610
|
+
if (!ok) return;
|
|
7611
|
+
for (const file of files) {
|
|
7612
|
+
const filePath = path.join(dir, file);
|
|
7613
|
+
const [statOk, statErr, fileStat] = await try_fn_default(() => stat(filePath));
|
|
7614
|
+
if (statOk) {
|
|
7615
|
+
if (fileStat.isDirectory()) {
|
|
7616
|
+
await this._calculateDirectoryStats(filePath, stats);
|
|
7617
|
+
} else {
|
|
7618
|
+
stats.totalFiles++;
|
|
7619
|
+
stats.totalSize += fileStat.size;
|
|
7620
|
+
}
|
|
7621
|
+
}
|
|
7622
|
+
}
|
|
7623
|
+
}
|
|
7624
|
+
async loadUsageStats() {
|
|
7625
|
+
const [ok, err, content] = await try_fn_default(async () => {
|
|
7626
|
+
const data = await readFile(this.usageStatsFile, "utf8");
|
|
7627
|
+
return JSON.parse(data);
|
|
7628
|
+
});
|
|
7629
|
+
if (ok && content) {
|
|
7630
|
+
this.partitionUsage = new Map(Object.entries(content));
|
|
7631
|
+
}
|
|
7632
|
+
}
|
|
7633
|
+
async _saveUsageStats() {
|
|
7634
|
+
const statsObject = Object.fromEntries(this.partitionUsage);
|
|
7635
|
+
await try_fn_default(async () => {
|
|
7636
|
+
await writeFile(
|
|
7637
|
+
this.usageStatsFile,
|
|
7638
|
+
JSON.stringify(statsObject, null, 2),
|
|
7639
|
+
"utf8"
|
|
7640
|
+
);
|
|
7641
|
+
});
|
|
7642
|
+
}
|
|
7643
|
+
async _writeFileWithMetadata(filePath, data) {
|
|
7644
|
+
const content = JSON.stringify(data);
|
|
7645
|
+
const [ok, err] = await try_fn_default(async () => {
|
|
7646
|
+
await writeFile(filePath, content, {
|
|
7647
|
+
encoding: this.encoding,
|
|
7648
|
+
mode: this.fileMode
|
|
7649
|
+
});
|
|
7650
|
+
});
|
|
7651
|
+
if (!ok) {
|
|
7652
|
+
throw new Error(`Failed to write cache file: ${err.message}`);
|
|
7653
|
+
}
|
|
7654
|
+
return true;
|
|
7655
|
+
}
|
|
7656
|
+
async _readFileWithMetadata(filePath) {
|
|
7657
|
+
const [ok, err, content] = await try_fn_default(async () => {
|
|
7658
|
+
return await readFile(filePath, this.encoding);
|
|
7659
|
+
});
|
|
7660
|
+
if (!ok || !content) return null;
|
|
7661
|
+
try {
|
|
7662
|
+
return JSON.parse(content);
|
|
7663
|
+
} catch (error) {
|
|
7664
|
+
return { data: content };
|
|
7665
|
+
}
|
|
7666
|
+
}
|
|
7667
|
+
}
|
|
7668
|
+
|
|
6815
7669
|
class CachePlugin extends plugin_class_default {
|
|
6816
7670
|
constructor(options = {}) {
|
|
6817
7671
|
super(options);
|
|
6818
7672
|
this.driver = options.driver;
|
|
6819
7673
|
this.config = {
|
|
6820
7674
|
includePartitions: options.includePartitions !== false,
|
|
7675
|
+
partitionStrategy: options.partitionStrategy || "hierarchical",
|
|
7676
|
+
partitionAware: options.partitionAware !== false,
|
|
7677
|
+
trackUsage: options.trackUsage !== false,
|
|
7678
|
+
preloadRelated: options.preloadRelated !== false,
|
|
6821
7679
|
...options
|
|
6822
7680
|
};
|
|
6823
7681
|
}
|
|
@@ -6829,6 +7687,17 @@ class CachePlugin extends plugin_class_default {
|
|
|
6829
7687
|
this.driver = this.config.driver;
|
|
6830
7688
|
} else if (this.config.driverType === "memory") {
|
|
6831
7689
|
this.driver = new memory_cache_class_default(this.config.memoryOptions || {});
|
|
7690
|
+
} else if (this.config.driverType === "filesystem") {
|
|
7691
|
+
if (this.config.partitionAware) {
|
|
7692
|
+
this.driver = new PartitionAwareFilesystemCache({
|
|
7693
|
+
partitionStrategy: this.config.partitionStrategy,
|
|
7694
|
+
trackUsage: this.config.trackUsage,
|
|
7695
|
+
preloadRelated: this.config.preloadRelated,
|
|
7696
|
+
...this.config.filesystemOptions
|
|
7697
|
+
});
|
|
7698
|
+
} else {
|
|
7699
|
+
this.driver = new FilesystemCache(this.config.filesystemOptions || {});
|
|
7700
|
+
}
|
|
6832
7701
|
} else {
|
|
6833
7702
|
this.driver = new s3_cache_class_default({ client: this.database.client, ...this.config.s3Options || {} });
|
|
6834
7703
|
}
|
|
@@ -6869,6 +7738,20 @@ class CachePlugin extends plugin_class_default {
|
|
|
6869
7738
|
const { action, params = {}, partition, partitionValues } = options;
|
|
6870
7739
|
return this.generateCacheKey(resource, action, params, partition, partitionValues);
|
|
6871
7740
|
};
|
|
7741
|
+
if (this.driver instanceof PartitionAwareFilesystemCache) {
|
|
7742
|
+
resource.clearPartitionCache = async (partition, partitionValues = {}) => {
|
|
7743
|
+
return await this.driver.clearPartition(resource.name, partition, partitionValues);
|
|
7744
|
+
};
|
|
7745
|
+
resource.getPartitionCacheStats = async (partition = null) => {
|
|
7746
|
+
return await this.driver.getPartitionStats(resource.name, partition);
|
|
7747
|
+
};
|
|
7748
|
+
resource.getCacheRecommendations = async () => {
|
|
7749
|
+
return await this.driver.getCacheRecommendations(resource.name);
|
|
7750
|
+
};
|
|
7751
|
+
resource.warmPartitionCache = async (partitions = [], options = {}) => {
|
|
7752
|
+
return await this.driver.warmPartitionCache(resource.name, { partitions, ...options });
|
|
7753
|
+
};
|
|
7754
|
+
}
|
|
6872
7755
|
const cacheMethods = [
|
|
6873
7756
|
"count",
|
|
6874
7757
|
"listIds",
|
|
@@ -6894,12 +7777,37 @@ class CachePlugin extends plugin_class_default {
|
|
|
6894
7777
|
} else if (method === "get") {
|
|
6895
7778
|
key = await resource.cacheKeyFor({ action: method, params: { id: ctx.args[0] } });
|
|
6896
7779
|
}
|
|
6897
|
-
|
|
6898
|
-
|
|
6899
|
-
|
|
6900
|
-
|
|
6901
|
-
|
|
6902
|
-
|
|
7780
|
+
if (this.driver instanceof PartitionAwareFilesystemCache) {
|
|
7781
|
+
let partition, partitionValues;
|
|
7782
|
+
if (method === "list" || method === "listIds" || method === "count" || method === "page") {
|
|
7783
|
+
const args = ctx.args[0] || {};
|
|
7784
|
+
partition = args.partition;
|
|
7785
|
+
partitionValues = args.partitionValues;
|
|
7786
|
+
}
|
|
7787
|
+
const [ok, err, result] = await try_fn_default(() => resource.cache._get(key, {
|
|
7788
|
+
resource: resource.name,
|
|
7789
|
+
action: method,
|
|
7790
|
+
partition,
|
|
7791
|
+
partitionValues
|
|
7792
|
+
}));
|
|
7793
|
+
if (ok && result !== null && result !== void 0) return result;
|
|
7794
|
+
if (!ok && err.name !== "NoSuchKey") throw err;
|
|
7795
|
+
const freshResult = await next();
|
|
7796
|
+
await resource.cache._set(key, freshResult, {
|
|
7797
|
+
resource: resource.name,
|
|
7798
|
+
action: method,
|
|
7799
|
+
partition,
|
|
7800
|
+
partitionValues
|
|
7801
|
+
});
|
|
7802
|
+
return freshResult;
|
|
7803
|
+
} else {
|
|
7804
|
+
const [ok, err, result] = await try_fn_default(() => resource.cache.get(key));
|
|
7805
|
+
if (ok && result !== null && result !== void 0) return result;
|
|
7806
|
+
if (!ok && err.name !== "NoSuchKey") throw err;
|
|
7807
|
+
const freshResult = await next();
|
|
7808
|
+
await resource.cache.set(key, freshResult);
|
|
7809
|
+
return freshResult;
|
|
7810
|
+
}
|
|
6903
7811
|
});
|
|
6904
7812
|
}
|
|
6905
7813
|
const writeMethods = ["insert", "update", "delete", "deleteMany"];
|
|
@@ -6992,6 +7900,10 @@ class CachePlugin extends plugin_class_default {
|
|
|
6992
7900
|
throw new Error(`Resource '${resourceName}' not found`);
|
|
6993
7901
|
}
|
|
6994
7902
|
const { includePartitions = true } = options;
|
|
7903
|
+
if (this.driver instanceof PartitionAwareFilesystemCache && resource.warmPartitionCache) {
|
|
7904
|
+
const partitionNames = resource.config.partitions ? Object.keys(resource.config.partitions) : [];
|
|
7905
|
+
return await resource.warmPartitionCache(partitionNames, options);
|
|
7906
|
+
}
|
|
6995
7907
|
await resource.getAll();
|
|
6996
7908
|
if (includePartitions && resource.config.partitions) {
|
|
6997
7909
|
for (const [partitionName, partitionDef] of Object.entries(resource.config.partitions)) {
|
|
@@ -7013,6 +7925,57 @@ class CachePlugin extends plugin_class_default {
|
|
|
7013
7925
|
}
|
|
7014
7926
|
}
|
|
7015
7927
|
}
|
|
7928
|
+
// Partition-specific methods
|
|
7929
|
+
async getPartitionCacheStats(resourceName, partition = null) {
|
|
7930
|
+
if (!(this.driver instanceof PartitionAwareFilesystemCache)) {
|
|
7931
|
+
throw new Error("Partition cache statistics are only available with PartitionAwareFilesystemCache");
|
|
7932
|
+
}
|
|
7933
|
+
return await this.driver.getPartitionStats(resourceName, partition);
|
|
7934
|
+
}
|
|
7935
|
+
async getCacheRecommendations(resourceName) {
|
|
7936
|
+
if (!(this.driver instanceof PartitionAwareFilesystemCache)) {
|
|
7937
|
+
throw new Error("Cache recommendations are only available with PartitionAwareFilesystemCache");
|
|
7938
|
+
}
|
|
7939
|
+
return await this.driver.getCacheRecommendations(resourceName);
|
|
7940
|
+
}
|
|
7941
|
+
async clearPartitionCache(resourceName, partition, partitionValues = {}) {
|
|
7942
|
+
if (!(this.driver instanceof PartitionAwareFilesystemCache)) {
|
|
7943
|
+
throw new Error("Partition cache clearing is only available with PartitionAwareFilesystemCache");
|
|
7944
|
+
}
|
|
7945
|
+
return await this.driver.clearPartition(resourceName, partition, partitionValues);
|
|
7946
|
+
}
|
|
7947
|
+
async analyzeCacheUsage() {
|
|
7948
|
+
if (!(this.driver instanceof PartitionAwareFilesystemCache)) {
|
|
7949
|
+
return { message: "Cache usage analysis is only available with PartitionAwareFilesystemCache" };
|
|
7950
|
+
}
|
|
7951
|
+
const analysis = {
|
|
7952
|
+
totalResources: Object.keys(this.database.resources).length,
|
|
7953
|
+
resourceStats: {},
|
|
7954
|
+
recommendations: {},
|
|
7955
|
+
summary: {
|
|
7956
|
+
mostUsedPartitions: [],
|
|
7957
|
+
leastUsedPartitions: [],
|
|
7958
|
+
suggestedOptimizations: []
|
|
7959
|
+
}
|
|
7960
|
+
};
|
|
7961
|
+
for (const [resourceName, resource] of Object.entries(this.database.resources)) {
|
|
7962
|
+
try {
|
|
7963
|
+
analysis.resourceStats[resourceName] = await this.driver.getPartitionStats(resourceName);
|
|
7964
|
+
analysis.recommendations[resourceName] = await this.driver.getCacheRecommendations(resourceName);
|
|
7965
|
+
} catch (error) {
|
|
7966
|
+
analysis.resourceStats[resourceName] = { error: error.message };
|
|
7967
|
+
}
|
|
7968
|
+
}
|
|
7969
|
+
const allRecommendations = Object.values(analysis.recommendations).flat();
|
|
7970
|
+
analysis.summary.mostUsedPartitions = allRecommendations.filter((r) => r.recommendation === "preload").sort((a, b) => b.priority - a.priority).slice(0, 5);
|
|
7971
|
+
analysis.summary.leastUsedPartitions = allRecommendations.filter((r) => r.recommendation === "archive").slice(0, 5);
|
|
7972
|
+
analysis.summary.suggestedOptimizations = [
|
|
7973
|
+
`Consider preloading ${analysis.summary.mostUsedPartitions.length} high-usage partitions`,
|
|
7974
|
+
`Archive ${analysis.summary.leastUsedPartitions.length} unused partitions`,
|
|
7975
|
+
`Monitor cache hit rates for partition efficiency`
|
|
7976
|
+
];
|
|
7977
|
+
return analysis;
|
|
7978
|
+
}
|
|
7016
7979
|
}
|
|
7017
7980
|
|
|
7018
7981
|
const CostsPlugin = {
|
|
@@ -7189,24 +8152,29 @@ class FullTextPlugin extends plugin_class_default {
|
|
|
7189
8152
|
resource._deleteMany = resource.deleteMany;
|
|
7190
8153
|
this.wrapResourceMethod(resource, "insert", async (result, args, methodName) => {
|
|
7191
8154
|
const [data] = args;
|
|
7192
|
-
this.indexRecord(resource.name, result.id, data).catch(
|
|
8155
|
+
this.indexRecord(resource.name, result.id, data).catch(() => {
|
|
8156
|
+
});
|
|
7193
8157
|
return result;
|
|
7194
8158
|
});
|
|
7195
8159
|
this.wrapResourceMethod(resource, "update", async (result, args, methodName) => {
|
|
7196
8160
|
const [id, data] = args;
|
|
7197
|
-
this.removeRecordFromIndex(resource.name, id).catch(
|
|
7198
|
-
|
|
8161
|
+
this.removeRecordFromIndex(resource.name, id).catch(() => {
|
|
8162
|
+
});
|
|
8163
|
+
this.indexRecord(resource.name, id, result).catch(() => {
|
|
8164
|
+
});
|
|
7199
8165
|
return result;
|
|
7200
8166
|
});
|
|
7201
8167
|
this.wrapResourceMethod(resource, "delete", async (result, args, methodName) => {
|
|
7202
8168
|
const [id] = args;
|
|
7203
|
-
this.removeRecordFromIndex(resource.name, id).catch(
|
|
8169
|
+
this.removeRecordFromIndex(resource.name, id).catch(() => {
|
|
8170
|
+
});
|
|
7204
8171
|
return result;
|
|
7205
8172
|
});
|
|
7206
8173
|
this.wrapResourceMethod(resource, "deleteMany", async (result, args, methodName) => {
|
|
7207
8174
|
const [ids] = args;
|
|
7208
8175
|
for (const id of ids) {
|
|
7209
|
-
this.removeRecordFromIndex(resource.name, id).catch(
|
|
8176
|
+
this.removeRecordFromIndex(resource.name, id).catch(() => {
|
|
8177
|
+
});
|
|
7210
8178
|
}
|
|
7211
8179
|
return result;
|
|
7212
8180
|
});
|
|
@@ -7696,7 +8664,8 @@ class MetricsPlugin extends plugin_class_default {
|
|
|
7696
8664
|
}
|
|
7697
8665
|
if (this.config.flushInterval > 0) {
|
|
7698
8666
|
this.flushTimer = setInterval(() => {
|
|
7699
|
-
this.flushMetrics().catch(
|
|
8667
|
+
this.flushMetrics().catch(() => {
|
|
8668
|
+
});
|
|
7700
8669
|
}, this.config.flushInterval);
|
|
7701
8670
|
}
|
|
7702
8671
|
}
|
|
@@ -7768,9 +8737,6 @@ class MetricsPlugin extends plugin_class_default {
|
|
|
7768
8737
|
}
|
|
7769
8738
|
this.resetMetrics();
|
|
7770
8739
|
});
|
|
7771
|
-
if (!ok) {
|
|
7772
|
-
console.error("Failed to flush metrics:", err);
|
|
7773
|
-
}
|
|
7774
8740
|
}
|
|
7775
8741
|
resetMetrics() {
|
|
7776
8742
|
for (const operation of Object.keys(this.metrics.operations)) {
|
|
@@ -10607,7 +11573,7 @@ class Resource extends EventEmitter {
|
|
|
10607
11573
|
if (okParse) contentType = "application/json";
|
|
10608
11574
|
}
|
|
10609
11575
|
if (this.behavior === "body-only" && (!body || body === "")) {
|
|
10610
|
-
throw new Error(`[Resource.insert]
|
|
11576
|
+
throw new Error(`[Resource.insert] Attempt to save object without body! Data: id=${finalId}, resource=${this.name}`);
|
|
10611
11577
|
}
|
|
10612
11578
|
const [okPut, errPut, putResult] = await try_fn_default(() => this.client.putObject({
|
|
10613
11579
|
key,
|
|
@@ -11954,8 +12920,8 @@ class Resource extends EventEmitter {
|
|
|
11954
12920
|
return mappedData;
|
|
11955
12921
|
}
|
|
11956
12922
|
/**
|
|
11957
|
-
* Compose the full object (metadata + body) as
|
|
11958
|
-
*
|
|
12923
|
+
* Compose the full object (metadata + body) as returned by .get(),
|
|
12924
|
+
* using in-memory data after insert/update, according to behavior
|
|
11959
12925
|
*/
|
|
11960
12926
|
async composeFullObjectFromWrite({ id, metadata, body, behavior }) {
|
|
11961
12927
|
const behaviorFlags = {};
|
|
@@ -12110,7 +13076,7 @@ class Resource extends EventEmitter {
|
|
|
12110
13076
|
if (!this._middlewares.has(method)) throw new ResourceError(`No such method for middleware: ${method}`, { operation: "useMiddleware", method });
|
|
12111
13077
|
this._middlewares.get(method).push(fn);
|
|
12112
13078
|
}
|
|
12113
|
-
//
|
|
13079
|
+
// Utility to apply schema default values
|
|
12114
13080
|
applyDefaults(data) {
|
|
12115
13081
|
const out = { ...data };
|
|
12116
13082
|
for (const [key, def] of Object.entries(this.attributes)) {
|
|
@@ -12242,7 +13208,7 @@ class Database extends EventEmitter {
|
|
|
12242
13208
|
super();
|
|
12243
13209
|
this.version = "1";
|
|
12244
13210
|
this.s3dbVersion = (() => {
|
|
12245
|
-
const [ok, err, version] = try_fn_default(() => true ? "7.2
|
|
13211
|
+
const [ok, err, version] = try_fn_default(() => true ? "7.3.2" : "latest");
|
|
12246
13212
|
return ok ? version : "latest";
|
|
12247
13213
|
})();
|
|
12248
13214
|
this.resources = {};
|
|
@@ -12315,7 +13281,7 @@ class Database extends EventEmitter {
|
|
|
12315
13281
|
name,
|
|
12316
13282
|
client: this.client,
|
|
12317
13283
|
database: this,
|
|
12318
|
-
//
|
|
13284
|
+
// ensure reference
|
|
12319
13285
|
version: currentVersion,
|
|
12320
13286
|
attributes: versionData.attributes,
|
|
12321
13287
|
behavior: versionData.behavior || "user-managed",
|
|
@@ -12816,21 +13782,33 @@ class S3dbReplicator extends base_replicator_class_default {
|
|
|
12816
13782
|
throw err;
|
|
12817
13783
|
}
|
|
12818
13784
|
}
|
|
12819
|
-
//
|
|
12820
|
-
async replicate(
|
|
13785
|
+
// Support both object and parameter signatures for flexibility
|
|
13786
|
+
async replicate(resourceOrObj, operation, data, recordId, beforeData) {
|
|
13787
|
+
let resource, op, payload, id;
|
|
13788
|
+
if (typeof resourceOrObj === "object" && resourceOrObj.resource) {
|
|
13789
|
+
resource = resourceOrObj.resource;
|
|
13790
|
+
op = resourceOrObj.operation;
|
|
13791
|
+
payload = resourceOrObj.data;
|
|
13792
|
+
id = resourceOrObj.id;
|
|
13793
|
+
} else {
|
|
13794
|
+
resource = resourceOrObj;
|
|
13795
|
+
op = operation;
|
|
13796
|
+
payload = data;
|
|
13797
|
+
id = recordId;
|
|
13798
|
+
}
|
|
12821
13799
|
const normResource = normalizeResourceName$1(resource);
|
|
12822
|
-
const destResource = this._resolveDestResource(normResource,
|
|
13800
|
+
const destResource = this._resolveDestResource(normResource, payload);
|
|
12823
13801
|
const destResourceObj = this._getDestResourceObj(destResource);
|
|
12824
|
-
const transformedData = this._applyTransformer(normResource,
|
|
13802
|
+
const transformedData = this._applyTransformer(normResource, payload);
|
|
12825
13803
|
let result;
|
|
12826
|
-
if (
|
|
13804
|
+
if (op === "insert") {
|
|
12827
13805
|
result = await destResourceObj.insert(transformedData);
|
|
12828
|
-
} else if (
|
|
12829
|
-
result = await destResourceObj.update(
|
|
12830
|
-
} else if (
|
|
12831
|
-
result = await destResourceObj.delete(
|
|
13806
|
+
} else if (op === "update") {
|
|
13807
|
+
result = await destResourceObj.update(id, transformedData);
|
|
13808
|
+
} else if (op === "delete") {
|
|
13809
|
+
result = await destResourceObj.delete(id);
|
|
12832
13810
|
} else {
|
|
12833
|
-
throw new Error(`Invalid operation: ${
|
|
13811
|
+
throw new Error(`Invalid operation: ${op}. Supported operations are: insert, update, delete`);
|
|
12834
13812
|
}
|
|
12835
13813
|
return result;
|
|
12836
13814
|
}
|
|
@@ -12863,7 +13841,7 @@ class S3dbReplicator extends base_replicator_class_default {
|
|
|
12863
13841
|
if (typeof entry[0] === "function") return resource;
|
|
12864
13842
|
}
|
|
12865
13843
|
if (typeof entry === "string") return entry;
|
|
12866
|
-
if (
|
|
13844
|
+
if (resource && !targetResourceName) targetResourceName = resource;
|
|
12867
13845
|
if (typeof entry === "object" && entry.resource) return entry.resource;
|
|
12868
13846
|
return resource;
|
|
12869
13847
|
}
|
|
@@ -12976,7 +13954,6 @@ var s3db_replicator_class_default = S3dbReplicator;
|
|
|
12976
13954
|
class SqsReplicator extends base_replicator_class_default {
|
|
12977
13955
|
constructor(config = {}, resources = [], client = null) {
|
|
12978
13956
|
super(config);
|
|
12979
|
-
this.resources = resources;
|
|
12980
13957
|
this.client = client;
|
|
12981
13958
|
this.queueUrl = config.queueUrl;
|
|
12982
13959
|
this.queues = config.queues || {};
|
|
@@ -12985,12 +13962,24 @@ class SqsReplicator extends base_replicator_class_default {
|
|
|
12985
13962
|
this.sqsClient = client || null;
|
|
12986
13963
|
this.messageGroupId = config.messageGroupId;
|
|
12987
13964
|
this.deduplicationId = config.deduplicationId;
|
|
12988
|
-
if (resources
|
|
13965
|
+
if (Array.isArray(resources)) {
|
|
13966
|
+
this.resources = {};
|
|
13967
|
+
for (const resource of resources) {
|
|
13968
|
+
if (typeof resource === "string") {
|
|
13969
|
+
this.resources[resource] = true;
|
|
13970
|
+
} else if (typeof resource === "object" && resource.name) {
|
|
13971
|
+
this.resources[resource.name] = resource;
|
|
13972
|
+
}
|
|
13973
|
+
}
|
|
13974
|
+
} else if (typeof resources === "object") {
|
|
13975
|
+
this.resources = resources;
|
|
12989
13976
|
for (const [resourceName, resourceConfig] of Object.entries(resources)) {
|
|
12990
|
-
if (resourceConfig.queueUrl) {
|
|
13977
|
+
if (resourceConfig && resourceConfig.queueUrl) {
|
|
12991
13978
|
this.queues[resourceName] = resourceConfig.queueUrl;
|
|
12992
13979
|
}
|
|
12993
13980
|
}
|
|
13981
|
+
} else {
|
|
13982
|
+
this.resources = {};
|
|
12994
13983
|
}
|
|
12995
13984
|
}
|
|
12996
13985
|
validateConfig() {
|
|
@@ -13226,7 +14215,7 @@ class SqsReplicator extends base_replicator_class_default {
|
|
|
13226
14215
|
connected: !!this.sqsClient,
|
|
13227
14216
|
queueUrl: this.queueUrl,
|
|
13228
14217
|
region: this.region,
|
|
13229
|
-
resources: this.resources,
|
|
14218
|
+
resources: Object.keys(this.resources || {}),
|
|
13230
14219
|
totalreplicators: this.listenerCount("replicated"),
|
|
13231
14220
|
totalErrors: this.listenerCount("replicator_error")
|
|
13232
14221
|
};
|
|
@@ -13264,33 +14253,28 @@ function normalizeResourceName(name) {
|
|
|
13264
14253
|
class ReplicatorPlugin extends plugin_class_default {
|
|
13265
14254
|
constructor(options = {}) {
|
|
13266
14255
|
super();
|
|
13267
|
-
if (options.verbose) {
|
|
13268
|
-
console.log("[PLUGIN][CONSTRUCTOR] ReplicatorPlugin constructor called");
|
|
13269
|
-
}
|
|
13270
|
-
if (options.verbose) {
|
|
13271
|
-
console.log("[PLUGIN][constructor] New ReplicatorPlugin instance created with config:", options);
|
|
13272
|
-
}
|
|
13273
14256
|
if (!options.replicators || !Array.isArray(options.replicators)) {
|
|
13274
14257
|
throw new Error("ReplicatorPlugin: replicators array is required");
|
|
13275
14258
|
}
|
|
13276
14259
|
for (const rep of options.replicators) {
|
|
13277
14260
|
if (!rep.driver) throw new Error("ReplicatorPlugin: each replicator must have a driver");
|
|
14261
|
+
if (!rep.resources || typeof rep.resources !== "object") throw new Error("ReplicatorPlugin: each replicator must have resources config");
|
|
14262
|
+
if (Object.keys(rep.resources).length === 0) throw new Error("ReplicatorPlugin: each replicator must have at least one resource configured");
|
|
13278
14263
|
}
|
|
13279
14264
|
this.config = {
|
|
13280
|
-
|
|
13281
|
-
|
|
13282
|
-
replicatorLogResource: options.replicatorLogResource
|
|
13283
|
-
|
|
14265
|
+
replicators: options.replicators || [],
|
|
14266
|
+
logErrors: options.logErrors !== false,
|
|
14267
|
+
replicatorLogResource: options.replicatorLogResource || "replicator_log",
|
|
14268
|
+
enabled: options.enabled !== false,
|
|
14269
|
+
batchSize: options.batchSize || 100,
|
|
14270
|
+
maxRetries: options.maxRetries || 3,
|
|
14271
|
+
timeout: options.timeout || 3e4,
|
|
14272
|
+
verbose: options.verbose || false,
|
|
14273
|
+
...options
|
|
13284
14274
|
};
|
|
13285
14275
|
this.replicators = [];
|
|
13286
|
-
this.
|
|
13287
|
-
this.
|
|
13288
|
-
this.stats = {
|
|
13289
|
-
totalOperations: 0,
|
|
13290
|
-
totalErrors: 0,
|
|
13291
|
-
lastError: null
|
|
13292
|
-
};
|
|
13293
|
-
this._installedListeners = [];
|
|
14276
|
+
this.database = null;
|
|
14277
|
+
this.eventListenersInstalled = /* @__PURE__ */ new Set();
|
|
13294
14278
|
}
|
|
13295
14279
|
/**
|
|
13296
14280
|
* Decompress data if it was compressed
|
|
@@ -13309,79 +14293,34 @@ class ReplicatorPlugin extends plugin_class_default {
|
|
|
13309
14293
|
}
|
|
13310
14294
|
return filtered;
|
|
13311
14295
|
}
|
|
13312
|
-
installEventListeners(resource) {
|
|
13313
|
-
|
|
13314
|
-
|
|
13315
|
-
console.log("[PLUGIN] installEventListeners called for:", resource && resource.name, {
|
|
13316
|
-
hasDatabase: !!resource.database,
|
|
13317
|
-
sameDatabase: resource.database === plugin.database,
|
|
13318
|
-
alreadyInstalled: resource._replicatorListenersInstalled,
|
|
13319
|
-
resourceObj: resource,
|
|
13320
|
-
resourceObjId: resource && resource.id,
|
|
13321
|
-
resourceObjType: typeof resource,
|
|
13322
|
-
resourceObjIs: resource && Object.is(resource, plugin.database.resources && plugin.database.resources[resource.name]),
|
|
13323
|
-
resourceObjEq: resource === (plugin.database.resources && plugin.database.resources[resource.name])
|
|
13324
|
-
});
|
|
13325
|
-
}
|
|
13326
|
-
if (!resource || resource.name === plugin.config.replicatorLogResource || !resource.database || resource.database !== plugin.database) return;
|
|
13327
|
-
if (resource._replicatorListenersInstalled) return;
|
|
13328
|
-
resource._replicatorListenersInstalled = true;
|
|
13329
|
-
this._installedListeners.push(resource);
|
|
13330
|
-
if (plugin.config.verbose) {
|
|
13331
|
-
console.log(`[PLUGIN] installEventListeners INSTALLED for resource: ${resource && resource.name}`);
|
|
14296
|
+
installEventListeners(resource, database, plugin) {
|
|
14297
|
+
if (!resource || this.eventListenersInstalled.has(resource.name) || resource.name === this.config.replicatorLogResource) {
|
|
14298
|
+
return;
|
|
13332
14299
|
}
|
|
13333
14300
|
resource.on("insert", async (data) => {
|
|
13334
|
-
if (plugin.config.verbose) {
|
|
13335
|
-
console.log("[PLUGIN] Listener INSERT on", resource.name, "plugin.replicators.length:", plugin.replicators.length, plugin.replicators.map((r) => ({ id: r.id, driver: r.driver })));
|
|
13336
|
-
}
|
|
13337
14301
|
try {
|
|
13338
|
-
const completeData =
|
|
13339
|
-
|
|
13340
|
-
|
|
13341
|
-
}
|
|
13342
|
-
await plugin.processReplicatorEvent(resource.name, "insert", data.id, completeData, null);
|
|
13343
|
-
} catch (err) {
|
|
13344
|
-
if (plugin.config.verbose) {
|
|
13345
|
-
console.error(`[PLUGIN] Listener INSERT error on ${resource.name} id=${data && data.id}:`, err);
|
|
13346
|
-
}
|
|
14302
|
+
const completeData = { ...data, createdAt: (/* @__PURE__ */ new Date()).toISOString() };
|
|
14303
|
+
await plugin.processReplicatorEvent("insert", resource.name, completeData.id, completeData);
|
|
14304
|
+
} catch (error) {
|
|
14305
|
+
this.emit("error", { operation: "insert", error: error.message, resource: resource.name });
|
|
13347
14306
|
}
|
|
13348
14307
|
});
|
|
13349
|
-
resource.on("update", async (data) => {
|
|
13350
|
-
console.log("[PLUGIN][Listener][UPDATE][START] triggered for resource:", resource.name, "data:", data);
|
|
13351
|
-
const beforeData = data && data.$before;
|
|
13352
|
-
if (plugin.config.verbose) {
|
|
13353
|
-
console.log("[PLUGIN] Listener UPDATE on", resource.name, "plugin.replicators.length:", plugin.replicators.length, plugin.replicators.map((r) => ({ id: r.id, driver: r.driver })), "data:", data, "beforeData:", beforeData);
|
|
13354
|
-
}
|
|
14308
|
+
resource.on("update", async (data, beforeData) => {
|
|
13355
14309
|
try {
|
|
13356
|
-
|
|
13357
|
-
|
|
13358
|
-
|
|
13359
|
-
|
|
13360
|
-
} else {
|
|
13361
|
-
completeData = data;
|
|
13362
|
-
}
|
|
13363
|
-
await plugin.processReplicatorEvent(resource.name, "update", data.id, completeData, beforeData);
|
|
13364
|
-
} catch (err) {
|
|
13365
|
-
if (plugin.config.verbose) {
|
|
13366
|
-
console.error(`[PLUGIN] Listener UPDATE erro em ${resource.name} id=${data && data.id}:`, err);
|
|
13367
|
-
}
|
|
14310
|
+
const completeData = { ...data, updatedAt: (/* @__PURE__ */ new Date()).toISOString() };
|
|
14311
|
+
await plugin.processReplicatorEvent("update", resource.name, completeData.id, completeData, beforeData);
|
|
14312
|
+
} catch (error) {
|
|
14313
|
+
this.emit("error", { operation: "update", error: error.message, resource: resource.name });
|
|
13368
14314
|
}
|
|
13369
14315
|
});
|
|
13370
|
-
resource.on("delete", async (data
|
|
13371
|
-
if (plugin.config.verbose) {
|
|
13372
|
-
console.log("[PLUGIN] Listener DELETE on", resource.name, "plugin.replicators.length:", plugin.replicators.length, plugin.replicators.map((r) => ({ id: r.id, driver: r.driver })));
|
|
13373
|
-
}
|
|
14316
|
+
resource.on("delete", async (data) => {
|
|
13374
14317
|
try {
|
|
13375
|
-
await plugin.processReplicatorEvent(resource.name,
|
|
13376
|
-
} catch (
|
|
13377
|
-
|
|
13378
|
-
console.error(`[PLUGIN] Listener DELETE erro em ${resource.name} id=${data && data.id}:`, err);
|
|
13379
|
-
}
|
|
14318
|
+
await plugin.processReplicatorEvent("delete", resource.name, data.id, data);
|
|
14319
|
+
} catch (error) {
|
|
14320
|
+
this.emit("error", { operation: "delete", error: error.message, resource: resource.name });
|
|
13380
14321
|
}
|
|
13381
14322
|
});
|
|
13382
|
-
|
|
13383
|
-
console.log(`[PLUGIN] Listeners instalados para resource: ${resource && resource.name} (insert: ${resource.listenerCount("insert")}, update: ${resource.listenerCount("update")}, delete: ${resource.listenerCount("delete")})`);
|
|
13384
|
-
}
|
|
14323
|
+
this.eventListenersInstalled.add(resource.name);
|
|
13385
14324
|
}
|
|
13386
14325
|
/**
|
|
13387
14326
|
* Get complete data by always fetching the full record from the resource
|
|
@@ -13392,112 +14331,58 @@ class ReplicatorPlugin extends plugin_class_default {
|
|
|
13392
14331
|
return ok ? completeRecord : data;
|
|
13393
14332
|
}
|
|
13394
14333
|
async setup(database) {
|
|
13395
|
-
console.log("[PLUGIN][SETUP] setup called");
|
|
13396
|
-
if (this.config.verbose) {
|
|
13397
|
-
console.log("[PLUGIN][setup] called with database:", database && database.name);
|
|
13398
|
-
}
|
|
13399
14334
|
this.database = database;
|
|
13400
|
-
|
|
13401
|
-
|
|
13402
|
-
|
|
13403
|
-
|
|
14335
|
+
try {
|
|
14336
|
+
await this.initializeReplicators(database);
|
|
14337
|
+
} catch (error) {
|
|
14338
|
+
this.emit("error", { operation: "setup", error: error.message });
|
|
14339
|
+
throw error;
|
|
14340
|
+
}
|
|
14341
|
+
try {
|
|
14342
|
+
if (this.config.replicatorLogResource) {
|
|
14343
|
+
const logRes = await database.createResource({
|
|
13404
14344
|
name: this.config.replicatorLogResource,
|
|
13405
|
-
behavior: "
|
|
14345
|
+
behavior: "body-overflow",
|
|
13406
14346
|
attributes: {
|
|
13407
|
-
|
|
13408
|
-
|
|
13409
|
-
|
|
13410
|
-
data: "
|
|
13411
|
-
|
|
13412
|
-
|
|
13413
|
-
|
|
13414
|
-
|
|
13415
|
-
byDate: { fields: { "createdAt": "string|maxlength:10" } }
|
|
14347
|
+
operation: "string",
|
|
14348
|
+
resourceName: "string",
|
|
14349
|
+
recordId: "string",
|
|
14350
|
+
data: "string",
|
|
14351
|
+
error: "string|optional",
|
|
14352
|
+
replicator: "string",
|
|
14353
|
+
timestamp: "string",
|
|
14354
|
+
status: "string"
|
|
13416
14355
|
}
|
|
13417
14356
|
});
|
|
13418
|
-
if (this.config.verbose) {
|
|
13419
|
-
console.log("[PLUGIN] Log resource created:", this.config.replicatorLogResource, !!logRes);
|
|
13420
|
-
}
|
|
13421
|
-
}
|
|
13422
|
-
database.resources[normalizeResourceName(this.config.replicatorLogResource)] = logRes;
|
|
13423
|
-
this.replicatorLog = logRes;
|
|
13424
|
-
if (this.config.verbose) {
|
|
13425
|
-
console.log("[PLUGIN] Log resource created and registered:", this.config.replicatorLogResource, !!database.resources[normalizeResourceName(this.config.replicatorLogResource)]);
|
|
13426
|
-
}
|
|
13427
|
-
if (typeof database.uploadMetadataFile === "function") {
|
|
13428
|
-
await database.uploadMetadataFile();
|
|
13429
|
-
if (this.config.verbose) {
|
|
13430
|
-
console.log("[PLUGIN] uploadMetadataFile called. database.resources keys:", Object.keys(database.resources));
|
|
13431
|
-
}
|
|
13432
|
-
}
|
|
13433
|
-
}
|
|
13434
|
-
if (this.config.replicators && this.config.replicators.length > 0 && this.replicators.length === 0) {
|
|
13435
|
-
await this.initializeReplicators();
|
|
13436
|
-
console.log("[PLUGIN][SETUP] after initializeReplicators, replicators.length:", this.replicators.length);
|
|
13437
|
-
if (this.config.verbose) {
|
|
13438
|
-
console.log("[PLUGIN][setup] After initializeReplicators, replicators.length:", this.replicators.length, this.replicators.map((r) => ({ id: r.id, driver: r.driver })));
|
|
13439
|
-
}
|
|
13440
|
-
}
|
|
13441
|
-
for (const resourceName in database.resources) {
|
|
13442
|
-
if (normalizeResourceName(resourceName) !== normalizeResourceName(this.config.replicatorLogResource)) {
|
|
13443
|
-
this.installEventListeners(database.resources[resourceName]);
|
|
13444
14357
|
}
|
|
14358
|
+
} catch (error) {
|
|
13445
14359
|
}
|
|
13446
|
-
|
|
13447
|
-
for (const resourceName in database.resources) {
|
|
13448
|
-
if (normalizeResourceName(resourceName) !== normalizeResourceName(this.config.replicatorLogResource)) {
|
|
13449
|
-
this.installEventListeners(database.resources[resourceName]);
|
|
13450
|
-
}
|
|
13451
|
-
}
|
|
13452
|
-
});
|
|
14360
|
+
await this.uploadMetadataFile(database);
|
|
13453
14361
|
const originalCreateResource = database.createResource.bind(database);
|
|
13454
14362
|
database.createResource = async (config) => {
|
|
13455
|
-
if (this.config.verbose) {
|
|
13456
|
-
console.log("[PLUGIN] createResource proxy called for:", config && config.name);
|
|
13457
|
-
}
|
|
13458
14363
|
const resource = await originalCreateResource(config);
|
|
13459
|
-
if (resource
|
|
13460
|
-
this.installEventListeners(resource);
|
|
14364
|
+
if (resource) {
|
|
14365
|
+
this.installEventListeners(resource, database, this);
|
|
13461
14366
|
}
|
|
13462
14367
|
return resource;
|
|
13463
14368
|
};
|
|
13464
|
-
|
|
13465
|
-
const resource = database.resources[resourceName];
|
|
13466
|
-
if (resource && resource.name !== this.config.replicatorLogResource) {
|
|
13467
|
-
this.installEventListeners(resource);
|
|
13468
|
-
}
|
|
13469
|
-
});
|
|
13470
|
-
database.on("s3db.resourceUpdated", (resourceName) => {
|
|
14369
|
+
for (const resourceName in database.resources) {
|
|
13471
14370
|
const resource = database.resources[resourceName];
|
|
13472
|
-
|
|
13473
|
-
|
|
13474
|
-
|
|
13475
|
-
|
|
14371
|
+
this.installEventListeners(resource, database, this);
|
|
14372
|
+
}
|
|
14373
|
+
}
|
|
14374
|
+
createReplicator(driver, config, resources, client) {
|
|
14375
|
+
return createReplicator(driver, config, resources, client);
|
|
13476
14376
|
}
|
|
13477
|
-
async initializeReplicators() {
|
|
13478
|
-
console.log("[PLUGIN][INIT] initializeReplicators called");
|
|
14377
|
+
async initializeReplicators(database) {
|
|
13479
14378
|
for (const replicatorConfig of this.config.replicators) {
|
|
13480
|
-
|
|
13481
|
-
|
|
13482
|
-
|
|
13483
|
-
|
|
13484
|
-
|
|
13485
|
-
|
|
13486
|
-
|
|
13487
|
-
await replicator.initialize(this.database);
|
|
13488
|
-
this.replicators.push({
|
|
13489
|
-
id: Math.random().toString(36).slice(2),
|
|
13490
|
-
driver,
|
|
13491
|
-
config: replicatorConfig,
|
|
13492
|
-
resources,
|
|
13493
|
-
instance: replicator
|
|
13494
|
-
});
|
|
13495
|
-
console.log("[PLUGIN][INIT] pushed replicator:", driver, resources);
|
|
13496
|
-
} else {
|
|
13497
|
-
console.log("[PLUGIN][INIT] createReplicator returned null/undefined for driver:", driver);
|
|
13498
|
-
}
|
|
13499
|
-
} catch (err) {
|
|
13500
|
-
console.error("[PLUGIN][INIT] Error creating replicator:", err);
|
|
14379
|
+
const { driver, config = {}, resources, client, ...otherConfig } = replicatorConfig;
|
|
14380
|
+
const replicatorResources = resources || config.resources || {};
|
|
14381
|
+
const mergedConfig = { ...config, ...otherConfig };
|
|
14382
|
+
const replicator = this.createReplicator(driver, mergedConfig, replicatorResources, client);
|
|
14383
|
+
if (replicator) {
|
|
14384
|
+
await replicator.initialize(database);
|
|
14385
|
+
this.replicators.push(replicator);
|
|
13501
14386
|
}
|
|
13502
14387
|
}
|
|
13503
14388
|
}
|
|
@@ -13505,160 +14390,162 @@ class ReplicatorPlugin extends plugin_class_default {
|
|
|
13505
14390
|
}
|
|
13506
14391
|
async stop() {
|
|
13507
14392
|
}
|
|
13508
|
-
|
|
13509
|
-
if (
|
|
13510
|
-
|
|
13511
|
-
|
|
14393
|
+
filterInternalFields(data) {
|
|
14394
|
+
if (!data || typeof data !== "object") return data;
|
|
14395
|
+
const filtered = {};
|
|
14396
|
+
for (const [key, value] of Object.entries(data)) {
|
|
14397
|
+
if (!key.startsWith("_") && !key.startsWith("$")) {
|
|
14398
|
+
filtered[key] = value;
|
|
14399
|
+
}
|
|
13512
14400
|
}
|
|
13513
|
-
|
|
13514
|
-
|
|
14401
|
+
return filtered;
|
|
14402
|
+
}
|
|
14403
|
+
async uploadMetadataFile(database) {
|
|
14404
|
+
if (typeof database.uploadMetadataFile === "function") {
|
|
14405
|
+
await database.uploadMetadataFile();
|
|
13515
14406
|
}
|
|
13516
|
-
|
|
13517
|
-
|
|
14407
|
+
}
|
|
14408
|
+
async getCompleteData(resource, data) {
|
|
14409
|
+
try {
|
|
14410
|
+
const [ok, err, record] = await try_fn_default(() => resource.get(data.id));
|
|
14411
|
+
if (ok && record) {
|
|
14412
|
+
return record;
|
|
14413
|
+
}
|
|
14414
|
+
} catch (error) {
|
|
13518
14415
|
}
|
|
13519
|
-
|
|
13520
|
-
|
|
13521
|
-
|
|
14416
|
+
return data;
|
|
14417
|
+
}
|
|
14418
|
+
async retryWithBackoff(operation, maxRetries = 3) {
|
|
14419
|
+
let lastError;
|
|
14420
|
+
for (let attempt = 1; attempt <= maxRetries; attempt++) {
|
|
14421
|
+
try {
|
|
14422
|
+
return await operation();
|
|
14423
|
+
} catch (error) {
|
|
14424
|
+
lastError = error;
|
|
14425
|
+
if (attempt === maxRetries) {
|
|
14426
|
+
throw error;
|
|
14427
|
+
}
|
|
14428
|
+
const delay = Math.pow(2, attempt - 1) * 1e3;
|
|
14429
|
+
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
13522
14430
|
}
|
|
13523
|
-
return;
|
|
13524
14431
|
}
|
|
13525
|
-
|
|
13526
|
-
|
|
13527
|
-
|
|
13528
|
-
|
|
14432
|
+
throw lastError;
|
|
14433
|
+
}
|
|
14434
|
+
async logError(replicator, resourceName, operation, recordId, data, error) {
|
|
14435
|
+
try {
|
|
14436
|
+
const logResourceName = this.config.replicatorLogResource;
|
|
14437
|
+
if (this.database && this.database.resources && this.database.resources[logResourceName]) {
|
|
14438
|
+
const logResource = this.database.resources[logResourceName];
|
|
14439
|
+
await logResource.insert({
|
|
14440
|
+
replicator: replicator.name || replicator.id,
|
|
14441
|
+
resourceName,
|
|
14442
|
+
operation,
|
|
14443
|
+
recordId,
|
|
14444
|
+
data: JSON.stringify(data),
|
|
14445
|
+
error: error.message,
|
|
14446
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
14447
|
+
status: "error"
|
|
14448
|
+
});
|
|
13529
14449
|
}
|
|
14450
|
+
} catch (logError) {
|
|
14451
|
+
}
|
|
14452
|
+
}
|
|
14453
|
+
async processReplicatorEvent(operation, resourceName, recordId, data, beforeData = null) {
|
|
14454
|
+
if (!this.config.enabled) return;
|
|
14455
|
+
const applicableReplicators = this.replicators.filter((replicator) => {
|
|
14456
|
+
const should = replicator.shouldReplicateResource && replicator.shouldReplicateResource(resourceName, operation);
|
|
13530
14457
|
return should;
|
|
13531
14458
|
});
|
|
13532
|
-
if (this.config.verbose) {
|
|
13533
|
-
console.log(`[PLUGIN] processReplicatorEvent: applicableReplicators for resource=${resourceName}:`, applicableReplicators.map((r) => r.driver));
|
|
13534
|
-
}
|
|
13535
14459
|
if (applicableReplicators.length === 0) {
|
|
13536
|
-
if (this.config.verbose) {
|
|
13537
|
-
console.log("[PLUGIN] No applicable replicators for resource", resourceName);
|
|
13538
|
-
}
|
|
13539
14460
|
return;
|
|
13540
14461
|
}
|
|
13541
|
-
const
|
|
13542
|
-
|
|
13543
|
-
|
|
13544
|
-
|
|
13545
|
-
|
|
13546
|
-
|
|
13547
|
-
|
|
13548
|
-
|
|
13549
|
-
|
|
13550
|
-
|
|
13551
|
-
|
|
13552
|
-
|
|
13553
|
-
|
|
13554
|
-
const [ok, err, result] = await try_fn_default(async () => this.processreplicatorItem(item));
|
|
13555
|
-
if (ok) {
|
|
13556
|
-
if (logId) {
|
|
13557
|
-
await this.updatereplicatorLog(logId, {
|
|
13558
|
-
status: result.success ? "success" : "failed",
|
|
13559
|
-
attempts: 1,
|
|
13560
|
-
error: result.success ? "" : JSON.stringify(result.results)
|
|
14462
|
+
const promises = applicableReplicators.map(async (replicator) => {
|
|
14463
|
+
try {
|
|
14464
|
+
const result = await this.retryWithBackoff(
|
|
14465
|
+
() => replicator.replicate(resourceName, operation, data, recordId, beforeData),
|
|
14466
|
+
this.config.maxRetries
|
|
14467
|
+
);
|
|
14468
|
+
this.emit("replicated", {
|
|
14469
|
+
replicator: replicator.name || replicator.id,
|
|
14470
|
+
resourceName,
|
|
14471
|
+
operation,
|
|
14472
|
+
recordId,
|
|
14473
|
+
result,
|
|
14474
|
+
success: true
|
|
13561
14475
|
});
|
|
13562
|
-
|
|
13563
|
-
|
|
13564
|
-
|
|
13565
|
-
|
|
13566
|
-
|
|
13567
|
-
|
|
13568
|
-
|
|
13569
|
-
|
|
13570
|
-
if (logId) {
|
|
13571
|
-
await this.updatereplicatorLog(logId, {
|
|
13572
|
-
status: "failed",
|
|
13573
|
-
attempts: 1,
|
|
13574
|
-
error: err.message
|
|
14476
|
+
return result;
|
|
14477
|
+
} catch (error) {
|
|
14478
|
+
this.emit("replicator_error", {
|
|
14479
|
+
replicator: replicator.name || replicator.id,
|
|
14480
|
+
resourceName,
|
|
14481
|
+
operation,
|
|
14482
|
+
recordId,
|
|
14483
|
+
error: error.message
|
|
13575
14484
|
});
|
|
14485
|
+
if (this.config.logErrors && this.database) {
|
|
14486
|
+
await this.logError(replicator, resourceName, operation, recordId, data, error);
|
|
14487
|
+
}
|
|
14488
|
+
throw error;
|
|
13576
14489
|
}
|
|
13577
|
-
|
|
13578
|
-
|
|
14490
|
+
});
|
|
14491
|
+
return Promise.allSettled(promises);
|
|
13579
14492
|
}
|
|
13580
14493
|
async processreplicatorItem(item) {
|
|
13581
|
-
if (this.config.verbose) {
|
|
13582
|
-
console.log("[PLUGIN][processreplicatorItem] called with item:", item);
|
|
13583
|
-
}
|
|
13584
14494
|
const applicableReplicators = this.replicators.filter((replicator) => {
|
|
13585
|
-
const should = replicator.
|
|
13586
|
-
if (this.config.verbose) {
|
|
13587
|
-
console.log(`[PLUGIN] processreplicatorItem: Replicator ${replicator.driver} shouldReplicateResource(${item.resourceName}, ${item.operation}):`, should);
|
|
13588
|
-
}
|
|
14495
|
+
const should = replicator.shouldReplicateResource && replicator.shouldReplicateResource(item.resourceName, item.operation);
|
|
13589
14496
|
return should;
|
|
13590
14497
|
});
|
|
13591
|
-
if (this.config.verbose) {
|
|
13592
|
-
console.log(`[PLUGIN] processreplicatorItem: applicableReplicators for resource=${item.resourceName}:`, applicableReplicators.map((r) => r.driver));
|
|
13593
|
-
}
|
|
13594
14498
|
if (applicableReplicators.length === 0) {
|
|
13595
|
-
|
|
13596
|
-
console.log("[PLUGIN] processreplicatorItem: No applicable replicators for resource", item.resourceName);
|
|
13597
|
-
}
|
|
13598
|
-
return { success: true, skipped: true, reason: "no_applicable_replicators" };
|
|
14499
|
+
return;
|
|
13599
14500
|
}
|
|
13600
|
-
const
|
|
13601
|
-
|
|
13602
|
-
|
|
13603
|
-
|
|
13604
|
-
|
|
13605
|
-
|
|
13606
|
-
|
|
14501
|
+
const promises = applicableReplicators.map(async (replicator) => {
|
|
14502
|
+
try {
|
|
14503
|
+
const [ok, err, result] = await try_fn_default(
|
|
14504
|
+
() => replicator.replicate(item.resourceName, item.operation, item.data, item.recordId, item.beforeData)
|
|
14505
|
+
);
|
|
14506
|
+
if (!ok) {
|
|
14507
|
+
this.emit("replicator_error", {
|
|
14508
|
+
replicator: replicator.name || replicator.id,
|
|
14509
|
+
resourceName: item.resourceName,
|
|
14510
|
+
operation: item.operation,
|
|
14511
|
+
recordId: item.recordId,
|
|
14512
|
+
error: err.message
|
|
14513
|
+
});
|
|
14514
|
+
if (this.config.logErrors && this.database) {
|
|
14515
|
+
await this.logError(replicator, item.resourceName, item.operation, item.recordId, item.data, err);
|
|
14516
|
+
}
|
|
14517
|
+
return { success: false, error: err.message };
|
|
14518
|
+
}
|
|
14519
|
+
this.emit("replicated", {
|
|
14520
|
+
replicator: replicator.name || replicator.id,
|
|
14521
|
+
resourceName: item.resourceName,
|
|
13607
14522
|
operation: item.operation,
|
|
13608
|
-
|
|
13609
|
-
|
|
13610
|
-
|
|
14523
|
+
recordId: item.recordId,
|
|
14524
|
+
result,
|
|
14525
|
+
success: true
|
|
13611
14526
|
});
|
|
14527
|
+
return { success: true, result };
|
|
14528
|
+
} catch (error) {
|
|
14529
|
+
this.emit("replicator_error", {
|
|
14530
|
+
replicator: replicator.name || replicator.id,
|
|
14531
|
+
resourceName: item.resourceName,
|
|
14532
|
+
operation: item.operation,
|
|
14533
|
+
recordId: item.recordId,
|
|
14534
|
+
error: error.message
|
|
14535
|
+
});
|
|
14536
|
+
if (this.config.logErrors && this.database) {
|
|
14537
|
+
await this.logError(replicator, item.resourceName, item.operation, item.recordId, item.data, error);
|
|
14538
|
+
}
|
|
14539
|
+
return { success: false, error: error.message };
|
|
13612
14540
|
}
|
|
13613
|
-
|
|
13614
|
-
|
|
13615
|
-
() => replicator.instance.replicate({
|
|
13616
|
-
resource: item.resourceName,
|
|
13617
|
-
operation: item.operation,
|
|
13618
|
-
data: item.data,
|
|
13619
|
-
id: item.recordId,
|
|
13620
|
-
beforeData: item.beforeData
|
|
13621
|
-
})
|
|
13622
|
-
);
|
|
13623
|
-
} else {
|
|
13624
|
-
[ok, err, result] = await try_fn_default(
|
|
13625
|
-
() => replicator.instance.replicate(
|
|
13626
|
-
item.resourceName,
|
|
13627
|
-
item.operation,
|
|
13628
|
-
item.data,
|
|
13629
|
-
item.recordId,
|
|
13630
|
-
item.beforeData
|
|
13631
|
-
)
|
|
13632
|
-
);
|
|
13633
|
-
}
|
|
13634
|
-
results.push({
|
|
13635
|
-
replicatorId: replicator.id,
|
|
13636
|
-
driver: replicator.driver,
|
|
13637
|
-
success: result && result.success,
|
|
13638
|
-
error: result && result.error,
|
|
13639
|
-
skipped: result && result.skipped
|
|
13640
|
-
});
|
|
13641
|
-
}
|
|
13642
|
-
return {
|
|
13643
|
-
success: results.every((r) => r.success || r.skipped),
|
|
13644
|
-
results
|
|
13645
|
-
};
|
|
14541
|
+
});
|
|
14542
|
+
return Promise.allSettled(promises);
|
|
13646
14543
|
}
|
|
13647
14544
|
async logreplicator(item) {
|
|
13648
14545
|
const logRes = this.replicatorLog || this.database.resources[normalizeResourceName(this.config.replicatorLogResource)];
|
|
13649
14546
|
if (!logRes) {
|
|
13650
|
-
if (this.config.verbose) {
|
|
13651
|
-
console.error("[PLUGIN] replicator log resource not found!");
|
|
13652
|
-
}
|
|
13653
14547
|
if (this.database) {
|
|
13654
|
-
if (this.
|
|
13655
|
-
console.warn("[PLUGIN] database.resources keys:", Object.keys(this.database.resources));
|
|
13656
|
-
}
|
|
13657
|
-
if (this.database.options && this.database.options.connectionString) {
|
|
13658
|
-
if (this.config.verbose) {
|
|
13659
|
-
console.warn("[PLUGIN] database connectionString:", this.database.options.connectionString);
|
|
13660
|
-
}
|
|
13661
|
-
}
|
|
14548
|
+
if (this.database.options && this.database.options.connectionString) ;
|
|
13662
14549
|
}
|
|
13663
14550
|
this.emit("replicator.log.failed", { error: "replicator log resource not found", item });
|
|
13664
14551
|
return;
|
|
@@ -13674,9 +14561,6 @@ class ReplicatorPlugin extends plugin_class_default {
|
|
|
13674
14561
|
try {
|
|
13675
14562
|
await logRes.insert(logItem);
|
|
13676
14563
|
} catch (err) {
|
|
13677
|
-
if (this.config.verbose) {
|
|
13678
|
-
console.error("[PLUGIN] Error writing to replicator log:", err);
|
|
13679
|
-
}
|
|
13680
14564
|
this.emit("replicator.log.failed", { error: err, item });
|
|
13681
14565
|
}
|
|
13682
14566
|
}
|
|
@@ -13696,7 +14580,7 @@ class ReplicatorPlugin extends plugin_class_default {
|
|
|
13696
14580
|
async getreplicatorStats() {
|
|
13697
14581
|
const replicatorStats = await Promise.all(
|
|
13698
14582
|
this.replicators.map(async (replicator) => {
|
|
13699
|
-
const status = await replicator.
|
|
14583
|
+
const status = await replicator.getStatus();
|
|
13700
14584
|
return {
|
|
13701
14585
|
id: replicator.id,
|
|
13702
14586
|
driver: replicator.driver,
|
|
@@ -13758,10 +14642,6 @@ class ReplicatorPlugin extends plugin_class_default {
|
|
|
13758
14642
|
});
|
|
13759
14643
|
if (ok) {
|
|
13760
14644
|
retried++;
|
|
13761
|
-
} else {
|
|
13762
|
-
if (this.config.verbose) {
|
|
13763
|
-
console.error("Failed to retry replicator:", err);
|
|
13764
|
-
}
|
|
13765
14645
|
}
|
|
13766
14646
|
}
|
|
13767
14647
|
return { retried };
|
|
@@ -13774,52 +14654,35 @@ class ReplicatorPlugin extends plugin_class_default {
|
|
|
13774
14654
|
this.stats.lastSync = (/* @__PURE__ */ new Date()).toISOString();
|
|
13775
14655
|
for (const resourceName in this.database.resources) {
|
|
13776
14656
|
if (normalizeResourceName(resourceName) === normalizeResourceName("replicator_logs")) continue;
|
|
13777
|
-
if (replicator.
|
|
14657
|
+
if (replicator.shouldReplicateResource(resourceName)) {
|
|
13778
14658
|
this.emit("replicator.sync.resource", { resourceName, replicatorId });
|
|
13779
14659
|
const resource = this.database.resources[resourceName];
|
|
13780
14660
|
const allRecords = await resource.getAll();
|
|
13781
14661
|
for (const record of allRecords) {
|
|
13782
|
-
await replicator.
|
|
14662
|
+
await replicator.replicate(resourceName, "insert", record, record.id);
|
|
13783
14663
|
}
|
|
13784
14664
|
}
|
|
13785
14665
|
}
|
|
13786
14666
|
this.emit("replicator.sync.completed", { replicatorId, stats: this.stats });
|
|
13787
14667
|
}
|
|
13788
14668
|
async cleanup() {
|
|
13789
|
-
|
|
13790
|
-
|
|
13791
|
-
|
|
13792
|
-
|
|
13793
|
-
|
|
13794
|
-
|
|
13795
|
-
|
|
13796
|
-
|
|
13797
|
-
|
|
13798
|
-
}
|
|
13799
|
-
|
|
13800
|
-
}
|
|
13801
|
-
this._installedListeners = [];
|
|
13802
|
-
}
|
|
13803
|
-
if (this.database && typeof this.database.removeAllListeners === "function") {
|
|
13804
|
-
this.database.removeAllListeners();
|
|
13805
|
-
}
|
|
13806
|
-
if (this.replicators && Array.isArray(this.replicators)) {
|
|
13807
|
-
for (const rep of this.replicators) {
|
|
13808
|
-
if (rep.instance && typeof rep.instance.cleanup === "function") {
|
|
13809
|
-
await rep.instance.cleanup();
|
|
13810
|
-
}
|
|
14669
|
+
try {
|
|
14670
|
+
if (this.replicators && this.replicators.length > 0) {
|
|
14671
|
+
const cleanupPromises = this.replicators.map(async (replicator) => {
|
|
14672
|
+
try {
|
|
14673
|
+
if (replicator && typeof replicator.cleanup === "function") {
|
|
14674
|
+
await replicator.cleanup();
|
|
14675
|
+
}
|
|
14676
|
+
} catch (error) {
|
|
14677
|
+
}
|
|
14678
|
+
});
|
|
14679
|
+
await Promise.allSettled(cleanupPromises);
|
|
13811
14680
|
}
|
|
13812
14681
|
this.replicators = [];
|
|
13813
|
-
|
|
13814
|
-
|
|
13815
|
-
|
|
13816
|
-
|
|
13817
|
-
totalOperations: 0,
|
|
13818
|
-
totalErrors: 0,
|
|
13819
|
-
lastError: null
|
|
13820
|
-
};
|
|
13821
|
-
if (this.config.verbose) {
|
|
13822
|
-
console.log("[PLUGIN][CLEANUP] ReplicatorPlugin cleanup complete");
|
|
14682
|
+
this.database = null;
|
|
14683
|
+
this.eventListenersInstalled.clear();
|
|
14684
|
+
this.removeAllListeners();
|
|
14685
|
+
} catch (error) {
|
|
13823
14686
|
}
|
|
13824
14687
|
}
|
|
13825
14688
|
}
|