s3db.js 7.2.1 → 7.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/s3db.cjs.js +1241 -378
- package/dist/s3db.cjs.min.js +1 -1
- package/dist/s3db.es.js +1242 -379
- package/dist/s3db.es.min.js +1 -1
- package/dist/s3db.iife.js +1240 -377
- package/dist/s3db.iife.min.js +1 -1
- package/package.json +30 -24
- package/src/behaviors/body-only.js +2 -2
- package/src/behaviors/truncate-data.js +2 -2
- package/src/client.class.js +1 -1
- package/src/database.class.js +1 -1
- package/src/errors.js +1 -1
- package/src/plugins/audit.plugin.js +5 -5
- package/src/plugins/cache/filesystem-cache.class.js +661 -0
- package/src/plugins/cache/index.js +4 -0
- package/src/plugins/cache/partition-aware-filesystem-cache.class.js +480 -0
- package/src/plugins/cache/s3-cache.class.js +1 -1
- package/src/plugins/cache.plugin.js +159 -9
- package/src/plugins/consumers/index.js +3 -3
- package/src/plugins/consumers/sqs-consumer.js +2 -2
- package/src/plugins/fulltext.plugin.js +5 -5
- package/src/plugins/metrics.plugin.js +2 -2
- package/src/plugins/queue-consumer.plugin.js +3 -3
- package/src/plugins/replicator.plugin.js +259 -362
- package/src/plugins/replicators/s3db-replicator.class.js +35 -19
- package/src/plugins/replicators/sqs-replicator.class.js +17 -5
- package/src/resource.class.js +14 -14
- package/src/schema.class.js +3 -3
package/dist/s3db.es.js
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { customAlphabet, urlAlphabet } from 'nanoid';
|
|
2
|
-
import zlib from 'zlib';
|
|
2
|
+
import zlib from 'node:zlib';
|
|
3
3
|
import { PromisePool } from '@supercharge/promise-pool';
|
|
4
4
|
import { ReadableStream } from 'node:stream/web';
|
|
5
|
-
import { chunk, merge, isString as isString$1, isEmpty, invert, uniq, cloneDeep, get, set, isObject as isObject$1, isFunction as isFunction$1
|
|
5
|
+
import { chunk, merge, isString as isString$1, isEmpty, invert, uniq, cloneDeep, get, set, isObject as isObject$1, isFunction as isFunction$1 } from 'lodash-es';
|
|
6
6
|
import { createHash } from 'crypto';
|
|
7
7
|
import jsonStableStringify from 'json-stable-stringify';
|
|
8
8
|
import { S3Client, PutObjectCommand, GetObjectCommand, HeadObjectCommand, CopyObjectCommand, DeleteObjectCommand, DeleteObjectsCommand, ListObjectsV2Command } from '@aws-sdk/client-s3';
|
|
@@ -1341,7 +1341,8 @@ class AuditPlugin extends plugin_class_default {
|
|
|
1341
1341
|
version: "2.0"
|
|
1342
1342
|
})
|
|
1343
1343
|
};
|
|
1344
|
-
this.logAudit(auditRecord).catch(
|
|
1344
|
+
this.logAudit(auditRecord).catch(() => {
|
|
1345
|
+
});
|
|
1345
1346
|
});
|
|
1346
1347
|
resource.on("update", async (data) => {
|
|
1347
1348
|
const recordId = data.id;
|
|
@@ -1367,7 +1368,8 @@ class AuditPlugin extends plugin_class_default {
|
|
|
1367
1368
|
version: "2.0"
|
|
1368
1369
|
})
|
|
1369
1370
|
};
|
|
1370
|
-
this.logAudit(auditRecord).catch(
|
|
1371
|
+
this.logAudit(auditRecord).catch(() => {
|
|
1372
|
+
});
|
|
1371
1373
|
});
|
|
1372
1374
|
resource.on("delete", async (data) => {
|
|
1373
1375
|
const recordId = data.id;
|
|
@@ -1393,7 +1395,8 @@ class AuditPlugin extends plugin_class_default {
|
|
|
1393
1395
|
version: "2.0"
|
|
1394
1396
|
})
|
|
1395
1397
|
};
|
|
1396
|
-
this.logAudit(auditRecord).catch(
|
|
1398
|
+
this.logAudit(auditRecord).catch(() => {
|
|
1399
|
+
});
|
|
1397
1400
|
});
|
|
1398
1401
|
resource.useMiddleware("deleteMany", async (ctx, next) => {
|
|
1399
1402
|
const ids = ctx.args[0];
|
|
@@ -1426,7 +1429,8 @@ class AuditPlugin extends plugin_class_default {
|
|
|
1426
1429
|
batchOperation: true
|
|
1427
1430
|
})
|
|
1428
1431
|
};
|
|
1429
|
-
this.logAudit(auditRecord).catch(
|
|
1432
|
+
this.logAudit(auditRecord).catch(() => {
|
|
1433
|
+
});
|
|
1430
1434
|
}
|
|
1431
1435
|
}
|
|
1432
1436
|
return result;
|
|
@@ -2073,6 +2077,16 @@ if (typeof Object.create === 'function'){
|
|
|
2073
2077
|
// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
|
|
2074
2078
|
// USE OR OTHER DEALINGS IN THE SOFTWARE.
|
|
2075
2079
|
|
|
2080
|
+
var getOwnPropertyDescriptors = Object.getOwnPropertyDescriptors ||
|
|
2081
|
+
function getOwnPropertyDescriptors(obj) {
|
|
2082
|
+
var keys = Object.keys(obj);
|
|
2083
|
+
var descriptors = {};
|
|
2084
|
+
for (var i = 0; i < keys.length; i++) {
|
|
2085
|
+
descriptors[keys[i]] = Object.getOwnPropertyDescriptor(obj, keys[i]);
|
|
2086
|
+
}
|
|
2087
|
+
return descriptors;
|
|
2088
|
+
};
|
|
2089
|
+
|
|
2076
2090
|
var formatRegExp = /%[sdj%]/g;
|
|
2077
2091
|
function format(f) {
|
|
2078
2092
|
if (!isString(f)) {
|
|
@@ -2558,6 +2572,64 @@ function hasOwnProperty(obj, prop) {
|
|
|
2558
2572
|
return Object.prototype.hasOwnProperty.call(obj, prop);
|
|
2559
2573
|
}
|
|
2560
2574
|
|
|
2575
|
+
var kCustomPromisifiedSymbol = typeof Symbol !== 'undefined' ? Symbol('util.promisify.custom') : undefined;
|
|
2576
|
+
|
|
2577
|
+
function promisify(original) {
|
|
2578
|
+
if (typeof original !== 'function')
|
|
2579
|
+
throw new TypeError('The "original" argument must be of type Function');
|
|
2580
|
+
|
|
2581
|
+
if (kCustomPromisifiedSymbol && original[kCustomPromisifiedSymbol]) {
|
|
2582
|
+
var fn = original[kCustomPromisifiedSymbol];
|
|
2583
|
+
if (typeof fn !== 'function') {
|
|
2584
|
+
throw new TypeError('The "util.promisify.custom" argument must be of type Function');
|
|
2585
|
+
}
|
|
2586
|
+
Object.defineProperty(fn, kCustomPromisifiedSymbol, {
|
|
2587
|
+
value: fn, enumerable: false, writable: false, configurable: true
|
|
2588
|
+
});
|
|
2589
|
+
return fn;
|
|
2590
|
+
}
|
|
2591
|
+
|
|
2592
|
+
function fn() {
|
|
2593
|
+
var promiseResolve, promiseReject;
|
|
2594
|
+
var promise = new Promise(function (resolve, reject) {
|
|
2595
|
+
promiseResolve = resolve;
|
|
2596
|
+
promiseReject = reject;
|
|
2597
|
+
});
|
|
2598
|
+
|
|
2599
|
+
var args = [];
|
|
2600
|
+
for (var i = 0; i < arguments.length; i++) {
|
|
2601
|
+
args.push(arguments[i]);
|
|
2602
|
+
}
|
|
2603
|
+
args.push(function (err, value) {
|
|
2604
|
+
if (err) {
|
|
2605
|
+
promiseReject(err);
|
|
2606
|
+
} else {
|
|
2607
|
+
promiseResolve(value);
|
|
2608
|
+
}
|
|
2609
|
+
});
|
|
2610
|
+
|
|
2611
|
+
try {
|
|
2612
|
+
original.apply(this, args);
|
|
2613
|
+
} catch (err) {
|
|
2614
|
+
promiseReject(err);
|
|
2615
|
+
}
|
|
2616
|
+
|
|
2617
|
+
return promise;
|
|
2618
|
+
}
|
|
2619
|
+
|
|
2620
|
+
Object.setPrototypeOf(fn, Object.getPrototypeOf(original));
|
|
2621
|
+
|
|
2622
|
+
if (kCustomPromisifiedSymbol) Object.defineProperty(fn, kCustomPromisifiedSymbol, {
|
|
2623
|
+
value: fn, enumerable: false, writable: false, configurable: true
|
|
2624
|
+
});
|
|
2625
|
+
return Object.defineProperties(
|
|
2626
|
+
fn,
|
|
2627
|
+
getOwnPropertyDescriptors(original)
|
|
2628
|
+
);
|
|
2629
|
+
}
|
|
2630
|
+
|
|
2631
|
+
promisify.custom = kCustomPromisifiedSymbol;
|
|
2632
|
+
|
|
2561
2633
|
var lookup = [];
|
|
2562
2634
|
var revLookup = [];
|
|
2563
2635
|
var Arr = typeof Uint8Array !== 'undefined' ? Uint8Array : Array;
|
|
@@ -6808,12 +6880,798 @@ class MemoryCache extends Cache {
|
|
|
6808
6880
|
}
|
|
6809
6881
|
var memory_cache_class_default = MemoryCache;
|
|
6810
6882
|
|
|
6883
|
+
var fs = {};
|
|
6884
|
+
|
|
6885
|
+
const readFile$1 = promisify(fs.readFile);
|
|
6886
|
+
const writeFile$1 = promisify(fs.writeFile);
|
|
6887
|
+
const unlink = promisify(fs.unlink);
|
|
6888
|
+
const readdir$1 = promisify(fs.readdir);
|
|
6889
|
+
const stat$1 = promisify(fs.stat);
|
|
6890
|
+
const mkdir = promisify(fs.mkdir);
|
|
6891
|
+
class FilesystemCache extends Cache {
|
|
6892
|
+
constructor({
|
|
6893
|
+
directory,
|
|
6894
|
+
prefix = "cache",
|
|
6895
|
+
ttl = 36e5,
|
|
6896
|
+
enableCompression = true,
|
|
6897
|
+
compressionThreshold = 1024,
|
|
6898
|
+
createDirectory = true,
|
|
6899
|
+
fileExtension = ".cache",
|
|
6900
|
+
enableMetadata = true,
|
|
6901
|
+
maxFileSize = 10485760,
|
|
6902
|
+
// 10MB
|
|
6903
|
+
enableStats = false,
|
|
6904
|
+
enableCleanup = true,
|
|
6905
|
+
cleanupInterval = 3e5,
|
|
6906
|
+
// 5 minutes
|
|
6907
|
+
encoding = "utf8",
|
|
6908
|
+
fileMode = 420,
|
|
6909
|
+
enableBackup = false,
|
|
6910
|
+
backupSuffix = ".bak",
|
|
6911
|
+
enableLocking = false,
|
|
6912
|
+
lockTimeout = 5e3,
|
|
6913
|
+
enableJournal = false,
|
|
6914
|
+
journalFile = "cache.journal",
|
|
6915
|
+
...config
|
|
6916
|
+
}) {
|
|
6917
|
+
super(config);
|
|
6918
|
+
if (!directory) {
|
|
6919
|
+
throw new Error("FilesystemCache: directory parameter is required");
|
|
6920
|
+
}
|
|
6921
|
+
this.directory = path.resolve(directory);
|
|
6922
|
+
this.prefix = prefix;
|
|
6923
|
+
this.ttl = ttl;
|
|
6924
|
+
this.enableCompression = enableCompression;
|
|
6925
|
+
this.compressionThreshold = compressionThreshold;
|
|
6926
|
+
this.createDirectory = createDirectory;
|
|
6927
|
+
this.fileExtension = fileExtension;
|
|
6928
|
+
this.enableMetadata = enableMetadata;
|
|
6929
|
+
this.maxFileSize = maxFileSize;
|
|
6930
|
+
this.enableStats = enableStats;
|
|
6931
|
+
this.enableCleanup = enableCleanup;
|
|
6932
|
+
this.cleanupInterval = cleanupInterval;
|
|
6933
|
+
this.encoding = encoding;
|
|
6934
|
+
this.fileMode = fileMode;
|
|
6935
|
+
this.enableBackup = enableBackup;
|
|
6936
|
+
this.backupSuffix = backupSuffix;
|
|
6937
|
+
this.enableLocking = enableLocking;
|
|
6938
|
+
this.lockTimeout = lockTimeout;
|
|
6939
|
+
this.enableJournal = enableJournal;
|
|
6940
|
+
this.journalFile = path.join(this.directory, journalFile);
|
|
6941
|
+
this.stats = {
|
|
6942
|
+
hits: 0,
|
|
6943
|
+
misses: 0,
|
|
6944
|
+
sets: 0,
|
|
6945
|
+
deletes: 0,
|
|
6946
|
+
clears: 0,
|
|
6947
|
+
errors: 0
|
|
6948
|
+
};
|
|
6949
|
+
this.locks = /* @__PURE__ */ new Map();
|
|
6950
|
+
this.cleanupTimer = null;
|
|
6951
|
+
this._init();
|
|
6952
|
+
}
|
|
6953
|
+
async _init() {
|
|
6954
|
+
if (this.createDirectory) {
|
|
6955
|
+
await this._ensureDirectory(this.directory);
|
|
6956
|
+
}
|
|
6957
|
+
if (this.enableCleanup && this.cleanupInterval > 0) {
|
|
6958
|
+
this.cleanupTimer = setInterval(() => {
|
|
6959
|
+
this._cleanup().catch((err) => {
|
|
6960
|
+
console.warn("FilesystemCache cleanup error:", err.message);
|
|
6961
|
+
});
|
|
6962
|
+
}, this.cleanupInterval);
|
|
6963
|
+
}
|
|
6964
|
+
}
|
|
6965
|
+
async _ensureDirectory(dir) {
|
|
6966
|
+
const [ok, err] = await try_fn_default(async () => {
|
|
6967
|
+
await mkdir(dir, { recursive: true });
|
|
6968
|
+
});
|
|
6969
|
+
if (!ok && err.code !== "EEXIST") {
|
|
6970
|
+
throw new Error(`Failed to create cache directory: ${err.message}`);
|
|
6971
|
+
}
|
|
6972
|
+
}
|
|
6973
|
+
_getFilePath(key) {
|
|
6974
|
+
const sanitizedKey = key.replace(/[<>:"/\\|?*]/g, "_");
|
|
6975
|
+
const filename = `${this.prefix}_${sanitizedKey}${this.fileExtension}`;
|
|
6976
|
+
return path.join(this.directory, filename);
|
|
6977
|
+
}
|
|
6978
|
+
_getMetadataPath(filePath) {
|
|
6979
|
+
return filePath + ".meta";
|
|
6980
|
+
}
|
|
6981
|
+
async _set(key, data) {
|
|
6982
|
+
const filePath = this._getFilePath(key);
|
|
6983
|
+
try {
|
|
6984
|
+
let serialized = JSON.stringify(data);
|
|
6985
|
+
const originalSize = Buffer.byteLength(serialized, this.encoding);
|
|
6986
|
+
if (originalSize > this.maxFileSize) {
|
|
6987
|
+
throw new Error(`Cache data exceeds maximum file size: ${originalSize} > ${this.maxFileSize}`);
|
|
6988
|
+
}
|
|
6989
|
+
let compressed = false;
|
|
6990
|
+
let finalData = serialized;
|
|
6991
|
+
if (this.enableCompression && originalSize >= this.compressionThreshold) {
|
|
6992
|
+
const compressedBuffer = zlib.gzipSync(Buffer.from(serialized, this.encoding));
|
|
6993
|
+
finalData = compressedBuffer.toString("base64");
|
|
6994
|
+
compressed = true;
|
|
6995
|
+
}
|
|
6996
|
+
if (this.enableBackup && await this._fileExists(filePath)) {
|
|
6997
|
+
const backupPath = filePath + this.backupSuffix;
|
|
6998
|
+
await this._copyFile(filePath, backupPath);
|
|
6999
|
+
}
|
|
7000
|
+
if (this.enableLocking) {
|
|
7001
|
+
await this._acquireLock(filePath);
|
|
7002
|
+
}
|
|
7003
|
+
try {
|
|
7004
|
+
await writeFile$1(filePath, finalData, {
|
|
7005
|
+
encoding: compressed ? "utf8" : this.encoding,
|
|
7006
|
+
mode: this.fileMode
|
|
7007
|
+
});
|
|
7008
|
+
if (this.enableMetadata) {
|
|
7009
|
+
const metadata = {
|
|
7010
|
+
key,
|
|
7011
|
+
timestamp: Date.now(),
|
|
7012
|
+
ttl: this.ttl,
|
|
7013
|
+
compressed,
|
|
7014
|
+
originalSize,
|
|
7015
|
+
compressedSize: compressed ? Buffer.byteLength(finalData, "utf8") : originalSize,
|
|
7016
|
+
compressionRatio: compressed ? (Buffer.byteLength(finalData, "utf8") / originalSize).toFixed(2) : 1
|
|
7017
|
+
};
|
|
7018
|
+
await writeFile$1(this._getMetadataPath(filePath), JSON.stringify(metadata), {
|
|
7019
|
+
encoding: this.encoding,
|
|
7020
|
+
mode: this.fileMode
|
|
7021
|
+
});
|
|
7022
|
+
}
|
|
7023
|
+
if (this.enableStats) {
|
|
7024
|
+
this.stats.sets++;
|
|
7025
|
+
}
|
|
7026
|
+
if (this.enableJournal) {
|
|
7027
|
+
await this._journalOperation("set", key, { size: originalSize, compressed });
|
|
7028
|
+
}
|
|
7029
|
+
} finally {
|
|
7030
|
+
if (this.enableLocking) {
|
|
7031
|
+
this._releaseLock(filePath);
|
|
7032
|
+
}
|
|
7033
|
+
}
|
|
7034
|
+
return data;
|
|
7035
|
+
} catch (error) {
|
|
7036
|
+
if (this.enableStats) {
|
|
7037
|
+
this.stats.errors++;
|
|
7038
|
+
}
|
|
7039
|
+
throw new Error(`Failed to set cache key '${key}': ${error.message}`);
|
|
7040
|
+
}
|
|
7041
|
+
}
|
|
7042
|
+
async _get(key) {
|
|
7043
|
+
const filePath = this._getFilePath(key);
|
|
7044
|
+
try {
|
|
7045
|
+
if (!await this._fileExists(filePath)) {
|
|
7046
|
+
if (this.enableStats) {
|
|
7047
|
+
this.stats.misses++;
|
|
7048
|
+
}
|
|
7049
|
+
return null;
|
|
7050
|
+
}
|
|
7051
|
+
let isExpired = false;
|
|
7052
|
+
if (this.enableMetadata) {
|
|
7053
|
+
const metadataPath = this._getMetadataPath(filePath);
|
|
7054
|
+
if (await this._fileExists(metadataPath)) {
|
|
7055
|
+
const [ok, err, metadata] = await try_fn_default(async () => {
|
|
7056
|
+
const metaContent = await readFile$1(metadataPath, this.encoding);
|
|
7057
|
+
return JSON.parse(metaContent);
|
|
7058
|
+
});
|
|
7059
|
+
if (ok && metadata.ttl > 0) {
|
|
7060
|
+
const age = Date.now() - metadata.timestamp;
|
|
7061
|
+
isExpired = age > metadata.ttl;
|
|
7062
|
+
}
|
|
7063
|
+
}
|
|
7064
|
+
} else if (this.ttl > 0) {
|
|
7065
|
+
const stats = await stat$1(filePath);
|
|
7066
|
+
const age = Date.now() - stats.mtime.getTime();
|
|
7067
|
+
isExpired = age > this.ttl;
|
|
7068
|
+
}
|
|
7069
|
+
if (isExpired) {
|
|
7070
|
+
await this._del(key);
|
|
7071
|
+
if (this.enableStats) {
|
|
7072
|
+
this.stats.misses++;
|
|
7073
|
+
}
|
|
7074
|
+
return null;
|
|
7075
|
+
}
|
|
7076
|
+
if (this.enableLocking) {
|
|
7077
|
+
await this._acquireLock(filePath);
|
|
7078
|
+
}
|
|
7079
|
+
try {
|
|
7080
|
+
const content = await readFile$1(filePath, this.encoding);
|
|
7081
|
+
let isCompressed = false;
|
|
7082
|
+
if (this.enableMetadata) {
|
|
7083
|
+
const metadataPath = this._getMetadataPath(filePath);
|
|
7084
|
+
if (await this._fileExists(metadataPath)) {
|
|
7085
|
+
const [ok, err, metadata] = await try_fn_default(async () => {
|
|
7086
|
+
const metaContent = await readFile$1(metadataPath, this.encoding);
|
|
7087
|
+
return JSON.parse(metaContent);
|
|
7088
|
+
});
|
|
7089
|
+
if (ok) {
|
|
7090
|
+
isCompressed = metadata.compressed;
|
|
7091
|
+
}
|
|
7092
|
+
}
|
|
7093
|
+
}
|
|
7094
|
+
let finalContent = content;
|
|
7095
|
+
if (isCompressed || this.enableCompression && content.match(/^[A-Za-z0-9+/=]+$/)) {
|
|
7096
|
+
try {
|
|
7097
|
+
const compressedBuffer = Buffer.from(content, "base64");
|
|
7098
|
+
finalContent = zlib.gunzipSync(compressedBuffer).toString(this.encoding);
|
|
7099
|
+
} catch (decompressError) {
|
|
7100
|
+
finalContent = content;
|
|
7101
|
+
}
|
|
7102
|
+
}
|
|
7103
|
+
const data = JSON.parse(finalContent);
|
|
7104
|
+
if (this.enableStats) {
|
|
7105
|
+
this.stats.hits++;
|
|
7106
|
+
}
|
|
7107
|
+
return data;
|
|
7108
|
+
} finally {
|
|
7109
|
+
if (this.enableLocking) {
|
|
7110
|
+
this._releaseLock(filePath);
|
|
7111
|
+
}
|
|
7112
|
+
}
|
|
7113
|
+
} catch (error) {
|
|
7114
|
+
if (this.enableStats) {
|
|
7115
|
+
this.stats.errors++;
|
|
7116
|
+
}
|
|
7117
|
+
await this._del(key);
|
|
7118
|
+
return null;
|
|
7119
|
+
}
|
|
7120
|
+
}
|
|
7121
|
+
async _del(key) {
|
|
7122
|
+
const filePath = this._getFilePath(key);
|
|
7123
|
+
try {
|
|
7124
|
+
if (await this._fileExists(filePath)) {
|
|
7125
|
+
await unlink(filePath);
|
|
7126
|
+
}
|
|
7127
|
+
if (this.enableMetadata) {
|
|
7128
|
+
const metadataPath = this._getMetadataPath(filePath);
|
|
7129
|
+
if (await this._fileExists(metadataPath)) {
|
|
7130
|
+
await unlink(metadataPath);
|
|
7131
|
+
}
|
|
7132
|
+
}
|
|
7133
|
+
if (this.enableBackup) {
|
|
7134
|
+
const backupPath = filePath + this.backupSuffix;
|
|
7135
|
+
if (await this._fileExists(backupPath)) {
|
|
7136
|
+
await unlink(backupPath);
|
|
7137
|
+
}
|
|
7138
|
+
}
|
|
7139
|
+
if (this.enableStats) {
|
|
7140
|
+
this.stats.deletes++;
|
|
7141
|
+
}
|
|
7142
|
+
if (this.enableJournal) {
|
|
7143
|
+
await this._journalOperation("delete", key);
|
|
7144
|
+
}
|
|
7145
|
+
return true;
|
|
7146
|
+
} catch (error) {
|
|
7147
|
+
if (this.enableStats) {
|
|
7148
|
+
this.stats.errors++;
|
|
7149
|
+
}
|
|
7150
|
+
throw new Error(`Failed to delete cache key '${key}': ${error.message}`);
|
|
7151
|
+
}
|
|
7152
|
+
}
|
|
7153
|
+
async _clear(prefix) {
|
|
7154
|
+
try {
|
|
7155
|
+
const files = await readdir$1(this.directory);
|
|
7156
|
+
const cacheFiles = files.filter((file) => {
|
|
7157
|
+
if (!file.startsWith(this.prefix)) return false;
|
|
7158
|
+
if (!file.endsWith(this.fileExtension)) return false;
|
|
7159
|
+
if (prefix) {
|
|
7160
|
+
const keyPart = file.slice(this.prefix.length + 1, -this.fileExtension.length);
|
|
7161
|
+
return keyPart.startsWith(prefix);
|
|
7162
|
+
}
|
|
7163
|
+
return true;
|
|
7164
|
+
});
|
|
7165
|
+
for (const file of cacheFiles) {
|
|
7166
|
+
const filePath = path.join(this.directory, file);
|
|
7167
|
+
if (await this._fileExists(filePath)) {
|
|
7168
|
+
await unlink(filePath);
|
|
7169
|
+
}
|
|
7170
|
+
if (this.enableMetadata) {
|
|
7171
|
+
const metadataPath = this._getMetadataPath(filePath);
|
|
7172
|
+
if (await this._fileExists(metadataPath)) {
|
|
7173
|
+
await unlink(metadataPath);
|
|
7174
|
+
}
|
|
7175
|
+
}
|
|
7176
|
+
if (this.enableBackup) {
|
|
7177
|
+
const backupPath = filePath + this.backupSuffix;
|
|
7178
|
+
if (await this._fileExists(backupPath)) {
|
|
7179
|
+
await unlink(backupPath);
|
|
7180
|
+
}
|
|
7181
|
+
}
|
|
7182
|
+
}
|
|
7183
|
+
if (this.enableStats) {
|
|
7184
|
+
this.stats.clears++;
|
|
7185
|
+
}
|
|
7186
|
+
if (this.enableJournal) {
|
|
7187
|
+
await this._journalOperation("clear", prefix || "all", { count: cacheFiles.length });
|
|
7188
|
+
}
|
|
7189
|
+
return true;
|
|
7190
|
+
} catch (error) {
|
|
7191
|
+
if (this.enableStats) {
|
|
7192
|
+
this.stats.errors++;
|
|
7193
|
+
}
|
|
7194
|
+
throw new Error(`Failed to clear cache: ${error.message}`);
|
|
7195
|
+
}
|
|
7196
|
+
}
|
|
7197
|
+
async size() {
|
|
7198
|
+
const keys = await this.keys();
|
|
7199
|
+
return keys.length;
|
|
7200
|
+
}
|
|
7201
|
+
async keys() {
|
|
7202
|
+
try {
|
|
7203
|
+
const files = await readdir$1(this.directory);
|
|
7204
|
+
const cacheFiles = files.filter(
|
|
7205
|
+
(file) => file.startsWith(this.prefix) && file.endsWith(this.fileExtension)
|
|
7206
|
+
);
|
|
7207
|
+
const keys = cacheFiles.map((file) => {
|
|
7208
|
+
const keyPart = file.slice(this.prefix.length + 1, -this.fileExtension.length);
|
|
7209
|
+
return keyPart;
|
|
7210
|
+
});
|
|
7211
|
+
return keys;
|
|
7212
|
+
} catch (error) {
|
|
7213
|
+
console.warn("FilesystemCache: Failed to list keys:", error.message);
|
|
7214
|
+
return [];
|
|
7215
|
+
}
|
|
7216
|
+
}
|
|
7217
|
+
// Helper methods
|
|
7218
|
+
async _fileExists(filePath) {
|
|
7219
|
+
const [ok] = await try_fn_default(async () => {
|
|
7220
|
+
await stat$1(filePath);
|
|
7221
|
+
});
|
|
7222
|
+
return ok;
|
|
7223
|
+
}
|
|
7224
|
+
async _copyFile(src, dest) {
|
|
7225
|
+
const [ok, err] = await try_fn_default(async () => {
|
|
7226
|
+
const content = await readFile$1(src);
|
|
7227
|
+
await writeFile$1(dest, content);
|
|
7228
|
+
});
|
|
7229
|
+
if (!ok) {
|
|
7230
|
+
console.warn("FilesystemCache: Failed to create backup:", err.message);
|
|
7231
|
+
}
|
|
7232
|
+
}
|
|
7233
|
+
async _cleanup() {
|
|
7234
|
+
if (!this.ttl || this.ttl <= 0) return;
|
|
7235
|
+
try {
|
|
7236
|
+
const files = await readdir$1(this.directory);
|
|
7237
|
+
const now = Date.now();
|
|
7238
|
+
for (const file of files) {
|
|
7239
|
+
if (!file.startsWith(this.prefix) || !file.endsWith(this.fileExtension)) {
|
|
7240
|
+
continue;
|
|
7241
|
+
}
|
|
7242
|
+
const filePath = path.join(this.directory, file);
|
|
7243
|
+
let shouldDelete = false;
|
|
7244
|
+
if (this.enableMetadata) {
|
|
7245
|
+
const metadataPath = this._getMetadataPath(filePath);
|
|
7246
|
+
if (await this._fileExists(metadataPath)) {
|
|
7247
|
+
const [ok, err, metadata] = await try_fn_default(async () => {
|
|
7248
|
+
const metaContent = await readFile$1(metadataPath, this.encoding);
|
|
7249
|
+
return JSON.parse(metaContent);
|
|
7250
|
+
});
|
|
7251
|
+
if (ok && metadata.ttl > 0) {
|
|
7252
|
+
const age = now - metadata.timestamp;
|
|
7253
|
+
shouldDelete = age > metadata.ttl;
|
|
7254
|
+
}
|
|
7255
|
+
}
|
|
7256
|
+
} else {
|
|
7257
|
+
const [ok, err, stats] = await try_fn_default(async () => {
|
|
7258
|
+
return await stat$1(filePath);
|
|
7259
|
+
});
|
|
7260
|
+
if (ok) {
|
|
7261
|
+
const age = now - stats.mtime.getTime();
|
|
7262
|
+
shouldDelete = age > this.ttl;
|
|
7263
|
+
}
|
|
7264
|
+
}
|
|
7265
|
+
if (shouldDelete) {
|
|
7266
|
+
const keyPart = file.slice(this.prefix.length + 1, -this.fileExtension.length);
|
|
7267
|
+
await this._del(keyPart);
|
|
7268
|
+
}
|
|
7269
|
+
}
|
|
7270
|
+
} catch (error) {
|
|
7271
|
+
console.warn("FilesystemCache cleanup error:", error.message);
|
|
7272
|
+
}
|
|
7273
|
+
}
|
|
7274
|
+
async _acquireLock(filePath) {
|
|
7275
|
+
if (!this.enableLocking) return;
|
|
7276
|
+
const lockKey = filePath;
|
|
7277
|
+
const startTime = Date.now();
|
|
7278
|
+
while (this.locks.has(lockKey)) {
|
|
7279
|
+
if (Date.now() - startTime > this.lockTimeout) {
|
|
7280
|
+
throw new Error(`Lock timeout for file: ${filePath}`);
|
|
7281
|
+
}
|
|
7282
|
+
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
7283
|
+
}
|
|
7284
|
+
this.locks.set(lockKey, Date.now());
|
|
7285
|
+
}
|
|
7286
|
+
_releaseLock(filePath) {
|
|
7287
|
+
if (!this.enableLocking) return;
|
|
7288
|
+
this.locks.delete(filePath);
|
|
7289
|
+
}
|
|
7290
|
+
async _journalOperation(operation, key, metadata = {}) {
|
|
7291
|
+
if (!this.enableJournal) return;
|
|
7292
|
+
const entry = {
|
|
7293
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
7294
|
+
operation,
|
|
7295
|
+
key,
|
|
7296
|
+
metadata
|
|
7297
|
+
};
|
|
7298
|
+
const [ok, err] = await try_fn_default(async () => {
|
|
7299
|
+
const line = JSON.stringify(entry) + "\n";
|
|
7300
|
+
await fs.promises.appendFile(this.journalFile, line, this.encoding);
|
|
7301
|
+
});
|
|
7302
|
+
if (!ok) {
|
|
7303
|
+
console.warn("FilesystemCache journal error:", err.message);
|
|
7304
|
+
}
|
|
7305
|
+
}
|
|
7306
|
+
// Cleanup on process exit
|
|
7307
|
+
destroy() {
|
|
7308
|
+
if (this.cleanupTimer) {
|
|
7309
|
+
clearInterval(this.cleanupTimer);
|
|
7310
|
+
this.cleanupTimer = null;
|
|
7311
|
+
}
|
|
7312
|
+
}
|
|
7313
|
+
// Get cache statistics
|
|
7314
|
+
getStats() {
|
|
7315
|
+
return {
|
|
7316
|
+
...this.stats,
|
|
7317
|
+
directory: this.directory,
|
|
7318
|
+
ttl: this.ttl,
|
|
7319
|
+
compression: this.enableCompression,
|
|
7320
|
+
metadata: this.enableMetadata,
|
|
7321
|
+
cleanup: this.enableCleanup,
|
|
7322
|
+
locking: this.enableLocking,
|
|
7323
|
+
journal: this.enableJournal
|
|
7324
|
+
};
|
|
7325
|
+
}
|
|
7326
|
+
}
|
|
7327
|
+
|
|
7328
|
+
promisify(fs.mkdir);
|
|
7329
|
+
const rmdir = promisify(fs.rmdir);
|
|
7330
|
+
const readdir = promisify(fs.readdir);
|
|
7331
|
+
const stat = promisify(fs.stat);
|
|
7332
|
+
const writeFile = promisify(fs.writeFile);
|
|
7333
|
+
const readFile = promisify(fs.readFile);
|
|
7334
|
+
class PartitionAwareFilesystemCache extends FilesystemCache {
|
|
7335
|
+
constructor({
|
|
7336
|
+
partitionStrategy = "hierarchical",
|
|
7337
|
+
// 'hierarchical', 'flat', 'temporal'
|
|
7338
|
+
trackUsage = true,
|
|
7339
|
+
preloadRelated = false,
|
|
7340
|
+
preloadThreshold = 10,
|
|
7341
|
+
maxCacheSize = null,
|
|
7342
|
+
usageStatsFile = "partition-usage.json",
|
|
7343
|
+
...config
|
|
7344
|
+
}) {
|
|
7345
|
+
super(config);
|
|
7346
|
+
this.partitionStrategy = partitionStrategy;
|
|
7347
|
+
this.trackUsage = trackUsage;
|
|
7348
|
+
this.preloadRelated = preloadRelated;
|
|
7349
|
+
this.preloadThreshold = preloadThreshold;
|
|
7350
|
+
this.maxCacheSize = maxCacheSize;
|
|
7351
|
+
this.usageStatsFile = path.join(this.directory, usageStatsFile);
|
|
7352
|
+
this.partitionUsage = /* @__PURE__ */ new Map();
|
|
7353
|
+
this.loadUsageStats();
|
|
7354
|
+
}
|
|
7355
|
+
/**
|
|
7356
|
+
* Generate partition-aware cache key
|
|
7357
|
+
*/
|
|
7358
|
+
_getPartitionCacheKey(resource, action, partition, partitionValues = {}, params = {}) {
|
|
7359
|
+
const keyParts = [`resource=${resource}`, `action=${action}`];
|
|
7360
|
+
if (partition && Object.keys(partitionValues).length > 0) {
|
|
7361
|
+
keyParts.push(`partition=${partition}`);
|
|
7362
|
+
const sortedFields = Object.entries(partitionValues).sort(([a], [b]) => a.localeCompare(b));
|
|
7363
|
+
for (const [field, value] of sortedFields) {
|
|
7364
|
+
if (value !== null && value !== void 0) {
|
|
7365
|
+
keyParts.push(`${field}=${value}`);
|
|
7366
|
+
}
|
|
7367
|
+
}
|
|
7368
|
+
}
|
|
7369
|
+
if (Object.keys(params).length > 0) {
|
|
7370
|
+
const paramsStr = Object.entries(params).sort(([a], [b]) => a.localeCompare(b)).map(([k, v]) => `${k}=${v}`).join("|");
|
|
7371
|
+
keyParts.push(`params=${Buffer.from(paramsStr).toString("base64")}`);
|
|
7372
|
+
}
|
|
7373
|
+
return keyParts.join("/") + this.fileExtension;
|
|
7374
|
+
}
|
|
7375
|
+
/**
|
|
7376
|
+
* Get directory path for partition cache
|
|
7377
|
+
*/
|
|
7378
|
+
_getPartitionDirectory(resource, partition, partitionValues = {}) {
|
|
7379
|
+
const basePath = path.join(this.directory, `resource=${resource}`);
|
|
7380
|
+
if (!partition) {
|
|
7381
|
+
return basePath;
|
|
7382
|
+
}
|
|
7383
|
+
if (this.partitionStrategy === "flat") {
|
|
7384
|
+
return path.join(basePath, "partitions");
|
|
7385
|
+
}
|
|
7386
|
+
if (this.partitionStrategy === "temporal" && this._isTemporalPartition(partition, partitionValues)) {
|
|
7387
|
+
return this._getTemporalDirectory(basePath, partition, partitionValues);
|
|
7388
|
+
}
|
|
7389
|
+
const pathParts = [basePath, `partition=${partition}`];
|
|
7390
|
+
const sortedFields = Object.entries(partitionValues).sort(([a], [b]) => a.localeCompare(b));
|
|
7391
|
+
for (const [field, value] of sortedFields) {
|
|
7392
|
+
if (value !== null && value !== void 0) {
|
|
7393
|
+
pathParts.push(`${field}=${this._sanitizePathValue(value)}`);
|
|
7394
|
+
}
|
|
7395
|
+
}
|
|
7396
|
+
return path.join(...pathParts);
|
|
7397
|
+
}
|
|
7398
|
+
/**
|
|
7399
|
+
* Enhanced set method with partition awareness
|
|
7400
|
+
*/
|
|
7401
|
+
async _set(key, data, options = {}) {
|
|
7402
|
+
const { resource, action, partition, partitionValues, params } = options;
|
|
7403
|
+
if (resource && partition) {
|
|
7404
|
+
const partitionKey = this._getPartitionCacheKey(resource, action, partition, partitionValues, params);
|
|
7405
|
+
const partitionDir = this._getPartitionDirectory(resource, partition, partitionValues);
|
|
7406
|
+
await this._ensureDirectory(partitionDir);
|
|
7407
|
+
const filePath = path.join(partitionDir, this._sanitizeFileName(partitionKey));
|
|
7408
|
+
if (this.trackUsage) {
|
|
7409
|
+
await this._trackPartitionUsage(resource, partition, partitionValues);
|
|
7410
|
+
}
|
|
7411
|
+
const partitionData = {
|
|
7412
|
+
data,
|
|
7413
|
+
metadata: {
|
|
7414
|
+
resource,
|
|
7415
|
+
partition,
|
|
7416
|
+
partitionValues,
|
|
7417
|
+
timestamp: Date.now(),
|
|
7418
|
+
ttl: this.ttl
|
|
7419
|
+
}
|
|
7420
|
+
};
|
|
7421
|
+
return this._writeFileWithMetadata(filePath, partitionData);
|
|
7422
|
+
}
|
|
7423
|
+
return super._set(key, data);
|
|
7424
|
+
}
|
|
7425
|
+
/**
|
|
7426
|
+
* Enhanced get method with partition awareness
|
|
7427
|
+
*/
|
|
7428
|
+
async _get(key, options = {}) {
|
|
7429
|
+
const { resource, action, partition, partitionValues, params } = options;
|
|
7430
|
+
if (resource && partition) {
|
|
7431
|
+
const partitionKey = this._getPartitionCacheKey(resource, action, partition, partitionValues, params);
|
|
7432
|
+
const partitionDir = this._getPartitionDirectory(resource, partition, partitionValues);
|
|
7433
|
+
const filePath = path.join(partitionDir, this._sanitizeFileName(partitionKey));
|
|
7434
|
+
if (!await this._fileExists(filePath)) {
|
|
7435
|
+
if (this.preloadRelated) {
|
|
7436
|
+
await this._preloadRelatedPartitions(resource, partition, partitionValues);
|
|
7437
|
+
}
|
|
7438
|
+
return null;
|
|
7439
|
+
}
|
|
7440
|
+
const result = await this._readFileWithMetadata(filePath);
|
|
7441
|
+
if (result && this.trackUsage) {
|
|
7442
|
+
await this._trackPartitionUsage(resource, partition, partitionValues);
|
|
7443
|
+
}
|
|
7444
|
+
return result?.data || null;
|
|
7445
|
+
}
|
|
7446
|
+
return super._get(key);
|
|
7447
|
+
}
|
|
7448
|
+
/**
|
|
7449
|
+
* Clear cache for specific partition
|
|
7450
|
+
*/
|
|
7451
|
+
async clearPartition(resource, partition, partitionValues = {}) {
|
|
7452
|
+
const partitionDir = this._getPartitionDirectory(resource, partition, partitionValues);
|
|
7453
|
+
const [ok, err] = await try_fn_default(async () => {
|
|
7454
|
+
if (await this._fileExists(partitionDir)) {
|
|
7455
|
+
await rmdir(partitionDir, { recursive: true });
|
|
7456
|
+
}
|
|
7457
|
+
});
|
|
7458
|
+
if (!ok) {
|
|
7459
|
+
console.warn(`Failed to clear partition cache: ${err.message}`);
|
|
7460
|
+
}
|
|
7461
|
+
const usageKey = this._getUsageKey(resource, partition, partitionValues);
|
|
7462
|
+
this.partitionUsage.delete(usageKey);
|
|
7463
|
+
await this._saveUsageStats();
|
|
7464
|
+
return ok;
|
|
7465
|
+
}
|
|
7466
|
+
/**
|
|
7467
|
+
* Clear all partitions for a resource
|
|
7468
|
+
*/
|
|
7469
|
+
async clearResourcePartitions(resource) {
|
|
7470
|
+
const resourceDir = path.join(this.directory, `resource=${resource}`);
|
|
7471
|
+
const [ok, err] = await try_fn_default(async () => {
|
|
7472
|
+
if (await this._fileExists(resourceDir)) {
|
|
7473
|
+
await rmdir(resourceDir, { recursive: true });
|
|
7474
|
+
}
|
|
7475
|
+
});
|
|
7476
|
+
for (const [key] of this.partitionUsage.entries()) {
|
|
7477
|
+
if (key.startsWith(`${resource}/`)) {
|
|
7478
|
+
this.partitionUsage.delete(key);
|
|
7479
|
+
}
|
|
7480
|
+
}
|
|
7481
|
+
await this._saveUsageStats();
|
|
7482
|
+
return ok;
|
|
7483
|
+
}
|
|
7484
|
+
/**
|
|
7485
|
+
* Get partition cache statistics
|
|
7486
|
+
*/
|
|
7487
|
+
async getPartitionStats(resource, partition = null) {
|
|
7488
|
+
const stats = {
|
|
7489
|
+
totalFiles: 0,
|
|
7490
|
+
totalSize: 0,
|
|
7491
|
+
partitions: {},
|
|
7492
|
+
usage: {}
|
|
7493
|
+
};
|
|
7494
|
+
const resourceDir = path.join(this.directory, `resource=${resource}`);
|
|
7495
|
+
if (!await this._fileExists(resourceDir)) {
|
|
7496
|
+
return stats;
|
|
7497
|
+
}
|
|
7498
|
+
await this._calculateDirectoryStats(resourceDir, stats);
|
|
7499
|
+
for (const [key, usage] of this.partitionUsage.entries()) {
|
|
7500
|
+
if (key.startsWith(`${resource}/`)) {
|
|
7501
|
+
const partitionName = key.split("/")[1];
|
|
7502
|
+
if (!partition || partitionName === partition) {
|
|
7503
|
+
stats.usage[partitionName] = usage;
|
|
7504
|
+
}
|
|
7505
|
+
}
|
|
7506
|
+
}
|
|
7507
|
+
return stats;
|
|
7508
|
+
}
|
|
7509
|
+
/**
|
|
7510
|
+
* Get cache recommendations based on usage patterns
|
|
7511
|
+
*/
|
|
7512
|
+
async getCacheRecommendations(resource) {
|
|
7513
|
+
const recommendations = [];
|
|
7514
|
+
const now = Date.now();
|
|
7515
|
+
const dayMs = 24 * 60 * 60 * 1e3;
|
|
7516
|
+
for (const [key, usage] of this.partitionUsage.entries()) {
|
|
7517
|
+
if (key.startsWith(`${resource}/`)) {
|
|
7518
|
+
const [, partition] = key.split("/");
|
|
7519
|
+
const daysSinceLastAccess = (now - usage.lastAccess) / dayMs;
|
|
7520
|
+
const accessesPerDay = usage.count / Math.max(1, daysSinceLastAccess);
|
|
7521
|
+
let recommendation = "keep";
|
|
7522
|
+
let priority = usage.count;
|
|
7523
|
+
if (daysSinceLastAccess > 30) {
|
|
7524
|
+
recommendation = "archive";
|
|
7525
|
+
priority = 0;
|
|
7526
|
+
} else if (accessesPerDay < 0.1) {
|
|
7527
|
+
recommendation = "reduce_ttl";
|
|
7528
|
+
priority = 1;
|
|
7529
|
+
} else if (accessesPerDay > 10) {
|
|
7530
|
+
recommendation = "preload";
|
|
7531
|
+
priority = 100;
|
|
7532
|
+
}
|
|
7533
|
+
recommendations.push({
|
|
7534
|
+
partition,
|
|
7535
|
+
recommendation,
|
|
7536
|
+
priority,
|
|
7537
|
+
usage: accessesPerDay,
|
|
7538
|
+
lastAccess: new Date(usage.lastAccess).toISOString()
|
|
7539
|
+
});
|
|
7540
|
+
}
|
|
7541
|
+
}
|
|
7542
|
+
return recommendations.sort((a, b) => b.priority - a.priority);
|
|
7543
|
+
}
|
|
7544
|
+
/**
|
|
7545
|
+
* Preload frequently accessed partitions
|
|
7546
|
+
*/
|
|
7547
|
+
async warmPartitionCache(resource, options = {}) {
|
|
7548
|
+
const { partitions = [], maxFiles = 1e3 } = options;
|
|
7549
|
+
let warmedCount = 0;
|
|
7550
|
+
for (const partition of partitions) {
|
|
7551
|
+
const usageKey = `${resource}/${partition}`;
|
|
7552
|
+
const usage = this.partitionUsage.get(usageKey);
|
|
7553
|
+
if (usage && usage.count >= this.preloadThreshold) {
|
|
7554
|
+
console.log(`\u{1F525} Warming cache for ${resource}/${partition} (${usage.count} accesses)`);
|
|
7555
|
+
warmedCount++;
|
|
7556
|
+
}
|
|
7557
|
+
if (warmedCount >= maxFiles) break;
|
|
7558
|
+
}
|
|
7559
|
+
return warmedCount;
|
|
7560
|
+
}
|
|
7561
|
+
// Private helper methods
|
|
7562
|
+
async _trackPartitionUsage(resource, partition, partitionValues) {
|
|
7563
|
+
const usageKey = this._getUsageKey(resource, partition, partitionValues);
|
|
7564
|
+
const current = this.partitionUsage.get(usageKey) || {
|
|
7565
|
+
count: 0,
|
|
7566
|
+
firstAccess: Date.now(),
|
|
7567
|
+
lastAccess: Date.now()
|
|
7568
|
+
};
|
|
7569
|
+
current.count++;
|
|
7570
|
+
current.lastAccess = Date.now();
|
|
7571
|
+
this.partitionUsage.set(usageKey, current);
|
|
7572
|
+
if (current.count % 10 === 0) {
|
|
7573
|
+
await this._saveUsageStats();
|
|
7574
|
+
}
|
|
7575
|
+
}
|
|
7576
|
+
_getUsageKey(resource, partition, partitionValues) {
|
|
7577
|
+
const valuePart = Object.entries(partitionValues).sort(([a], [b]) => a.localeCompare(b)).map(([k, v]) => `${k}=${v}`).join("|");
|
|
7578
|
+
return `${resource}/${partition}/${valuePart}`;
|
|
7579
|
+
}
|
|
7580
|
+
async _preloadRelatedPartitions(resource, partition, partitionValues) {
|
|
7581
|
+
console.log(`\u{1F3AF} Preloading related partitions for ${resource}/${partition}`);
|
|
7582
|
+
if (partitionValues.timestamp || partitionValues.date) ;
|
|
7583
|
+
}
|
|
7584
|
+
_isTemporalPartition(partition, partitionValues) {
|
|
7585
|
+
const temporalFields = ["date", "timestamp", "createdAt", "updatedAt"];
|
|
7586
|
+
return Object.keys(partitionValues).some(
|
|
7587
|
+
(field) => temporalFields.some((tf) => field.toLowerCase().includes(tf))
|
|
7588
|
+
);
|
|
7589
|
+
}
|
|
7590
|
+
_getTemporalDirectory(basePath, partition, partitionValues) {
|
|
7591
|
+
const dateValue = Object.values(partitionValues)[0];
|
|
7592
|
+
if (typeof dateValue === "string" && dateValue.match(/^\d{4}-\d{2}-\d{2}/)) {
|
|
7593
|
+
const [year, month, day] = dateValue.split("-");
|
|
7594
|
+
return path.join(basePath, "temporal", year, month, day);
|
|
7595
|
+
}
|
|
7596
|
+
return path.join(basePath, `partition=${partition}`);
|
|
7597
|
+
}
|
|
7598
|
+
_sanitizePathValue(value) {
|
|
7599
|
+
return String(value).replace(/[<>:"/\\|?*]/g, "_");
|
|
7600
|
+
}
|
|
7601
|
+
_sanitizeFileName(filename) {
|
|
7602
|
+
return filename.replace(/[<>:"/\\|?*]/g, "_");
|
|
7603
|
+
}
|
|
7604
|
+
async _calculateDirectoryStats(dir, stats) {
|
|
7605
|
+
const [ok, err, files] = await try_fn_default(() => readdir(dir));
|
|
7606
|
+
if (!ok) return;
|
|
7607
|
+
for (const file of files) {
|
|
7608
|
+
const filePath = path.join(dir, file);
|
|
7609
|
+
const [statOk, statErr, fileStat] = await try_fn_default(() => stat(filePath));
|
|
7610
|
+
if (statOk) {
|
|
7611
|
+
if (fileStat.isDirectory()) {
|
|
7612
|
+
await this._calculateDirectoryStats(filePath, stats);
|
|
7613
|
+
} else {
|
|
7614
|
+
stats.totalFiles++;
|
|
7615
|
+
stats.totalSize += fileStat.size;
|
|
7616
|
+
}
|
|
7617
|
+
}
|
|
7618
|
+
}
|
|
7619
|
+
}
|
|
7620
|
+
async loadUsageStats() {
|
|
7621
|
+
const [ok, err, content] = await try_fn_default(async () => {
|
|
7622
|
+
const data = await readFile(this.usageStatsFile, "utf8");
|
|
7623
|
+
return JSON.parse(data);
|
|
7624
|
+
});
|
|
7625
|
+
if (ok && content) {
|
|
7626
|
+
this.partitionUsage = new Map(Object.entries(content));
|
|
7627
|
+
}
|
|
7628
|
+
}
|
|
7629
|
+
async _saveUsageStats() {
|
|
7630
|
+
const statsObject = Object.fromEntries(this.partitionUsage);
|
|
7631
|
+
await try_fn_default(async () => {
|
|
7632
|
+
await writeFile(
|
|
7633
|
+
this.usageStatsFile,
|
|
7634
|
+
JSON.stringify(statsObject, null, 2),
|
|
7635
|
+
"utf8"
|
|
7636
|
+
);
|
|
7637
|
+
});
|
|
7638
|
+
}
|
|
7639
|
+
async _writeFileWithMetadata(filePath, data) {
|
|
7640
|
+
const content = JSON.stringify(data);
|
|
7641
|
+
const [ok, err] = await try_fn_default(async () => {
|
|
7642
|
+
await writeFile(filePath, content, {
|
|
7643
|
+
encoding: this.encoding,
|
|
7644
|
+
mode: this.fileMode
|
|
7645
|
+
});
|
|
7646
|
+
});
|
|
7647
|
+
if (!ok) {
|
|
7648
|
+
throw new Error(`Failed to write cache file: ${err.message}`);
|
|
7649
|
+
}
|
|
7650
|
+
return true;
|
|
7651
|
+
}
|
|
7652
|
+
async _readFileWithMetadata(filePath) {
|
|
7653
|
+
const [ok, err, content] = await try_fn_default(async () => {
|
|
7654
|
+
return await readFile(filePath, this.encoding);
|
|
7655
|
+
});
|
|
7656
|
+
if (!ok || !content) return null;
|
|
7657
|
+
try {
|
|
7658
|
+
return JSON.parse(content);
|
|
7659
|
+
} catch (error) {
|
|
7660
|
+
return { data: content };
|
|
7661
|
+
}
|
|
7662
|
+
}
|
|
7663
|
+
}
|
|
7664
|
+
|
|
6811
7665
|
class CachePlugin extends plugin_class_default {
|
|
6812
7666
|
constructor(options = {}) {
|
|
6813
7667
|
super(options);
|
|
6814
7668
|
this.driver = options.driver;
|
|
6815
7669
|
this.config = {
|
|
6816
7670
|
includePartitions: options.includePartitions !== false,
|
|
7671
|
+
partitionStrategy: options.partitionStrategy || "hierarchical",
|
|
7672
|
+
partitionAware: options.partitionAware !== false,
|
|
7673
|
+
trackUsage: options.trackUsage !== false,
|
|
7674
|
+
preloadRelated: options.preloadRelated !== false,
|
|
6817
7675
|
...options
|
|
6818
7676
|
};
|
|
6819
7677
|
}
|
|
@@ -6825,6 +7683,17 @@ class CachePlugin extends plugin_class_default {
|
|
|
6825
7683
|
this.driver = this.config.driver;
|
|
6826
7684
|
} else if (this.config.driverType === "memory") {
|
|
6827
7685
|
this.driver = new memory_cache_class_default(this.config.memoryOptions || {});
|
|
7686
|
+
} else if (this.config.driverType === "filesystem") {
|
|
7687
|
+
if (this.config.partitionAware) {
|
|
7688
|
+
this.driver = new PartitionAwareFilesystemCache({
|
|
7689
|
+
partitionStrategy: this.config.partitionStrategy,
|
|
7690
|
+
trackUsage: this.config.trackUsage,
|
|
7691
|
+
preloadRelated: this.config.preloadRelated,
|
|
7692
|
+
...this.config.filesystemOptions
|
|
7693
|
+
});
|
|
7694
|
+
} else {
|
|
7695
|
+
this.driver = new FilesystemCache(this.config.filesystemOptions || {});
|
|
7696
|
+
}
|
|
6828
7697
|
} else {
|
|
6829
7698
|
this.driver = new s3_cache_class_default({ client: this.database.client, ...this.config.s3Options || {} });
|
|
6830
7699
|
}
|
|
@@ -6865,6 +7734,20 @@ class CachePlugin extends plugin_class_default {
|
|
|
6865
7734
|
const { action, params = {}, partition, partitionValues } = options;
|
|
6866
7735
|
return this.generateCacheKey(resource, action, params, partition, partitionValues);
|
|
6867
7736
|
};
|
|
7737
|
+
if (this.driver instanceof PartitionAwareFilesystemCache) {
|
|
7738
|
+
resource.clearPartitionCache = async (partition, partitionValues = {}) => {
|
|
7739
|
+
return await this.driver.clearPartition(resource.name, partition, partitionValues);
|
|
7740
|
+
};
|
|
7741
|
+
resource.getPartitionCacheStats = async (partition = null) => {
|
|
7742
|
+
return await this.driver.getPartitionStats(resource.name, partition);
|
|
7743
|
+
};
|
|
7744
|
+
resource.getCacheRecommendations = async () => {
|
|
7745
|
+
return await this.driver.getCacheRecommendations(resource.name);
|
|
7746
|
+
};
|
|
7747
|
+
resource.warmPartitionCache = async (partitions = [], options = {}) => {
|
|
7748
|
+
return await this.driver.warmPartitionCache(resource.name, { partitions, ...options });
|
|
7749
|
+
};
|
|
7750
|
+
}
|
|
6868
7751
|
const cacheMethods = [
|
|
6869
7752
|
"count",
|
|
6870
7753
|
"listIds",
|
|
@@ -6890,12 +7773,37 @@ class CachePlugin extends plugin_class_default {
|
|
|
6890
7773
|
} else if (method === "get") {
|
|
6891
7774
|
key = await resource.cacheKeyFor({ action: method, params: { id: ctx.args[0] } });
|
|
6892
7775
|
}
|
|
6893
|
-
|
|
6894
|
-
|
|
6895
|
-
|
|
6896
|
-
|
|
6897
|
-
|
|
6898
|
-
|
|
7776
|
+
if (this.driver instanceof PartitionAwareFilesystemCache) {
|
|
7777
|
+
let partition, partitionValues;
|
|
7778
|
+
if (method === "list" || method === "listIds" || method === "count" || method === "page") {
|
|
7779
|
+
const args = ctx.args[0] || {};
|
|
7780
|
+
partition = args.partition;
|
|
7781
|
+
partitionValues = args.partitionValues;
|
|
7782
|
+
}
|
|
7783
|
+
const [ok, err, result] = await try_fn_default(() => resource.cache._get(key, {
|
|
7784
|
+
resource: resource.name,
|
|
7785
|
+
action: method,
|
|
7786
|
+
partition,
|
|
7787
|
+
partitionValues
|
|
7788
|
+
}));
|
|
7789
|
+
if (ok && result !== null && result !== void 0) return result;
|
|
7790
|
+
if (!ok && err.name !== "NoSuchKey") throw err;
|
|
7791
|
+
const freshResult = await next();
|
|
7792
|
+
await resource.cache._set(key, freshResult, {
|
|
7793
|
+
resource: resource.name,
|
|
7794
|
+
action: method,
|
|
7795
|
+
partition,
|
|
7796
|
+
partitionValues
|
|
7797
|
+
});
|
|
7798
|
+
return freshResult;
|
|
7799
|
+
} else {
|
|
7800
|
+
const [ok, err, result] = await try_fn_default(() => resource.cache.get(key));
|
|
7801
|
+
if (ok && result !== null && result !== void 0) return result;
|
|
7802
|
+
if (!ok && err.name !== "NoSuchKey") throw err;
|
|
7803
|
+
const freshResult = await next();
|
|
7804
|
+
await resource.cache.set(key, freshResult);
|
|
7805
|
+
return freshResult;
|
|
7806
|
+
}
|
|
6899
7807
|
});
|
|
6900
7808
|
}
|
|
6901
7809
|
const writeMethods = ["insert", "update", "delete", "deleteMany"];
|
|
@@ -6988,6 +7896,10 @@ class CachePlugin extends plugin_class_default {
|
|
|
6988
7896
|
throw new Error(`Resource '${resourceName}' not found`);
|
|
6989
7897
|
}
|
|
6990
7898
|
const { includePartitions = true } = options;
|
|
7899
|
+
if (this.driver instanceof PartitionAwareFilesystemCache && resource.warmPartitionCache) {
|
|
7900
|
+
const partitionNames = resource.config.partitions ? Object.keys(resource.config.partitions) : [];
|
|
7901
|
+
return await resource.warmPartitionCache(partitionNames, options);
|
|
7902
|
+
}
|
|
6991
7903
|
await resource.getAll();
|
|
6992
7904
|
if (includePartitions && resource.config.partitions) {
|
|
6993
7905
|
for (const [partitionName, partitionDef] of Object.entries(resource.config.partitions)) {
|
|
@@ -7009,6 +7921,57 @@ class CachePlugin extends plugin_class_default {
|
|
|
7009
7921
|
}
|
|
7010
7922
|
}
|
|
7011
7923
|
}
|
|
7924
|
+
// Partition-specific methods
|
|
7925
|
+
async getPartitionCacheStats(resourceName, partition = null) {
|
|
7926
|
+
if (!(this.driver instanceof PartitionAwareFilesystemCache)) {
|
|
7927
|
+
throw new Error("Partition cache statistics are only available with PartitionAwareFilesystemCache");
|
|
7928
|
+
}
|
|
7929
|
+
return await this.driver.getPartitionStats(resourceName, partition);
|
|
7930
|
+
}
|
|
7931
|
+
async getCacheRecommendations(resourceName) {
|
|
7932
|
+
if (!(this.driver instanceof PartitionAwareFilesystemCache)) {
|
|
7933
|
+
throw new Error("Cache recommendations are only available with PartitionAwareFilesystemCache");
|
|
7934
|
+
}
|
|
7935
|
+
return await this.driver.getCacheRecommendations(resourceName);
|
|
7936
|
+
}
|
|
7937
|
+
async clearPartitionCache(resourceName, partition, partitionValues = {}) {
|
|
7938
|
+
if (!(this.driver instanceof PartitionAwareFilesystemCache)) {
|
|
7939
|
+
throw new Error("Partition cache clearing is only available with PartitionAwareFilesystemCache");
|
|
7940
|
+
}
|
|
7941
|
+
return await this.driver.clearPartition(resourceName, partition, partitionValues);
|
|
7942
|
+
}
|
|
7943
|
+
async analyzeCacheUsage() {
|
|
7944
|
+
if (!(this.driver instanceof PartitionAwareFilesystemCache)) {
|
|
7945
|
+
return { message: "Cache usage analysis is only available with PartitionAwareFilesystemCache" };
|
|
7946
|
+
}
|
|
7947
|
+
const analysis = {
|
|
7948
|
+
totalResources: Object.keys(this.database.resources).length,
|
|
7949
|
+
resourceStats: {},
|
|
7950
|
+
recommendations: {},
|
|
7951
|
+
summary: {
|
|
7952
|
+
mostUsedPartitions: [],
|
|
7953
|
+
leastUsedPartitions: [],
|
|
7954
|
+
suggestedOptimizations: []
|
|
7955
|
+
}
|
|
7956
|
+
};
|
|
7957
|
+
for (const [resourceName, resource] of Object.entries(this.database.resources)) {
|
|
7958
|
+
try {
|
|
7959
|
+
analysis.resourceStats[resourceName] = await this.driver.getPartitionStats(resourceName);
|
|
7960
|
+
analysis.recommendations[resourceName] = await this.driver.getCacheRecommendations(resourceName);
|
|
7961
|
+
} catch (error) {
|
|
7962
|
+
analysis.resourceStats[resourceName] = { error: error.message };
|
|
7963
|
+
}
|
|
7964
|
+
}
|
|
7965
|
+
const allRecommendations = Object.values(analysis.recommendations).flat();
|
|
7966
|
+
analysis.summary.mostUsedPartitions = allRecommendations.filter((r) => r.recommendation === "preload").sort((a, b) => b.priority - a.priority).slice(0, 5);
|
|
7967
|
+
analysis.summary.leastUsedPartitions = allRecommendations.filter((r) => r.recommendation === "archive").slice(0, 5);
|
|
7968
|
+
analysis.summary.suggestedOptimizations = [
|
|
7969
|
+
`Consider preloading ${analysis.summary.mostUsedPartitions.length} high-usage partitions`,
|
|
7970
|
+
`Archive ${analysis.summary.leastUsedPartitions.length} unused partitions`,
|
|
7971
|
+
`Monitor cache hit rates for partition efficiency`
|
|
7972
|
+
];
|
|
7973
|
+
return analysis;
|
|
7974
|
+
}
|
|
7012
7975
|
}
|
|
7013
7976
|
|
|
7014
7977
|
const CostsPlugin = {
|
|
@@ -7185,24 +8148,29 @@ class FullTextPlugin extends plugin_class_default {
|
|
|
7185
8148
|
resource._deleteMany = resource.deleteMany;
|
|
7186
8149
|
this.wrapResourceMethod(resource, "insert", async (result, args, methodName) => {
|
|
7187
8150
|
const [data] = args;
|
|
7188
|
-
this.indexRecord(resource.name, result.id, data).catch(
|
|
8151
|
+
this.indexRecord(resource.name, result.id, data).catch(() => {
|
|
8152
|
+
});
|
|
7189
8153
|
return result;
|
|
7190
8154
|
});
|
|
7191
8155
|
this.wrapResourceMethod(resource, "update", async (result, args, methodName) => {
|
|
7192
8156
|
const [id, data] = args;
|
|
7193
|
-
this.removeRecordFromIndex(resource.name, id).catch(
|
|
7194
|
-
|
|
8157
|
+
this.removeRecordFromIndex(resource.name, id).catch(() => {
|
|
8158
|
+
});
|
|
8159
|
+
this.indexRecord(resource.name, id, result).catch(() => {
|
|
8160
|
+
});
|
|
7195
8161
|
return result;
|
|
7196
8162
|
});
|
|
7197
8163
|
this.wrapResourceMethod(resource, "delete", async (result, args, methodName) => {
|
|
7198
8164
|
const [id] = args;
|
|
7199
|
-
this.removeRecordFromIndex(resource.name, id).catch(
|
|
8165
|
+
this.removeRecordFromIndex(resource.name, id).catch(() => {
|
|
8166
|
+
});
|
|
7200
8167
|
return result;
|
|
7201
8168
|
});
|
|
7202
8169
|
this.wrapResourceMethod(resource, "deleteMany", async (result, args, methodName) => {
|
|
7203
8170
|
const [ids] = args;
|
|
7204
8171
|
for (const id of ids) {
|
|
7205
|
-
this.removeRecordFromIndex(resource.name, id).catch(
|
|
8172
|
+
this.removeRecordFromIndex(resource.name, id).catch(() => {
|
|
8173
|
+
});
|
|
7206
8174
|
}
|
|
7207
8175
|
return result;
|
|
7208
8176
|
});
|
|
@@ -7692,7 +8660,8 @@ class MetricsPlugin extends plugin_class_default {
|
|
|
7692
8660
|
}
|
|
7693
8661
|
if (this.config.flushInterval > 0) {
|
|
7694
8662
|
this.flushTimer = setInterval(() => {
|
|
7695
|
-
this.flushMetrics().catch(
|
|
8663
|
+
this.flushMetrics().catch(() => {
|
|
8664
|
+
});
|
|
7696
8665
|
}, this.config.flushInterval);
|
|
7697
8666
|
}
|
|
7698
8667
|
}
|
|
@@ -7764,9 +8733,6 @@ class MetricsPlugin extends plugin_class_default {
|
|
|
7764
8733
|
}
|
|
7765
8734
|
this.resetMetrics();
|
|
7766
8735
|
});
|
|
7767
|
-
if (!ok) {
|
|
7768
|
-
console.error("Failed to flush metrics:", err);
|
|
7769
|
-
}
|
|
7770
8736
|
}
|
|
7771
8737
|
resetMetrics() {
|
|
7772
8738
|
for (const operation of Object.keys(this.metrics.operations)) {
|
|
@@ -10603,7 +11569,7 @@ class Resource extends EventEmitter {
|
|
|
10603
11569
|
if (okParse) contentType = "application/json";
|
|
10604
11570
|
}
|
|
10605
11571
|
if (this.behavior === "body-only" && (!body || body === "")) {
|
|
10606
|
-
throw new Error(`[Resource.insert]
|
|
11572
|
+
throw new Error(`[Resource.insert] Attempt to save object without body! Data: id=${finalId}, resource=${this.name}`);
|
|
10607
11573
|
}
|
|
10608
11574
|
const [okPut, errPut, putResult] = await try_fn_default(() => this.client.putObject({
|
|
10609
11575
|
key,
|
|
@@ -11950,8 +12916,8 @@ class Resource extends EventEmitter {
|
|
|
11950
12916
|
return mappedData;
|
|
11951
12917
|
}
|
|
11952
12918
|
/**
|
|
11953
|
-
* Compose the full object (metadata + body) as
|
|
11954
|
-
*
|
|
12919
|
+
* Compose the full object (metadata + body) as returned by .get(),
|
|
12920
|
+
* using in-memory data after insert/update, according to behavior
|
|
11955
12921
|
*/
|
|
11956
12922
|
async composeFullObjectFromWrite({ id, metadata, body, behavior }) {
|
|
11957
12923
|
const behaviorFlags = {};
|
|
@@ -12106,7 +13072,7 @@ class Resource extends EventEmitter {
|
|
|
12106
13072
|
if (!this._middlewares.has(method)) throw new ResourceError(`No such method for middleware: ${method}`, { operation: "useMiddleware", method });
|
|
12107
13073
|
this._middlewares.get(method).push(fn);
|
|
12108
13074
|
}
|
|
12109
|
-
//
|
|
13075
|
+
// Utility to apply schema default values
|
|
12110
13076
|
applyDefaults(data) {
|
|
12111
13077
|
const out = { ...data };
|
|
12112
13078
|
for (const [key, def] of Object.entries(this.attributes)) {
|
|
@@ -12238,7 +13204,7 @@ class Database extends EventEmitter {
|
|
|
12238
13204
|
super();
|
|
12239
13205
|
this.version = "1";
|
|
12240
13206
|
this.s3dbVersion = (() => {
|
|
12241
|
-
const [ok, err, version] = try_fn_default(() => true ? "7.2
|
|
13207
|
+
const [ok, err, version] = try_fn_default(() => true ? "7.3.2" : "latest");
|
|
12242
13208
|
return ok ? version : "latest";
|
|
12243
13209
|
})();
|
|
12244
13210
|
this.resources = {};
|
|
@@ -12311,7 +13277,7 @@ class Database extends EventEmitter {
|
|
|
12311
13277
|
name,
|
|
12312
13278
|
client: this.client,
|
|
12313
13279
|
database: this,
|
|
12314
|
-
//
|
|
13280
|
+
// ensure reference
|
|
12315
13281
|
version: currentVersion,
|
|
12316
13282
|
attributes: versionData.attributes,
|
|
12317
13283
|
behavior: versionData.behavior || "user-managed",
|
|
@@ -12812,21 +13778,33 @@ class S3dbReplicator extends base_replicator_class_default {
|
|
|
12812
13778
|
throw err;
|
|
12813
13779
|
}
|
|
12814
13780
|
}
|
|
12815
|
-
//
|
|
12816
|
-
async replicate(
|
|
13781
|
+
// Support both object and parameter signatures for flexibility
|
|
13782
|
+
async replicate(resourceOrObj, operation, data, recordId, beforeData) {
|
|
13783
|
+
let resource, op, payload, id;
|
|
13784
|
+
if (typeof resourceOrObj === "object" && resourceOrObj.resource) {
|
|
13785
|
+
resource = resourceOrObj.resource;
|
|
13786
|
+
op = resourceOrObj.operation;
|
|
13787
|
+
payload = resourceOrObj.data;
|
|
13788
|
+
id = resourceOrObj.id;
|
|
13789
|
+
} else {
|
|
13790
|
+
resource = resourceOrObj;
|
|
13791
|
+
op = operation;
|
|
13792
|
+
payload = data;
|
|
13793
|
+
id = recordId;
|
|
13794
|
+
}
|
|
12817
13795
|
const normResource = normalizeResourceName$1(resource);
|
|
12818
|
-
const destResource = this._resolveDestResource(normResource,
|
|
13796
|
+
const destResource = this._resolveDestResource(normResource, payload);
|
|
12819
13797
|
const destResourceObj = this._getDestResourceObj(destResource);
|
|
12820
|
-
const transformedData = this._applyTransformer(normResource,
|
|
13798
|
+
const transformedData = this._applyTransformer(normResource, payload);
|
|
12821
13799
|
let result;
|
|
12822
|
-
if (
|
|
13800
|
+
if (op === "insert") {
|
|
12823
13801
|
result = await destResourceObj.insert(transformedData);
|
|
12824
|
-
} else if (
|
|
12825
|
-
result = await destResourceObj.update(
|
|
12826
|
-
} else if (
|
|
12827
|
-
result = await destResourceObj.delete(
|
|
13802
|
+
} else if (op === "update") {
|
|
13803
|
+
result = await destResourceObj.update(id, transformedData);
|
|
13804
|
+
} else if (op === "delete") {
|
|
13805
|
+
result = await destResourceObj.delete(id);
|
|
12828
13806
|
} else {
|
|
12829
|
-
throw new Error(`Invalid operation: ${
|
|
13807
|
+
throw new Error(`Invalid operation: ${op}. Supported operations are: insert, update, delete`);
|
|
12830
13808
|
}
|
|
12831
13809
|
return result;
|
|
12832
13810
|
}
|
|
@@ -12859,7 +13837,7 @@ class S3dbReplicator extends base_replicator_class_default {
|
|
|
12859
13837
|
if (typeof entry[0] === "function") return resource;
|
|
12860
13838
|
}
|
|
12861
13839
|
if (typeof entry === "string") return entry;
|
|
12862
|
-
if (
|
|
13840
|
+
if (resource && !targetResourceName) targetResourceName = resource;
|
|
12863
13841
|
if (typeof entry === "object" && entry.resource) return entry.resource;
|
|
12864
13842
|
return resource;
|
|
12865
13843
|
}
|
|
@@ -12972,7 +13950,6 @@ var s3db_replicator_class_default = S3dbReplicator;
|
|
|
12972
13950
|
class SqsReplicator extends base_replicator_class_default {
|
|
12973
13951
|
constructor(config = {}, resources = [], client = null) {
|
|
12974
13952
|
super(config);
|
|
12975
|
-
this.resources = resources;
|
|
12976
13953
|
this.client = client;
|
|
12977
13954
|
this.queueUrl = config.queueUrl;
|
|
12978
13955
|
this.queues = config.queues || {};
|
|
@@ -12981,12 +13958,24 @@ class SqsReplicator extends base_replicator_class_default {
|
|
|
12981
13958
|
this.sqsClient = client || null;
|
|
12982
13959
|
this.messageGroupId = config.messageGroupId;
|
|
12983
13960
|
this.deduplicationId = config.deduplicationId;
|
|
12984
|
-
if (resources
|
|
13961
|
+
if (Array.isArray(resources)) {
|
|
13962
|
+
this.resources = {};
|
|
13963
|
+
for (const resource of resources) {
|
|
13964
|
+
if (typeof resource === "string") {
|
|
13965
|
+
this.resources[resource] = true;
|
|
13966
|
+
} else if (typeof resource === "object" && resource.name) {
|
|
13967
|
+
this.resources[resource.name] = resource;
|
|
13968
|
+
}
|
|
13969
|
+
}
|
|
13970
|
+
} else if (typeof resources === "object") {
|
|
13971
|
+
this.resources = resources;
|
|
12985
13972
|
for (const [resourceName, resourceConfig] of Object.entries(resources)) {
|
|
12986
|
-
if (resourceConfig.queueUrl) {
|
|
13973
|
+
if (resourceConfig && resourceConfig.queueUrl) {
|
|
12987
13974
|
this.queues[resourceName] = resourceConfig.queueUrl;
|
|
12988
13975
|
}
|
|
12989
13976
|
}
|
|
13977
|
+
} else {
|
|
13978
|
+
this.resources = {};
|
|
12990
13979
|
}
|
|
12991
13980
|
}
|
|
12992
13981
|
validateConfig() {
|
|
@@ -13222,7 +14211,7 @@ class SqsReplicator extends base_replicator_class_default {
|
|
|
13222
14211
|
connected: !!this.sqsClient,
|
|
13223
14212
|
queueUrl: this.queueUrl,
|
|
13224
14213
|
region: this.region,
|
|
13225
|
-
resources: this.resources,
|
|
14214
|
+
resources: Object.keys(this.resources || {}),
|
|
13226
14215
|
totalreplicators: this.listenerCount("replicated"),
|
|
13227
14216
|
totalErrors: this.listenerCount("replicator_error")
|
|
13228
14217
|
};
|
|
@@ -13260,33 +14249,28 @@ function normalizeResourceName(name) {
|
|
|
13260
14249
|
class ReplicatorPlugin extends plugin_class_default {
|
|
13261
14250
|
constructor(options = {}) {
|
|
13262
14251
|
super();
|
|
13263
|
-
if (options.verbose) {
|
|
13264
|
-
console.log("[PLUGIN][CONSTRUCTOR] ReplicatorPlugin constructor called");
|
|
13265
|
-
}
|
|
13266
|
-
if (options.verbose) {
|
|
13267
|
-
console.log("[PLUGIN][constructor] New ReplicatorPlugin instance created with config:", options);
|
|
13268
|
-
}
|
|
13269
14252
|
if (!options.replicators || !Array.isArray(options.replicators)) {
|
|
13270
14253
|
throw new Error("ReplicatorPlugin: replicators array is required");
|
|
13271
14254
|
}
|
|
13272
14255
|
for (const rep of options.replicators) {
|
|
13273
14256
|
if (!rep.driver) throw new Error("ReplicatorPlugin: each replicator must have a driver");
|
|
14257
|
+
if (!rep.resources || typeof rep.resources !== "object") throw new Error("ReplicatorPlugin: each replicator must have resources config");
|
|
14258
|
+
if (Object.keys(rep.resources).length === 0) throw new Error("ReplicatorPlugin: each replicator must have at least one resource configured");
|
|
13274
14259
|
}
|
|
13275
14260
|
this.config = {
|
|
13276
|
-
|
|
13277
|
-
|
|
13278
|
-
replicatorLogResource: options.replicatorLogResource
|
|
13279
|
-
|
|
14261
|
+
replicators: options.replicators || [],
|
|
14262
|
+
logErrors: options.logErrors !== false,
|
|
14263
|
+
replicatorLogResource: options.replicatorLogResource || "replicator_log",
|
|
14264
|
+
enabled: options.enabled !== false,
|
|
14265
|
+
batchSize: options.batchSize || 100,
|
|
14266
|
+
maxRetries: options.maxRetries || 3,
|
|
14267
|
+
timeout: options.timeout || 3e4,
|
|
14268
|
+
verbose: options.verbose || false,
|
|
14269
|
+
...options
|
|
13280
14270
|
};
|
|
13281
14271
|
this.replicators = [];
|
|
13282
|
-
this.
|
|
13283
|
-
this.
|
|
13284
|
-
this.stats = {
|
|
13285
|
-
totalOperations: 0,
|
|
13286
|
-
totalErrors: 0,
|
|
13287
|
-
lastError: null
|
|
13288
|
-
};
|
|
13289
|
-
this._installedListeners = [];
|
|
14272
|
+
this.database = null;
|
|
14273
|
+
this.eventListenersInstalled = /* @__PURE__ */ new Set();
|
|
13290
14274
|
}
|
|
13291
14275
|
/**
|
|
13292
14276
|
* Decompress data if it was compressed
|
|
@@ -13305,79 +14289,34 @@ class ReplicatorPlugin extends plugin_class_default {
|
|
|
13305
14289
|
}
|
|
13306
14290
|
return filtered;
|
|
13307
14291
|
}
|
|
13308
|
-
installEventListeners(resource) {
|
|
13309
|
-
|
|
13310
|
-
|
|
13311
|
-
console.log("[PLUGIN] installEventListeners called for:", resource && resource.name, {
|
|
13312
|
-
hasDatabase: !!resource.database,
|
|
13313
|
-
sameDatabase: resource.database === plugin.database,
|
|
13314
|
-
alreadyInstalled: resource._replicatorListenersInstalled,
|
|
13315
|
-
resourceObj: resource,
|
|
13316
|
-
resourceObjId: resource && resource.id,
|
|
13317
|
-
resourceObjType: typeof resource,
|
|
13318
|
-
resourceObjIs: resource && Object.is(resource, plugin.database.resources && plugin.database.resources[resource.name]),
|
|
13319
|
-
resourceObjEq: resource === (plugin.database.resources && plugin.database.resources[resource.name])
|
|
13320
|
-
});
|
|
13321
|
-
}
|
|
13322
|
-
if (!resource || resource.name === plugin.config.replicatorLogResource || !resource.database || resource.database !== plugin.database) return;
|
|
13323
|
-
if (resource._replicatorListenersInstalled) return;
|
|
13324
|
-
resource._replicatorListenersInstalled = true;
|
|
13325
|
-
this._installedListeners.push(resource);
|
|
13326
|
-
if (plugin.config.verbose) {
|
|
13327
|
-
console.log(`[PLUGIN] installEventListeners INSTALLED for resource: ${resource && resource.name}`);
|
|
14292
|
+
installEventListeners(resource, database, plugin) {
|
|
14293
|
+
if (!resource || this.eventListenersInstalled.has(resource.name) || resource.name === this.config.replicatorLogResource) {
|
|
14294
|
+
return;
|
|
13328
14295
|
}
|
|
13329
14296
|
resource.on("insert", async (data) => {
|
|
13330
|
-
if (plugin.config.verbose) {
|
|
13331
|
-
console.log("[PLUGIN] Listener INSERT on", resource.name, "plugin.replicators.length:", plugin.replicators.length, plugin.replicators.map((r) => ({ id: r.id, driver: r.driver })));
|
|
13332
|
-
}
|
|
13333
14297
|
try {
|
|
13334
|
-
const completeData =
|
|
13335
|
-
|
|
13336
|
-
|
|
13337
|
-
}
|
|
13338
|
-
await plugin.processReplicatorEvent(resource.name, "insert", data.id, completeData, null);
|
|
13339
|
-
} catch (err) {
|
|
13340
|
-
if (plugin.config.verbose) {
|
|
13341
|
-
console.error(`[PLUGIN] Listener INSERT error on ${resource.name} id=${data && data.id}:`, err);
|
|
13342
|
-
}
|
|
14298
|
+
const completeData = { ...data, createdAt: (/* @__PURE__ */ new Date()).toISOString() };
|
|
14299
|
+
await plugin.processReplicatorEvent("insert", resource.name, completeData.id, completeData);
|
|
14300
|
+
} catch (error) {
|
|
14301
|
+
this.emit("error", { operation: "insert", error: error.message, resource: resource.name });
|
|
13343
14302
|
}
|
|
13344
14303
|
});
|
|
13345
|
-
resource.on("update", async (data) => {
|
|
13346
|
-
console.log("[PLUGIN][Listener][UPDATE][START] triggered for resource:", resource.name, "data:", data);
|
|
13347
|
-
const beforeData = data && data.$before;
|
|
13348
|
-
if (plugin.config.verbose) {
|
|
13349
|
-
console.log("[PLUGIN] Listener UPDATE on", resource.name, "plugin.replicators.length:", plugin.replicators.length, plugin.replicators.map((r) => ({ id: r.id, driver: r.driver })), "data:", data, "beforeData:", beforeData);
|
|
13350
|
-
}
|
|
14304
|
+
resource.on("update", async (data, beforeData) => {
|
|
13351
14305
|
try {
|
|
13352
|
-
|
|
13353
|
-
|
|
13354
|
-
|
|
13355
|
-
|
|
13356
|
-
} else {
|
|
13357
|
-
completeData = data;
|
|
13358
|
-
}
|
|
13359
|
-
await plugin.processReplicatorEvent(resource.name, "update", data.id, completeData, beforeData);
|
|
13360
|
-
} catch (err) {
|
|
13361
|
-
if (plugin.config.verbose) {
|
|
13362
|
-
console.error(`[PLUGIN] Listener UPDATE erro em ${resource.name} id=${data && data.id}:`, err);
|
|
13363
|
-
}
|
|
14306
|
+
const completeData = { ...data, updatedAt: (/* @__PURE__ */ new Date()).toISOString() };
|
|
14307
|
+
await plugin.processReplicatorEvent("update", resource.name, completeData.id, completeData, beforeData);
|
|
14308
|
+
} catch (error) {
|
|
14309
|
+
this.emit("error", { operation: "update", error: error.message, resource: resource.name });
|
|
13364
14310
|
}
|
|
13365
14311
|
});
|
|
13366
|
-
resource.on("delete", async (data
|
|
13367
|
-
if (plugin.config.verbose) {
|
|
13368
|
-
console.log("[PLUGIN] Listener DELETE on", resource.name, "plugin.replicators.length:", plugin.replicators.length, plugin.replicators.map((r) => ({ id: r.id, driver: r.driver })));
|
|
13369
|
-
}
|
|
14312
|
+
resource.on("delete", async (data) => {
|
|
13370
14313
|
try {
|
|
13371
|
-
await plugin.processReplicatorEvent(resource.name,
|
|
13372
|
-
} catch (
|
|
13373
|
-
|
|
13374
|
-
console.error(`[PLUGIN] Listener DELETE erro em ${resource.name} id=${data && data.id}:`, err);
|
|
13375
|
-
}
|
|
14314
|
+
await plugin.processReplicatorEvent("delete", resource.name, data.id, data);
|
|
14315
|
+
} catch (error) {
|
|
14316
|
+
this.emit("error", { operation: "delete", error: error.message, resource: resource.name });
|
|
13376
14317
|
}
|
|
13377
14318
|
});
|
|
13378
|
-
|
|
13379
|
-
console.log(`[PLUGIN] Listeners instalados para resource: ${resource && resource.name} (insert: ${resource.listenerCount("insert")}, update: ${resource.listenerCount("update")}, delete: ${resource.listenerCount("delete")})`);
|
|
13380
|
-
}
|
|
14319
|
+
this.eventListenersInstalled.add(resource.name);
|
|
13381
14320
|
}
|
|
13382
14321
|
/**
|
|
13383
14322
|
* Get complete data by always fetching the full record from the resource
|
|
@@ -13388,112 +14327,58 @@ class ReplicatorPlugin extends plugin_class_default {
|
|
|
13388
14327
|
return ok ? completeRecord : data;
|
|
13389
14328
|
}
|
|
13390
14329
|
async setup(database) {
|
|
13391
|
-
console.log("[PLUGIN][SETUP] setup called");
|
|
13392
|
-
if (this.config.verbose) {
|
|
13393
|
-
console.log("[PLUGIN][setup] called with database:", database && database.name);
|
|
13394
|
-
}
|
|
13395
14330
|
this.database = database;
|
|
13396
|
-
|
|
13397
|
-
|
|
13398
|
-
|
|
13399
|
-
|
|
14331
|
+
try {
|
|
14332
|
+
await this.initializeReplicators(database);
|
|
14333
|
+
} catch (error) {
|
|
14334
|
+
this.emit("error", { operation: "setup", error: error.message });
|
|
14335
|
+
throw error;
|
|
14336
|
+
}
|
|
14337
|
+
try {
|
|
14338
|
+
if (this.config.replicatorLogResource) {
|
|
14339
|
+
const logRes = await database.createResource({
|
|
13400
14340
|
name: this.config.replicatorLogResource,
|
|
13401
|
-
behavior: "
|
|
14341
|
+
behavior: "body-overflow",
|
|
13402
14342
|
attributes: {
|
|
13403
|
-
|
|
13404
|
-
|
|
13405
|
-
|
|
13406
|
-
data: "
|
|
13407
|
-
|
|
13408
|
-
|
|
13409
|
-
|
|
13410
|
-
|
|
13411
|
-
byDate: { fields: { "createdAt": "string|maxlength:10" } }
|
|
14343
|
+
operation: "string",
|
|
14344
|
+
resourceName: "string",
|
|
14345
|
+
recordId: "string",
|
|
14346
|
+
data: "string",
|
|
14347
|
+
error: "string|optional",
|
|
14348
|
+
replicator: "string",
|
|
14349
|
+
timestamp: "string",
|
|
14350
|
+
status: "string"
|
|
13412
14351
|
}
|
|
13413
14352
|
});
|
|
13414
|
-
if (this.config.verbose) {
|
|
13415
|
-
console.log("[PLUGIN] Log resource created:", this.config.replicatorLogResource, !!logRes);
|
|
13416
|
-
}
|
|
13417
|
-
}
|
|
13418
|
-
database.resources[normalizeResourceName(this.config.replicatorLogResource)] = logRes;
|
|
13419
|
-
this.replicatorLog = logRes;
|
|
13420
|
-
if (this.config.verbose) {
|
|
13421
|
-
console.log("[PLUGIN] Log resource created and registered:", this.config.replicatorLogResource, !!database.resources[normalizeResourceName(this.config.replicatorLogResource)]);
|
|
13422
|
-
}
|
|
13423
|
-
if (typeof database.uploadMetadataFile === "function") {
|
|
13424
|
-
await database.uploadMetadataFile();
|
|
13425
|
-
if (this.config.verbose) {
|
|
13426
|
-
console.log("[PLUGIN] uploadMetadataFile called. database.resources keys:", Object.keys(database.resources));
|
|
13427
|
-
}
|
|
13428
|
-
}
|
|
13429
|
-
}
|
|
13430
|
-
if (this.config.replicators && this.config.replicators.length > 0 && this.replicators.length === 0) {
|
|
13431
|
-
await this.initializeReplicators();
|
|
13432
|
-
console.log("[PLUGIN][SETUP] after initializeReplicators, replicators.length:", this.replicators.length);
|
|
13433
|
-
if (this.config.verbose) {
|
|
13434
|
-
console.log("[PLUGIN][setup] After initializeReplicators, replicators.length:", this.replicators.length, this.replicators.map((r) => ({ id: r.id, driver: r.driver })));
|
|
13435
|
-
}
|
|
13436
|
-
}
|
|
13437
|
-
for (const resourceName in database.resources) {
|
|
13438
|
-
if (normalizeResourceName(resourceName) !== normalizeResourceName(this.config.replicatorLogResource)) {
|
|
13439
|
-
this.installEventListeners(database.resources[resourceName]);
|
|
13440
14353
|
}
|
|
14354
|
+
} catch (error) {
|
|
13441
14355
|
}
|
|
13442
|
-
|
|
13443
|
-
for (const resourceName in database.resources) {
|
|
13444
|
-
if (normalizeResourceName(resourceName) !== normalizeResourceName(this.config.replicatorLogResource)) {
|
|
13445
|
-
this.installEventListeners(database.resources[resourceName]);
|
|
13446
|
-
}
|
|
13447
|
-
}
|
|
13448
|
-
});
|
|
14356
|
+
await this.uploadMetadataFile(database);
|
|
13449
14357
|
const originalCreateResource = database.createResource.bind(database);
|
|
13450
14358
|
database.createResource = async (config) => {
|
|
13451
|
-
if (this.config.verbose) {
|
|
13452
|
-
console.log("[PLUGIN] createResource proxy called for:", config && config.name);
|
|
13453
|
-
}
|
|
13454
14359
|
const resource = await originalCreateResource(config);
|
|
13455
|
-
if (resource
|
|
13456
|
-
this.installEventListeners(resource);
|
|
14360
|
+
if (resource) {
|
|
14361
|
+
this.installEventListeners(resource, database, this);
|
|
13457
14362
|
}
|
|
13458
14363
|
return resource;
|
|
13459
14364
|
};
|
|
13460
|
-
|
|
13461
|
-
const resource = database.resources[resourceName];
|
|
13462
|
-
if (resource && resource.name !== this.config.replicatorLogResource) {
|
|
13463
|
-
this.installEventListeners(resource);
|
|
13464
|
-
}
|
|
13465
|
-
});
|
|
13466
|
-
database.on("s3db.resourceUpdated", (resourceName) => {
|
|
14365
|
+
for (const resourceName in database.resources) {
|
|
13467
14366
|
const resource = database.resources[resourceName];
|
|
13468
|
-
|
|
13469
|
-
|
|
13470
|
-
|
|
13471
|
-
|
|
14367
|
+
this.installEventListeners(resource, database, this);
|
|
14368
|
+
}
|
|
14369
|
+
}
|
|
14370
|
+
createReplicator(driver, config, resources, client) {
|
|
14371
|
+
return createReplicator(driver, config, resources, client);
|
|
13472
14372
|
}
|
|
13473
|
-
async initializeReplicators() {
|
|
13474
|
-
console.log("[PLUGIN][INIT] initializeReplicators called");
|
|
14373
|
+
async initializeReplicators(database) {
|
|
13475
14374
|
for (const replicatorConfig of this.config.replicators) {
|
|
13476
|
-
|
|
13477
|
-
|
|
13478
|
-
|
|
13479
|
-
|
|
13480
|
-
|
|
13481
|
-
|
|
13482
|
-
|
|
13483
|
-
await replicator.initialize(this.database);
|
|
13484
|
-
this.replicators.push({
|
|
13485
|
-
id: Math.random().toString(36).slice(2),
|
|
13486
|
-
driver,
|
|
13487
|
-
config: replicatorConfig,
|
|
13488
|
-
resources,
|
|
13489
|
-
instance: replicator
|
|
13490
|
-
});
|
|
13491
|
-
console.log("[PLUGIN][INIT] pushed replicator:", driver, resources);
|
|
13492
|
-
} else {
|
|
13493
|
-
console.log("[PLUGIN][INIT] createReplicator returned null/undefined for driver:", driver);
|
|
13494
|
-
}
|
|
13495
|
-
} catch (err) {
|
|
13496
|
-
console.error("[PLUGIN][INIT] Error creating replicator:", err);
|
|
14375
|
+
const { driver, config = {}, resources, client, ...otherConfig } = replicatorConfig;
|
|
14376
|
+
const replicatorResources = resources || config.resources || {};
|
|
14377
|
+
const mergedConfig = { ...config, ...otherConfig };
|
|
14378
|
+
const replicator = this.createReplicator(driver, mergedConfig, replicatorResources, client);
|
|
14379
|
+
if (replicator) {
|
|
14380
|
+
await replicator.initialize(database);
|
|
14381
|
+
this.replicators.push(replicator);
|
|
13497
14382
|
}
|
|
13498
14383
|
}
|
|
13499
14384
|
}
|
|
@@ -13501,160 +14386,162 @@ class ReplicatorPlugin extends plugin_class_default {
|
|
|
13501
14386
|
}
|
|
13502
14387
|
async stop() {
|
|
13503
14388
|
}
|
|
13504
|
-
|
|
13505
|
-
if (
|
|
13506
|
-
|
|
13507
|
-
|
|
14389
|
+
filterInternalFields(data) {
|
|
14390
|
+
if (!data || typeof data !== "object") return data;
|
|
14391
|
+
const filtered = {};
|
|
14392
|
+
for (const [key, value] of Object.entries(data)) {
|
|
14393
|
+
if (!key.startsWith("_") && !key.startsWith("$")) {
|
|
14394
|
+
filtered[key] = value;
|
|
14395
|
+
}
|
|
13508
14396
|
}
|
|
13509
|
-
|
|
13510
|
-
|
|
14397
|
+
return filtered;
|
|
14398
|
+
}
|
|
14399
|
+
async uploadMetadataFile(database) {
|
|
14400
|
+
if (typeof database.uploadMetadataFile === "function") {
|
|
14401
|
+
await database.uploadMetadataFile();
|
|
13511
14402
|
}
|
|
13512
|
-
|
|
13513
|
-
|
|
14403
|
+
}
|
|
14404
|
+
async getCompleteData(resource, data) {
|
|
14405
|
+
try {
|
|
14406
|
+
const [ok, err, record] = await try_fn_default(() => resource.get(data.id));
|
|
14407
|
+
if (ok && record) {
|
|
14408
|
+
return record;
|
|
14409
|
+
}
|
|
14410
|
+
} catch (error) {
|
|
13514
14411
|
}
|
|
13515
|
-
|
|
13516
|
-
|
|
13517
|
-
|
|
14412
|
+
return data;
|
|
14413
|
+
}
|
|
14414
|
+
async retryWithBackoff(operation, maxRetries = 3) {
|
|
14415
|
+
let lastError;
|
|
14416
|
+
for (let attempt = 1; attempt <= maxRetries; attempt++) {
|
|
14417
|
+
try {
|
|
14418
|
+
return await operation();
|
|
14419
|
+
} catch (error) {
|
|
14420
|
+
lastError = error;
|
|
14421
|
+
if (attempt === maxRetries) {
|
|
14422
|
+
throw error;
|
|
14423
|
+
}
|
|
14424
|
+
const delay = Math.pow(2, attempt - 1) * 1e3;
|
|
14425
|
+
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
13518
14426
|
}
|
|
13519
|
-
return;
|
|
13520
14427
|
}
|
|
13521
|
-
|
|
13522
|
-
|
|
13523
|
-
|
|
13524
|
-
|
|
14428
|
+
throw lastError;
|
|
14429
|
+
}
|
|
14430
|
+
async logError(replicator, resourceName, operation, recordId, data, error) {
|
|
14431
|
+
try {
|
|
14432
|
+
const logResourceName = this.config.replicatorLogResource;
|
|
14433
|
+
if (this.database && this.database.resources && this.database.resources[logResourceName]) {
|
|
14434
|
+
const logResource = this.database.resources[logResourceName];
|
|
14435
|
+
await logResource.insert({
|
|
14436
|
+
replicator: replicator.name || replicator.id,
|
|
14437
|
+
resourceName,
|
|
14438
|
+
operation,
|
|
14439
|
+
recordId,
|
|
14440
|
+
data: JSON.stringify(data),
|
|
14441
|
+
error: error.message,
|
|
14442
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
14443
|
+
status: "error"
|
|
14444
|
+
});
|
|
13525
14445
|
}
|
|
14446
|
+
} catch (logError) {
|
|
14447
|
+
}
|
|
14448
|
+
}
|
|
14449
|
+
async processReplicatorEvent(operation, resourceName, recordId, data, beforeData = null) {
|
|
14450
|
+
if (!this.config.enabled) return;
|
|
14451
|
+
const applicableReplicators = this.replicators.filter((replicator) => {
|
|
14452
|
+
const should = replicator.shouldReplicateResource && replicator.shouldReplicateResource(resourceName, operation);
|
|
13526
14453
|
return should;
|
|
13527
14454
|
});
|
|
13528
|
-
if (this.config.verbose) {
|
|
13529
|
-
console.log(`[PLUGIN] processReplicatorEvent: applicableReplicators for resource=${resourceName}:`, applicableReplicators.map((r) => r.driver));
|
|
13530
|
-
}
|
|
13531
14455
|
if (applicableReplicators.length === 0) {
|
|
13532
|
-
if (this.config.verbose) {
|
|
13533
|
-
console.log("[PLUGIN] No applicable replicators for resource", resourceName);
|
|
13534
|
-
}
|
|
13535
14456
|
return;
|
|
13536
14457
|
}
|
|
13537
|
-
const
|
|
13538
|
-
|
|
13539
|
-
|
|
13540
|
-
|
|
13541
|
-
|
|
13542
|
-
|
|
13543
|
-
|
|
13544
|
-
|
|
13545
|
-
|
|
13546
|
-
|
|
13547
|
-
|
|
13548
|
-
|
|
13549
|
-
|
|
13550
|
-
const [ok, err, result] = await try_fn_default(async () => this.processreplicatorItem(item));
|
|
13551
|
-
if (ok) {
|
|
13552
|
-
if (logId) {
|
|
13553
|
-
await this.updatereplicatorLog(logId, {
|
|
13554
|
-
status: result.success ? "success" : "failed",
|
|
13555
|
-
attempts: 1,
|
|
13556
|
-
error: result.success ? "" : JSON.stringify(result.results)
|
|
14458
|
+
const promises = applicableReplicators.map(async (replicator) => {
|
|
14459
|
+
try {
|
|
14460
|
+
const result = await this.retryWithBackoff(
|
|
14461
|
+
() => replicator.replicate(resourceName, operation, data, recordId, beforeData),
|
|
14462
|
+
this.config.maxRetries
|
|
14463
|
+
);
|
|
14464
|
+
this.emit("replicated", {
|
|
14465
|
+
replicator: replicator.name || replicator.id,
|
|
14466
|
+
resourceName,
|
|
14467
|
+
operation,
|
|
14468
|
+
recordId,
|
|
14469
|
+
result,
|
|
14470
|
+
success: true
|
|
13557
14471
|
});
|
|
13558
|
-
|
|
13559
|
-
|
|
13560
|
-
|
|
13561
|
-
|
|
13562
|
-
|
|
13563
|
-
|
|
13564
|
-
|
|
13565
|
-
|
|
13566
|
-
if (logId) {
|
|
13567
|
-
await this.updatereplicatorLog(logId, {
|
|
13568
|
-
status: "failed",
|
|
13569
|
-
attempts: 1,
|
|
13570
|
-
error: err.message
|
|
14472
|
+
return result;
|
|
14473
|
+
} catch (error) {
|
|
14474
|
+
this.emit("replicator_error", {
|
|
14475
|
+
replicator: replicator.name || replicator.id,
|
|
14476
|
+
resourceName,
|
|
14477
|
+
operation,
|
|
14478
|
+
recordId,
|
|
14479
|
+
error: error.message
|
|
13571
14480
|
});
|
|
14481
|
+
if (this.config.logErrors && this.database) {
|
|
14482
|
+
await this.logError(replicator, resourceName, operation, recordId, data, error);
|
|
14483
|
+
}
|
|
14484
|
+
throw error;
|
|
13572
14485
|
}
|
|
13573
|
-
|
|
13574
|
-
|
|
14486
|
+
});
|
|
14487
|
+
return Promise.allSettled(promises);
|
|
13575
14488
|
}
|
|
13576
14489
|
async processreplicatorItem(item) {
|
|
13577
|
-
if (this.config.verbose) {
|
|
13578
|
-
console.log("[PLUGIN][processreplicatorItem] called with item:", item);
|
|
13579
|
-
}
|
|
13580
14490
|
const applicableReplicators = this.replicators.filter((replicator) => {
|
|
13581
|
-
const should = replicator.
|
|
13582
|
-
if (this.config.verbose) {
|
|
13583
|
-
console.log(`[PLUGIN] processreplicatorItem: Replicator ${replicator.driver} shouldReplicateResource(${item.resourceName}, ${item.operation}):`, should);
|
|
13584
|
-
}
|
|
14491
|
+
const should = replicator.shouldReplicateResource && replicator.shouldReplicateResource(item.resourceName, item.operation);
|
|
13585
14492
|
return should;
|
|
13586
14493
|
});
|
|
13587
|
-
if (this.config.verbose) {
|
|
13588
|
-
console.log(`[PLUGIN] processreplicatorItem: applicableReplicators for resource=${item.resourceName}:`, applicableReplicators.map((r) => r.driver));
|
|
13589
|
-
}
|
|
13590
14494
|
if (applicableReplicators.length === 0) {
|
|
13591
|
-
|
|
13592
|
-
console.log("[PLUGIN] processreplicatorItem: No applicable replicators for resource", item.resourceName);
|
|
13593
|
-
}
|
|
13594
|
-
return { success: true, skipped: true, reason: "no_applicable_replicators" };
|
|
14495
|
+
return;
|
|
13595
14496
|
}
|
|
13596
|
-
const
|
|
13597
|
-
|
|
13598
|
-
|
|
13599
|
-
|
|
13600
|
-
|
|
13601
|
-
|
|
13602
|
-
|
|
14497
|
+
const promises = applicableReplicators.map(async (replicator) => {
|
|
14498
|
+
try {
|
|
14499
|
+
const [ok, err, result] = await try_fn_default(
|
|
14500
|
+
() => replicator.replicate(item.resourceName, item.operation, item.data, item.recordId, item.beforeData)
|
|
14501
|
+
);
|
|
14502
|
+
if (!ok) {
|
|
14503
|
+
this.emit("replicator_error", {
|
|
14504
|
+
replicator: replicator.name || replicator.id,
|
|
14505
|
+
resourceName: item.resourceName,
|
|
14506
|
+
operation: item.operation,
|
|
14507
|
+
recordId: item.recordId,
|
|
14508
|
+
error: err.message
|
|
14509
|
+
});
|
|
14510
|
+
if (this.config.logErrors && this.database) {
|
|
14511
|
+
await this.logError(replicator, item.resourceName, item.operation, item.recordId, item.data, err);
|
|
14512
|
+
}
|
|
14513
|
+
return { success: false, error: err.message };
|
|
14514
|
+
}
|
|
14515
|
+
this.emit("replicated", {
|
|
14516
|
+
replicator: replicator.name || replicator.id,
|
|
14517
|
+
resourceName: item.resourceName,
|
|
13603
14518
|
operation: item.operation,
|
|
13604
|
-
|
|
13605
|
-
|
|
13606
|
-
|
|
14519
|
+
recordId: item.recordId,
|
|
14520
|
+
result,
|
|
14521
|
+
success: true
|
|
13607
14522
|
});
|
|
14523
|
+
return { success: true, result };
|
|
14524
|
+
} catch (error) {
|
|
14525
|
+
this.emit("replicator_error", {
|
|
14526
|
+
replicator: replicator.name || replicator.id,
|
|
14527
|
+
resourceName: item.resourceName,
|
|
14528
|
+
operation: item.operation,
|
|
14529
|
+
recordId: item.recordId,
|
|
14530
|
+
error: error.message
|
|
14531
|
+
});
|
|
14532
|
+
if (this.config.logErrors && this.database) {
|
|
14533
|
+
await this.logError(replicator, item.resourceName, item.operation, item.recordId, item.data, error);
|
|
14534
|
+
}
|
|
14535
|
+
return { success: false, error: error.message };
|
|
13608
14536
|
}
|
|
13609
|
-
|
|
13610
|
-
|
|
13611
|
-
() => replicator.instance.replicate({
|
|
13612
|
-
resource: item.resourceName,
|
|
13613
|
-
operation: item.operation,
|
|
13614
|
-
data: item.data,
|
|
13615
|
-
id: item.recordId,
|
|
13616
|
-
beforeData: item.beforeData
|
|
13617
|
-
})
|
|
13618
|
-
);
|
|
13619
|
-
} else {
|
|
13620
|
-
[ok, err, result] = await try_fn_default(
|
|
13621
|
-
() => replicator.instance.replicate(
|
|
13622
|
-
item.resourceName,
|
|
13623
|
-
item.operation,
|
|
13624
|
-
item.data,
|
|
13625
|
-
item.recordId,
|
|
13626
|
-
item.beforeData
|
|
13627
|
-
)
|
|
13628
|
-
);
|
|
13629
|
-
}
|
|
13630
|
-
results.push({
|
|
13631
|
-
replicatorId: replicator.id,
|
|
13632
|
-
driver: replicator.driver,
|
|
13633
|
-
success: result && result.success,
|
|
13634
|
-
error: result && result.error,
|
|
13635
|
-
skipped: result && result.skipped
|
|
13636
|
-
});
|
|
13637
|
-
}
|
|
13638
|
-
return {
|
|
13639
|
-
success: results.every((r) => r.success || r.skipped),
|
|
13640
|
-
results
|
|
13641
|
-
};
|
|
14537
|
+
});
|
|
14538
|
+
return Promise.allSettled(promises);
|
|
13642
14539
|
}
|
|
13643
14540
|
async logreplicator(item) {
|
|
13644
14541
|
const logRes = this.replicatorLog || this.database.resources[normalizeResourceName(this.config.replicatorLogResource)];
|
|
13645
14542
|
if (!logRes) {
|
|
13646
|
-
if (this.config.verbose) {
|
|
13647
|
-
console.error("[PLUGIN] replicator log resource not found!");
|
|
13648
|
-
}
|
|
13649
14543
|
if (this.database) {
|
|
13650
|
-
if (this.
|
|
13651
|
-
console.warn("[PLUGIN] database.resources keys:", Object.keys(this.database.resources));
|
|
13652
|
-
}
|
|
13653
|
-
if (this.database.options && this.database.options.connectionString) {
|
|
13654
|
-
if (this.config.verbose) {
|
|
13655
|
-
console.warn("[PLUGIN] database connectionString:", this.database.options.connectionString);
|
|
13656
|
-
}
|
|
13657
|
-
}
|
|
14544
|
+
if (this.database.options && this.database.options.connectionString) ;
|
|
13658
14545
|
}
|
|
13659
14546
|
this.emit("replicator.log.failed", { error: "replicator log resource not found", item });
|
|
13660
14547
|
return;
|
|
@@ -13670,9 +14557,6 @@ class ReplicatorPlugin extends plugin_class_default {
|
|
|
13670
14557
|
try {
|
|
13671
14558
|
await logRes.insert(logItem);
|
|
13672
14559
|
} catch (err) {
|
|
13673
|
-
if (this.config.verbose) {
|
|
13674
|
-
console.error("[PLUGIN] Error writing to replicator log:", err);
|
|
13675
|
-
}
|
|
13676
14560
|
this.emit("replicator.log.failed", { error: err, item });
|
|
13677
14561
|
}
|
|
13678
14562
|
}
|
|
@@ -13692,7 +14576,7 @@ class ReplicatorPlugin extends plugin_class_default {
|
|
|
13692
14576
|
async getreplicatorStats() {
|
|
13693
14577
|
const replicatorStats = await Promise.all(
|
|
13694
14578
|
this.replicators.map(async (replicator) => {
|
|
13695
|
-
const status = await replicator.
|
|
14579
|
+
const status = await replicator.getStatus();
|
|
13696
14580
|
return {
|
|
13697
14581
|
id: replicator.id,
|
|
13698
14582
|
driver: replicator.driver,
|
|
@@ -13754,10 +14638,6 @@ class ReplicatorPlugin extends plugin_class_default {
|
|
|
13754
14638
|
});
|
|
13755
14639
|
if (ok) {
|
|
13756
14640
|
retried++;
|
|
13757
|
-
} else {
|
|
13758
|
-
if (this.config.verbose) {
|
|
13759
|
-
console.error("Failed to retry replicator:", err);
|
|
13760
|
-
}
|
|
13761
14641
|
}
|
|
13762
14642
|
}
|
|
13763
14643
|
return { retried };
|
|
@@ -13770,52 +14650,35 @@ class ReplicatorPlugin extends plugin_class_default {
|
|
|
13770
14650
|
this.stats.lastSync = (/* @__PURE__ */ new Date()).toISOString();
|
|
13771
14651
|
for (const resourceName in this.database.resources) {
|
|
13772
14652
|
if (normalizeResourceName(resourceName) === normalizeResourceName("replicator_logs")) continue;
|
|
13773
|
-
if (replicator.
|
|
14653
|
+
if (replicator.shouldReplicateResource(resourceName)) {
|
|
13774
14654
|
this.emit("replicator.sync.resource", { resourceName, replicatorId });
|
|
13775
14655
|
const resource = this.database.resources[resourceName];
|
|
13776
14656
|
const allRecords = await resource.getAll();
|
|
13777
14657
|
for (const record of allRecords) {
|
|
13778
|
-
await replicator.
|
|
14658
|
+
await replicator.replicate(resourceName, "insert", record, record.id);
|
|
13779
14659
|
}
|
|
13780
14660
|
}
|
|
13781
14661
|
}
|
|
13782
14662
|
this.emit("replicator.sync.completed", { replicatorId, stats: this.stats });
|
|
13783
14663
|
}
|
|
13784
14664
|
async cleanup() {
|
|
13785
|
-
|
|
13786
|
-
|
|
13787
|
-
|
|
13788
|
-
|
|
13789
|
-
|
|
13790
|
-
|
|
13791
|
-
|
|
13792
|
-
|
|
13793
|
-
|
|
13794
|
-
}
|
|
13795
|
-
|
|
13796
|
-
}
|
|
13797
|
-
this._installedListeners = [];
|
|
13798
|
-
}
|
|
13799
|
-
if (this.database && typeof this.database.removeAllListeners === "function") {
|
|
13800
|
-
this.database.removeAllListeners();
|
|
13801
|
-
}
|
|
13802
|
-
if (this.replicators && Array.isArray(this.replicators)) {
|
|
13803
|
-
for (const rep of this.replicators) {
|
|
13804
|
-
if (rep.instance && typeof rep.instance.cleanup === "function") {
|
|
13805
|
-
await rep.instance.cleanup();
|
|
13806
|
-
}
|
|
14665
|
+
try {
|
|
14666
|
+
if (this.replicators && this.replicators.length > 0) {
|
|
14667
|
+
const cleanupPromises = this.replicators.map(async (replicator) => {
|
|
14668
|
+
try {
|
|
14669
|
+
if (replicator && typeof replicator.cleanup === "function") {
|
|
14670
|
+
await replicator.cleanup();
|
|
14671
|
+
}
|
|
14672
|
+
} catch (error) {
|
|
14673
|
+
}
|
|
14674
|
+
});
|
|
14675
|
+
await Promise.allSettled(cleanupPromises);
|
|
13807
14676
|
}
|
|
13808
14677
|
this.replicators = [];
|
|
13809
|
-
|
|
13810
|
-
|
|
13811
|
-
|
|
13812
|
-
|
|
13813
|
-
totalOperations: 0,
|
|
13814
|
-
totalErrors: 0,
|
|
13815
|
-
lastError: null
|
|
13816
|
-
};
|
|
13817
|
-
if (this.config.verbose) {
|
|
13818
|
-
console.log("[PLUGIN][CLEANUP] ReplicatorPlugin cleanup complete");
|
|
14678
|
+
this.database = null;
|
|
14679
|
+
this.eventListenersInstalled.clear();
|
|
14680
|
+
this.removeAllListeners();
|
|
14681
|
+
} catch (error) {
|
|
13819
14682
|
}
|
|
13820
14683
|
}
|
|
13821
14684
|
}
|