s3db.js 9.3.0 → 10.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +72 -13
- package/dist/s3db.cjs.js +2342 -540
- package/dist/s3db.cjs.js.map +1 -1
- package/dist/s3db.es.js +2341 -541
- package/dist/s3db.es.js.map +1 -1
- package/package.json +1 -1
- package/src/client.class.js +8 -7
- package/src/concerns/high-performance-inserter.js +285 -0
- package/src/concerns/partition-queue.js +171 -0
- package/src/errors.js +10 -2
- package/src/partition-drivers/base-partition-driver.js +96 -0
- package/src/partition-drivers/index.js +60 -0
- package/src/partition-drivers/memory-partition-driver.js +274 -0
- package/src/partition-drivers/sqs-partition-driver.js +332 -0
- package/src/partition-drivers/sync-partition-driver.js +38 -0
- package/src/plugins/audit.plugin.js +4 -4
- package/src/plugins/backup.plugin.js +380 -105
- package/src/plugins/backup.plugin.js.backup +1 -1
- package/src/plugins/cache.plugin.js +203 -150
- package/src/plugins/eventual-consistency.plugin.js +1012 -0
- package/src/plugins/fulltext.plugin.js +6 -6
- package/src/plugins/index.js +2 -0
- package/src/plugins/metrics.plugin.js +13 -13
- package/src/plugins/replicator.plugin.js +108 -70
- package/src/plugins/replicators/s3db-replicator.class.js +7 -3
- package/src/plugins/replicators/sqs-replicator.class.js +11 -3
- package/src/plugins/s3-queue.plugin.js +776 -0
- package/src/plugins/scheduler.plugin.js +226 -164
- package/src/plugins/state-machine.plugin.js +109 -81
- package/src/resource.class.js +205 -0
- package/PLUGINS.md +0 -5036
package/dist/s3db.cjs.js
CHANGED
|
@@ -10,11 +10,12 @@ var promises$1 = require('stream/promises');
|
|
|
10
10
|
var path = require('path');
|
|
11
11
|
var crypto = require('crypto');
|
|
12
12
|
var zlib = require('node:zlib');
|
|
13
|
+
var os = require('os');
|
|
14
|
+
var jsonStableStringify = require('json-stable-stringify');
|
|
13
15
|
var stream = require('stream');
|
|
14
16
|
var promisePool = require('@supercharge/promise-pool');
|
|
15
17
|
var web = require('node:stream/web');
|
|
16
18
|
var lodashEs = require('lodash-es');
|
|
17
|
-
var jsonStableStringify = require('json-stable-stringify');
|
|
18
19
|
var http = require('http');
|
|
19
20
|
var https = require('https');
|
|
20
21
|
var nodeHttpHandler = require('@smithy/node-http-handler');
|
|
@@ -423,8 +424,14 @@ function mapAwsError(err, context = {}) {
|
|
|
423
424
|
suggestion = "Check if the object metadata is present and valid.";
|
|
424
425
|
return new MissingMetadata({ ...context, original: err, metadata, commandName, commandInput, suggestion });
|
|
425
426
|
}
|
|
426
|
-
|
|
427
|
-
|
|
427
|
+
const errorDetails = [
|
|
428
|
+
`Unknown error: ${err.message || err.toString()}`,
|
|
429
|
+
err.code && `Code: ${err.code}`,
|
|
430
|
+
err.statusCode && `Status: ${err.statusCode}`,
|
|
431
|
+
err.stack && `Stack: ${err.stack.split("\n")[0]}`
|
|
432
|
+
].filter(Boolean).join(" | ");
|
|
433
|
+
suggestion = `Check the error details and AWS documentation. Original error: ${err.message || err.toString()}`;
|
|
434
|
+
return new UnknownError(errorDetails, { ...context, original: err, metadata, commandName, commandInput, suggestion });
|
|
428
435
|
}
|
|
429
436
|
class ConnectionStringError extends S3dbError {
|
|
430
437
|
constructor(message, details = {}) {
|
|
@@ -836,7 +843,7 @@ class AuditPlugin extends Plugin {
|
|
|
836
843
|
}
|
|
837
844
|
async onSetup() {
|
|
838
845
|
const [ok, err, auditResource] = await tryFn(() => this.database.createResource({
|
|
839
|
-
name: "
|
|
846
|
+
name: "plg_audits",
|
|
840
847
|
attributes: {
|
|
841
848
|
id: "string|required",
|
|
842
849
|
resourceName: "string|required",
|
|
@@ -852,15 +859,15 @@ class AuditPlugin extends Plugin {
|
|
|
852
859
|
},
|
|
853
860
|
behavior: "body-overflow"
|
|
854
861
|
}));
|
|
855
|
-
this.auditResource = ok ? auditResource : this.database.resources.
|
|
862
|
+
this.auditResource = ok ? auditResource : this.database.resources.plg_audits || null;
|
|
856
863
|
if (!ok && !this.auditResource) return;
|
|
857
864
|
this.database.addHook("afterCreateResource", (context) => {
|
|
858
|
-
if (context.resource.name !== "
|
|
865
|
+
if (context.resource.name !== "plg_audits") {
|
|
859
866
|
this.setupResourceAuditing(context.resource);
|
|
860
867
|
}
|
|
861
868
|
});
|
|
862
869
|
for (const resource of Object.values(this.database.resources)) {
|
|
863
|
-
if (resource.name !== "
|
|
870
|
+
if (resource.name !== "plg_audits") {
|
|
864
871
|
this.setupResourceAuditing(resource);
|
|
865
872
|
}
|
|
866
873
|
}
|
|
@@ -1880,11 +1887,10 @@ function validateBackupConfig(driver, config = {}) {
|
|
|
1880
1887
|
class BackupPlugin extends Plugin {
|
|
1881
1888
|
constructor(options = {}) {
|
|
1882
1889
|
super();
|
|
1883
|
-
this.driverName = options.driver || "filesystem";
|
|
1884
|
-
this.driverConfig = options.config || {};
|
|
1885
1890
|
this.config = {
|
|
1886
|
-
//
|
|
1887
|
-
|
|
1891
|
+
// Driver configuration
|
|
1892
|
+
driver: options.driver || "filesystem",
|
|
1893
|
+
driverConfig: options.config || {},
|
|
1888
1894
|
// Scheduling configuration
|
|
1889
1895
|
schedule: options.schedule || {},
|
|
1890
1896
|
// Retention policy (Grandfather-Father-Son)
|
|
@@ -1902,8 +1908,8 @@ class BackupPlugin extends Plugin {
|
|
|
1902
1908
|
parallelism: options.parallelism || 4,
|
|
1903
1909
|
include: options.include || null,
|
|
1904
1910
|
exclude: options.exclude || [],
|
|
1905
|
-
backupMetadataResource: options.backupMetadataResource || "
|
|
1906
|
-
tempDir: options.tempDir || "
|
|
1911
|
+
backupMetadataResource: options.backupMetadataResource || "plg_backup_metadata",
|
|
1912
|
+
tempDir: options.tempDir || path.join(os.tmpdir(), "s3db", "backups"),
|
|
1907
1913
|
verbose: options.verbose || false,
|
|
1908
1914
|
// Hooks
|
|
1909
1915
|
onBackupStart: options.onBackupStart || null,
|
|
@@ -1915,32 +1921,9 @@ class BackupPlugin extends Plugin {
|
|
|
1915
1921
|
};
|
|
1916
1922
|
this.driver = null;
|
|
1917
1923
|
this.activeBackups = /* @__PURE__ */ new Set();
|
|
1918
|
-
this.
|
|
1919
|
-
validateBackupConfig(this.driverName, this.driverConfig);
|
|
1924
|
+
validateBackupConfig(this.config.driver, this.config.driverConfig);
|
|
1920
1925
|
this._validateConfiguration();
|
|
1921
1926
|
}
|
|
1922
|
-
/**
|
|
1923
|
-
* Convert legacy destinations format to multi driver format
|
|
1924
|
-
*/
|
|
1925
|
-
_handleLegacyDestinations() {
|
|
1926
|
-
if (this.config.destinations && Array.isArray(this.config.destinations)) {
|
|
1927
|
-
this.driverName = "multi";
|
|
1928
|
-
this.driverConfig = {
|
|
1929
|
-
strategy: "all",
|
|
1930
|
-
destinations: this.config.destinations.map((dest) => {
|
|
1931
|
-
const { type, ...config } = dest;
|
|
1932
|
-
return {
|
|
1933
|
-
driver: type,
|
|
1934
|
-
config
|
|
1935
|
-
};
|
|
1936
|
-
})
|
|
1937
|
-
};
|
|
1938
|
-
this.config.destinations = null;
|
|
1939
|
-
if (this.config.verbose) {
|
|
1940
|
-
console.log("[BackupPlugin] Converted legacy destinations format to multi driver");
|
|
1941
|
-
}
|
|
1942
|
-
}
|
|
1943
|
-
}
|
|
1944
1927
|
_validateConfiguration() {
|
|
1945
1928
|
if (this.config.encryption && (!this.config.encryption.key || !this.config.encryption.algorithm)) {
|
|
1946
1929
|
throw new Error("BackupPlugin: Encryption requires both key and algorithm");
|
|
@@ -1950,7 +1933,7 @@ class BackupPlugin extends Plugin {
|
|
|
1950
1933
|
}
|
|
1951
1934
|
}
|
|
1952
1935
|
async onSetup() {
|
|
1953
|
-
this.driver = createBackupDriver(this.
|
|
1936
|
+
this.driver = createBackupDriver(this.config.driver, this.config.driverConfig);
|
|
1954
1937
|
await this.driver.setup(this.database);
|
|
1955
1938
|
await promises.mkdir(this.config.tempDir, { recursive: true });
|
|
1956
1939
|
await this._createBackupMetadataResource();
|
|
@@ -1998,6 +1981,9 @@ class BackupPlugin extends Plugin {
|
|
|
1998
1981
|
async backup(type = "full", options = {}) {
|
|
1999
1982
|
const backupId = this._generateBackupId(type);
|
|
2000
1983
|
const startTime = Date.now();
|
|
1984
|
+
if (this.activeBackups.has(backupId)) {
|
|
1985
|
+
throw new Error(`Backup '${backupId}' is already in progress`);
|
|
1986
|
+
}
|
|
2001
1987
|
try {
|
|
2002
1988
|
this.activeBackups.add(backupId);
|
|
2003
1989
|
if (this.config.onBackupStart) {
|
|
@@ -2013,16 +1999,9 @@ class BackupPlugin extends Plugin {
|
|
|
2013
1999
|
if (exportedFiles.length === 0) {
|
|
2014
2000
|
throw new Error("No resources were exported for backup");
|
|
2015
2001
|
}
|
|
2016
|
-
|
|
2017
|
-
|
|
2018
|
-
|
|
2019
|
-
finalPath = path.join(tempBackupDir, `${backupId}.tar.gz`);
|
|
2020
|
-
totalSize = await this._createCompressedArchive(exportedFiles, finalPath);
|
|
2021
|
-
} else {
|
|
2022
|
-
finalPath = exportedFiles[0];
|
|
2023
|
-
const [statOk, , stats] = await tryFn(() => promises.stat(finalPath));
|
|
2024
|
-
totalSize = statOk ? stats.size : 0;
|
|
2025
|
-
}
|
|
2002
|
+
const archiveExtension = this.config.compression !== "none" ? ".tar.gz" : ".json";
|
|
2003
|
+
const finalPath = path.join(tempBackupDir, `${backupId}${archiveExtension}`);
|
|
2004
|
+
const totalSize = await this._createArchive(exportedFiles, finalPath, this.config.compression);
|
|
2026
2005
|
const checksum = await this._generateChecksum(finalPath);
|
|
2027
2006
|
const uploadResult = await this.driver.upload(finalPath, backupId, manifest);
|
|
2028
2007
|
if (this.config.verification) {
|
|
@@ -2131,15 +2110,35 @@ class BackupPlugin extends Plugin {
|
|
|
2131
2110
|
for (const resourceName of resourceNames) {
|
|
2132
2111
|
const resource = this.database.resources[resourceName];
|
|
2133
2112
|
if (!resource) {
|
|
2134
|
-
|
|
2113
|
+
if (this.config.verbose) {
|
|
2114
|
+
console.warn(`[BackupPlugin] Resource '${resourceName}' not found, skipping`);
|
|
2115
|
+
}
|
|
2135
2116
|
continue;
|
|
2136
2117
|
}
|
|
2137
2118
|
const exportPath = path.join(tempDir, `${resourceName}.json`);
|
|
2138
2119
|
let records;
|
|
2139
2120
|
if (type === "incremental") {
|
|
2140
|
-
const
|
|
2121
|
+
const [lastBackupOk, , lastBackups] = await tryFn(
|
|
2122
|
+
() => this.database.resource(this.config.backupMetadataResource).list({
|
|
2123
|
+
filter: {
|
|
2124
|
+
status: "completed",
|
|
2125
|
+
type: { $in: ["full", "incremental"] }
|
|
2126
|
+
},
|
|
2127
|
+
sort: { timestamp: -1 },
|
|
2128
|
+
limit: 1
|
|
2129
|
+
})
|
|
2130
|
+
);
|
|
2131
|
+
let sinceTimestamp;
|
|
2132
|
+
if (lastBackupOk && lastBackups && lastBackups.length > 0) {
|
|
2133
|
+
sinceTimestamp = new Date(lastBackups[0].timestamp);
|
|
2134
|
+
} else {
|
|
2135
|
+
sinceTimestamp = new Date(Date.now() - 24 * 60 * 60 * 1e3);
|
|
2136
|
+
}
|
|
2137
|
+
if (this.config.verbose) {
|
|
2138
|
+
console.log(`[BackupPlugin] Incremental backup for '${resourceName}' since ${sinceTimestamp.toISOString()}`);
|
|
2139
|
+
}
|
|
2141
2140
|
records = await resource.list({
|
|
2142
|
-
filter: { updatedAt: { ">":
|
|
2141
|
+
filter: { updatedAt: { ">": sinceTimestamp.toISOString() } }
|
|
2143
2142
|
});
|
|
2144
2143
|
} else {
|
|
2145
2144
|
records = await resource.list();
|
|
@@ -2159,29 +2158,57 @@ class BackupPlugin extends Plugin {
|
|
|
2159
2158
|
}
|
|
2160
2159
|
return exportedFiles;
|
|
2161
2160
|
}
|
|
2162
|
-
async
|
|
2163
|
-
const
|
|
2164
|
-
|
|
2161
|
+
async _createArchive(files, targetPath, compressionType) {
|
|
2162
|
+
const archive = {
|
|
2163
|
+
version: "1.0",
|
|
2164
|
+
created: (/* @__PURE__ */ new Date()).toISOString(),
|
|
2165
|
+
files: []
|
|
2166
|
+
};
|
|
2165
2167
|
let totalSize = 0;
|
|
2166
|
-
|
|
2167
|
-
|
|
2168
|
-
|
|
2169
|
-
|
|
2170
|
-
|
|
2171
|
-
yield content;
|
|
2168
|
+
for (const filePath of files) {
|
|
2169
|
+
const [readOk, readErr, content] = await tryFn(() => promises.readFile(filePath, "utf8"));
|
|
2170
|
+
if (!readOk) {
|
|
2171
|
+
if (this.config.verbose) {
|
|
2172
|
+
console.warn(`[BackupPlugin] Failed to read ${filePath}: ${readErr?.message}`);
|
|
2172
2173
|
}
|
|
2173
|
-
|
|
2174
|
-
|
|
2175
|
-
|
|
2176
|
-
|
|
2174
|
+
continue;
|
|
2175
|
+
}
|
|
2176
|
+
const fileName = path.basename(filePath);
|
|
2177
|
+
totalSize += content.length;
|
|
2178
|
+
archive.files.push({
|
|
2179
|
+
name: fileName,
|
|
2180
|
+
size: content.length,
|
|
2181
|
+
content
|
|
2182
|
+
});
|
|
2183
|
+
}
|
|
2184
|
+
const archiveJson = JSON.stringify(archive);
|
|
2185
|
+
if (compressionType === "none") {
|
|
2186
|
+
await promises.writeFile(targetPath, archiveJson, "utf8");
|
|
2187
|
+
} else {
|
|
2188
|
+
const output = fs.createWriteStream(targetPath);
|
|
2189
|
+
const gzip = zlib.createGzip({ level: 6 });
|
|
2190
|
+
await promises$1.pipeline(
|
|
2191
|
+
async function* () {
|
|
2192
|
+
yield Buffer.from(archiveJson, "utf8");
|
|
2193
|
+
},
|
|
2194
|
+
gzip,
|
|
2195
|
+
output
|
|
2196
|
+
);
|
|
2197
|
+
}
|
|
2177
2198
|
const [statOk, , stats] = await tryFn(() => promises.stat(targetPath));
|
|
2178
2199
|
return statOk ? stats.size : totalSize;
|
|
2179
2200
|
}
|
|
2180
2201
|
async _generateChecksum(filePath) {
|
|
2181
|
-
const
|
|
2182
|
-
|
|
2183
|
-
|
|
2184
|
-
|
|
2202
|
+
const [ok, err, result] = await tryFn(async () => {
|
|
2203
|
+
const hash = crypto.createHash("sha256");
|
|
2204
|
+
const stream = fs.createReadStream(filePath);
|
|
2205
|
+
await promises$1.pipeline(stream, hash);
|
|
2206
|
+
return hash.digest("hex");
|
|
2207
|
+
});
|
|
2208
|
+
if (!ok) {
|
|
2209
|
+
throw new Error(`Failed to generate checksum for ${filePath}: ${err?.message}`);
|
|
2210
|
+
}
|
|
2211
|
+
return result;
|
|
2185
2212
|
}
|
|
2186
2213
|
async _cleanupTempFiles(tempDir) {
|
|
2187
2214
|
const [ok] = await tryFn(
|
|
@@ -2243,7 +2270,109 @@ class BackupPlugin extends Plugin {
|
|
|
2243
2270
|
}
|
|
2244
2271
|
async _restoreFromBackup(backupPath, options) {
|
|
2245
2272
|
const restoredResources = [];
|
|
2246
|
-
|
|
2273
|
+
try {
|
|
2274
|
+
let archiveData = "";
|
|
2275
|
+
if (this.config.compression !== "none") {
|
|
2276
|
+
const input = fs.createReadStream(backupPath);
|
|
2277
|
+
const gunzip = zlib.createGunzip();
|
|
2278
|
+
const chunks = [];
|
|
2279
|
+
await new Promise((resolve, reject) => {
|
|
2280
|
+
input.pipe(gunzip).on("data", (chunk) => chunks.push(chunk)).on("end", resolve).on("error", reject);
|
|
2281
|
+
});
|
|
2282
|
+
archiveData = Buffer.concat(chunks).toString("utf8");
|
|
2283
|
+
} else {
|
|
2284
|
+
archiveData = await promises.readFile(backupPath, "utf8");
|
|
2285
|
+
}
|
|
2286
|
+
let archive;
|
|
2287
|
+
try {
|
|
2288
|
+
archive = JSON.parse(archiveData);
|
|
2289
|
+
} catch (parseError) {
|
|
2290
|
+
throw new Error(`Failed to parse backup archive: ${parseError.message}`);
|
|
2291
|
+
}
|
|
2292
|
+
if (!archive || typeof archive !== "object") {
|
|
2293
|
+
throw new Error("Invalid backup archive: not a valid JSON object");
|
|
2294
|
+
}
|
|
2295
|
+
if (!archive.version || !archive.files) {
|
|
2296
|
+
throw new Error("Invalid backup archive format: missing version or files array");
|
|
2297
|
+
}
|
|
2298
|
+
if (this.config.verbose) {
|
|
2299
|
+
console.log(`[BackupPlugin] Restoring ${archive.files.length} files from backup`);
|
|
2300
|
+
}
|
|
2301
|
+
for (const file of archive.files) {
|
|
2302
|
+
try {
|
|
2303
|
+
const resourceData = JSON.parse(file.content);
|
|
2304
|
+
if (!resourceData.resourceName || !resourceData.definition) {
|
|
2305
|
+
if (this.config.verbose) {
|
|
2306
|
+
console.warn(`[BackupPlugin] Skipping invalid file: ${file.name}`);
|
|
2307
|
+
}
|
|
2308
|
+
continue;
|
|
2309
|
+
}
|
|
2310
|
+
const resourceName = resourceData.resourceName;
|
|
2311
|
+
if (options.resources && !options.resources.includes(resourceName)) {
|
|
2312
|
+
continue;
|
|
2313
|
+
}
|
|
2314
|
+
let resource = this.database.resources[resourceName];
|
|
2315
|
+
if (!resource) {
|
|
2316
|
+
if (this.config.verbose) {
|
|
2317
|
+
console.log(`[BackupPlugin] Creating resource '${resourceName}'`);
|
|
2318
|
+
}
|
|
2319
|
+
const [createOk, createErr] = await tryFn(
|
|
2320
|
+
() => this.database.createResource(resourceData.definition)
|
|
2321
|
+
);
|
|
2322
|
+
if (!createOk) {
|
|
2323
|
+
if (this.config.verbose) {
|
|
2324
|
+
console.warn(`[BackupPlugin] Failed to create resource '${resourceName}': ${createErr?.message}`);
|
|
2325
|
+
}
|
|
2326
|
+
continue;
|
|
2327
|
+
}
|
|
2328
|
+
resource = this.database.resources[resourceName];
|
|
2329
|
+
}
|
|
2330
|
+
if (resourceData.records && Array.isArray(resourceData.records)) {
|
|
2331
|
+
const mode = options.mode || "merge";
|
|
2332
|
+
if (mode === "replace") {
|
|
2333
|
+
const ids = await resource.listIds();
|
|
2334
|
+
for (const id of ids) {
|
|
2335
|
+
await resource.delete(id);
|
|
2336
|
+
}
|
|
2337
|
+
}
|
|
2338
|
+
let insertedCount = 0;
|
|
2339
|
+
for (const record of resourceData.records) {
|
|
2340
|
+
const [insertOk] = await tryFn(async () => {
|
|
2341
|
+
if (mode === "skip") {
|
|
2342
|
+
const existing = await resource.get(record.id);
|
|
2343
|
+
if (existing) {
|
|
2344
|
+
return false;
|
|
2345
|
+
}
|
|
2346
|
+
}
|
|
2347
|
+
await resource.insert(record);
|
|
2348
|
+
return true;
|
|
2349
|
+
});
|
|
2350
|
+
if (insertOk) {
|
|
2351
|
+
insertedCount++;
|
|
2352
|
+
}
|
|
2353
|
+
}
|
|
2354
|
+
restoredResources.push({
|
|
2355
|
+
name: resourceName,
|
|
2356
|
+
recordsRestored: insertedCount,
|
|
2357
|
+
totalRecords: resourceData.records.length
|
|
2358
|
+
});
|
|
2359
|
+
if (this.config.verbose) {
|
|
2360
|
+
console.log(`[BackupPlugin] Restored ${insertedCount}/${resourceData.records.length} records to '${resourceName}'`);
|
|
2361
|
+
}
|
|
2362
|
+
}
|
|
2363
|
+
} catch (fileError) {
|
|
2364
|
+
if (this.config.verbose) {
|
|
2365
|
+
console.warn(`[BackupPlugin] Error processing file ${file.name}: ${fileError.message}`);
|
|
2366
|
+
}
|
|
2367
|
+
}
|
|
2368
|
+
}
|
|
2369
|
+
return restoredResources;
|
|
2370
|
+
} catch (error) {
|
|
2371
|
+
if (this.config.verbose) {
|
|
2372
|
+
console.error(`[BackupPlugin] Error restoring backup: ${error.message}`);
|
|
2373
|
+
}
|
|
2374
|
+
throw new Error(`Failed to restore backup: ${error.message}`);
|
|
2375
|
+
}
|
|
2247
2376
|
}
|
|
2248
2377
|
/**
|
|
2249
2378
|
* List available backups
|
|
@@ -2287,6 +2416,90 @@ class BackupPlugin extends Plugin {
|
|
|
2287
2416
|
return ok ? backup : null;
|
|
2288
2417
|
}
|
|
2289
2418
|
async _cleanupOldBackups() {
|
|
2419
|
+
try {
|
|
2420
|
+
const [listOk, , allBackups] = await tryFn(
|
|
2421
|
+
() => this.database.resource(this.config.backupMetadataResource).list({
|
|
2422
|
+
filter: { status: "completed" },
|
|
2423
|
+
sort: { timestamp: -1 }
|
|
2424
|
+
})
|
|
2425
|
+
);
|
|
2426
|
+
if (!listOk || !allBackups || allBackups.length === 0) {
|
|
2427
|
+
return;
|
|
2428
|
+
}
|
|
2429
|
+
const now = Date.now();
|
|
2430
|
+
const msPerDay = 24 * 60 * 60 * 1e3;
|
|
2431
|
+
const msPerWeek = 7 * msPerDay;
|
|
2432
|
+
const msPerMonth = 30 * msPerDay;
|
|
2433
|
+
const msPerYear = 365 * msPerDay;
|
|
2434
|
+
const categorized = {
|
|
2435
|
+
daily: [],
|
|
2436
|
+
weekly: [],
|
|
2437
|
+
monthly: [],
|
|
2438
|
+
yearly: []
|
|
2439
|
+
};
|
|
2440
|
+
for (const backup of allBackups) {
|
|
2441
|
+
const age = now - backup.timestamp;
|
|
2442
|
+
if (age <= msPerDay * this.config.retention.daily) {
|
|
2443
|
+
categorized.daily.push(backup);
|
|
2444
|
+
} else if (age <= msPerWeek * this.config.retention.weekly) {
|
|
2445
|
+
categorized.weekly.push(backup);
|
|
2446
|
+
} else if (age <= msPerMonth * this.config.retention.monthly) {
|
|
2447
|
+
categorized.monthly.push(backup);
|
|
2448
|
+
} else if (age <= msPerYear * this.config.retention.yearly) {
|
|
2449
|
+
categorized.yearly.push(backup);
|
|
2450
|
+
}
|
|
2451
|
+
}
|
|
2452
|
+
const toKeep = /* @__PURE__ */ new Set();
|
|
2453
|
+
categorized.daily.forEach((b) => toKeep.add(b.id));
|
|
2454
|
+
const weeklyByWeek = /* @__PURE__ */ new Map();
|
|
2455
|
+
for (const backup of categorized.weekly) {
|
|
2456
|
+
const weekNum = Math.floor((now - backup.timestamp) / msPerWeek);
|
|
2457
|
+
if (!weeklyByWeek.has(weekNum)) {
|
|
2458
|
+
weeklyByWeek.set(weekNum, backup);
|
|
2459
|
+
toKeep.add(backup.id);
|
|
2460
|
+
}
|
|
2461
|
+
}
|
|
2462
|
+
const monthlyByMonth = /* @__PURE__ */ new Map();
|
|
2463
|
+
for (const backup of categorized.monthly) {
|
|
2464
|
+
const monthNum = Math.floor((now - backup.timestamp) / msPerMonth);
|
|
2465
|
+
if (!monthlyByMonth.has(monthNum)) {
|
|
2466
|
+
monthlyByMonth.set(monthNum, backup);
|
|
2467
|
+
toKeep.add(backup.id);
|
|
2468
|
+
}
|
|
2469
|
+
}
|
|
2470
|
+
const yearlyByYear = /* @__PURE__ */ new Map();
|
|
2471
|
+
for (const backup of categorized.yearly) {
|
|
2472
|
+
const yearNum = Math.floor((now - backup.timestamp) / msPerYear);
|
|
2473
|
+
if (!yearlyByYear.has(yearNum)) {
|
|
2474
|
+
yearlyByYear.set(yearNum, backup);
|
|
2475
|
+
toKeep.add(backup.id);
|
|
2476
|
+
}
|
|
2477
|
+
}
|
|
2478
|
+
const backupsToDelete = allBackups.filter((b) => !toKeep.has(b.id));
|
|
2479
|
+
if (backupsToDelete.length === 0) {
|
|
2480
|
+
return;
|
|
2481
|
+
}
|
|
2482
|
+
if (this.config.verbose) {
|
|
2483
|
+
console.log(`[BackupPlugin] Cleaning up ${backupsToDelete.length} old backups (keeping ${toKeep.size})`);
|
|
2484
|
+
}
|
|
2485
|
+
for (const backup of backupsToDelete) {
|
|
2486
|
+
try {
|
|
2487
|
+
await this.driver.delete(backup.id, backup.driverInfo);
|
|
2488
|
+
await this.database.resource(this.config.backupMetadataResource).delete(backup.id);
|
|
2489
|
+
if (this.config.verbose) {
|
|
2490
|
+
console.log(`[BackupPlugin] Deleted old backup: ${backup.id}`);
|
|
2491
|
+
}
|
|
2492
|
+
} catch (deleteError) {
|
|
2493
|
+
if (this.config.verbose) {
|
|
2494
|
+
console.warn(`[BackupPlugin] Failed to delete backup ${backup.id}: ${deleteError.message}`);
|
|
2495
|
+
}
|
|
2496
|
+
}
|
|
2497
|
+
}
|
|
2498
|
+
} catch (error) {
|
|
2499
|
+
if (this.config.verbose) {
|
|
2500
|
+
console.warn(`[BackupPlugin] Error during cleanup: ${error.message}`);
|
|
2501
|
+
}
|
|
2502
|
+
}
|
|
2290
2503
|
}
|
|
2291
2504
|
async _executeHook(hook, ...args) {
|
|
2292
2505
|
if (typeof hook === "function") {
|
|
@@ -3576,81 +3789,58 @@ class PartitionAwareFilesystemCache extends FilesystemCache {
|
|
|
3576
3789
|
class CachePlugin extends Plugin {
|
|
3577
3790
|
constructor(options = {}) {
|
|
3578
3791
|
super(options);
|
|
3579
|
-
this.
|
|
3580
|
-
|
|
3581
|
-
|
|
3582
|
-
|
|
3583
|
-
|
|
3584
|
-
|
|
3585
|
-
|
|
3586
|
-
|
|
3587
|
-
|
|
3588
|
-
|
|
3589
|
-
|
|
3590
|
-
|
|
3591
|
-
|
|
3592
|
-
|
|
3792
|
+
this.config = {
|
|
3793
|
+
// Driver configuration
|
|
3794
|
+
driver: options.driver || "s3",
|
|
3795
|
+
config: {
|
|
3796
|
+
ttl: options.ttl,
|
|
3797
|
+
maxSize: options.maxSize,
|
|
3798
|
+
...options.config
|
|
3799
|
+
// Driver-specific config (can override ttl/maxSize)
|
|
3800
|
+
},
|
|
3801
|
+
// Resource filtering
|
|
3802
|
+
include: options.include || null,
|
|
3803
|
+
// Array of resource names to cache (null = all)
|
|
3804
|
+
exclude: options.exclude || [],
|
|
3805
|
+
// Array of resource names to exclude
|
|
3806
|
+
// Partition settings
|
|
3807
|
+
includePartitions: options.includePartitions !== false,
|
|
3808
|
+
partitionStrategy: options.partitionStrategy || "hierarchical",
|
|
3809
|
+
partitionAware: options.partitionAware !== false,
|
|
3810
|
+
trackUsage: options.trackUsage !== false,
|
|
3811
|
+
preloadRelated: options.preloadRelated !== false,
|
|
3812
|
+
// Retry configuration
|
|
3813
|
+
retryAttempts: options.retryAttempts || 3,
|
|
3814
|
+
retryDelay: options.retryDelay || 100,
|
|
3815
|
+
// ms
|
|
3816
|
+
// Logging
|
|
3817
|
+
verbose: options.verbose || false
|
|
3593
3818
|
};
|
|
3594
3819
|
}
|
|
3595
3820
|
async setup(database) {
|
|
3596
3821
|
await super.setup(database);
|
|
3597
3822
|
}
|
|
3598
3823
|
async onSetup() {
|
|
3599
|
-
if (this.
|
|
3600
|
-
this.driver = this.
|
|
3601
|
-
} else if (this.
|
|
3602
|
-
|
|
3603
|
-
|
|
3604
|
-
|
|
3605
|
-
...this.config
|
|
3606
|
-
// New config format (medium priority)
|
|
3607
|
-
};
|
|
3608
|
-
if (this.ttl !== void 0) {
|
|
3609
|
-
driverConfig.ttl = this.ttl;
|
|
3610
|
-
}
|
|
3611
|
-
if (this.maxSize !== void 0) {
|
|
3612
|
-
driverConfig.maxSize = this.maxSize;
|
|
3613
|
-
}
|
|
3614
|
-
this.driver = new MemoryCache(driverConfig);
|
|
3615
|
-
} else if (this.driverName === "filesystem") {
|
|
3616
|
-
const driverConfig = {
|
|
3617
|
-
...this.legacyConfig.filesystemOptions,
|
|
3618
|
-
// Legacy support (lowest priority)
|
|
3619
|
-
...this.config
|
|
3620
|
-
// New config format (medium priority)
|
|
3621
|
-
};
|
|
3622
|
-
if (this.ttl !== void 0) {
|
|
3623
|
-
driverConfig.ttl = this.ttl;
|
|
3624
|
-
}
|
|
3625
|
-
if (this.maxSize !== void 0) {
|
|
3626
|
-
driverConfig.maxSize = this.maxSize;
|
|
3627
|
-
}
|
|
3628
|
-
if (this.partitionAware) {
|
|
3824
|
+
if (this.config.driver && typeof this.config.driver === "object") {
|
|
3825
|
+
this.driver = this.config.driver;
|
|
3826
|
+
} else if (this.config.driver === "memory") {
|
|
3827
|
+
this.driver = new MemoryCache(this.config.config);
|
|
3828
|
+
} else if (this.config.driver === "filesystem") {
|
|
3829
|
+
if (this.config.partitionAware) {
|
|
3629
3830
|
this.driver = new PartitionAwareFilesystemCache({
|
|
3630
|
-
partitionStrategy: this.partitionStrategy,
|
|
3631
|
-
trackUsage: this.trackUsage,
|
|
3632
|
-
preloadRelated: this.preloadRelated,
|
|
3633
|
-
...
|
|
3831
|
+
partitionStrategy: this.config.partitionStrategy,
|
|
3832
|
+
trackUsage: this.config.trackUsage,
|
|
3833
|
+
preloadRelated: this.config.preloadRelated,
|
|
3834
|
+
...this.config.config
|
|
3634
3835
|
});
|
|
3635
3836
|
} else {
|
|
3636
|
-
this.driver = new FilesystemCache(
|
|
3837
|
+
this.driver = new FilesystemCache(this.config.config);
|
|
3637
3838
|
}
|
|
3638
3839
|
} else {
|
|
3639
|
-
|
|
3840
|
+
this.driver = new S3Cache({
|
|
3640
3841
|
client: this.database.client,
|
|
3641
|
-
|
|
3642
|
-
|
|
3643
|
-
// Legacy support (lowest priority)
|
|
3644
|
-
...this.config
|
|
3645
|
-
// New config format (medium priority)
|
|
3646
|
-
};
|
|
3647
|
-
if (this.ttl !== void 0) {
|
|
3648
|
-
driverConfig.ttl = this.ttl;
|
|
3649
|
-
}
|
|
3650
|
-
if (this.maxSize !== void 0) {
|
|
3651
|
-
driverConfig.maxSize = this.maxSize;
|
|
3652
|
-
}
|
|
3653
|
-
this.driver = new S3Cache(driverConfig);
|
|
3842
|
+
...this.config.config
|
|
3843
|
+
});
|
|
3654
3844
|
}
|
|
3655
3845
|
this.installDatabaseHooks();
|
|
3656
3846
|
this.installResourceHooks();
|
|
@@ -3660,7 +3850,9 @@ class CachePlugin extends Plugin {
|
|
|
3660
3850
|
*/
|
|
3661
3851
|
installDatabaseHooks() {
|
|
3662
3852
|
this.database.addHook("afterCreateResource", async ({ resource }) => {
|
|
3663
|
-
this.
|
|
3853
|
+
if (this.shouldCacheResource(resource.name)) {
|
|
3854
|
+
this.installResourceHooksForResource(resource);
|
|
3855
|
+
}
|
|
3664
3856
|
});
|
|
3665
3857
|
}
|
|
3666
3858
|
async onStart() {
|
|
@@ -3670,9 +3862,24 @@ class CachePlugin extends Plugin {
|
|
|
3670
3862
|
// Remove the old installDatabaseProxy method
|
|
3671
3863
|
installResourceHooks() {
|
|
3672
3864
|
for (const resource of Object.values(this.database.resources)) {
|
|
3865
|
+
if (!this.shouldCacheResource(resource.name)) {
|
|
3866
|
+
continue;
|
|
3867
|
+
}
|
|
3673
3868
|
this.installResourceHooksForResource(resource);
|
|
3674
3869
|
}
|
|
3675
3870
|
}
|
|
3871
|
+
shouldCacheResource(resourceName) {
|
|
3872
|
+
if (resourceName.startsWith("plg_") && !this.config.include) {
|
|
3873
|
+
return false;
|
|
3874
|
+
}
|
|
3875
|
+
if (this.config.exclude.includes(resourceName)) {
|
|
3876
|
+
return false;
|
|
3877
|
+
}
|
|
3878
|
+
if (this.config.include && !this.config.include.includes(resourceName)) {
|
|
3879
|
+
return false;
|
|
3880
|
+
}
|
|
3881
|
+
return true;
|
|
3882
|
+
}
|
|
3676
3883
|
installResourceHooksForResource(resource) {
|
|
3677
3884
|
if (!this.driver) return;
|
|
3678
3885
|
Object.defineProperty(resource, "cache", {
|
|
@@ -3821,37 +4028,74 @@ class CachePlugin extends Plugin {
|
|
|
3821
4028
|
if (data && data.id) {
|
|
3822
4029
|
const itemSpecificMethods = ["get", "exists", "content", "hasContent"];
|
|
3823
4030
|
for (const method of itemSpecificMethods) {
|
|
3824
|
-
|
|
3825
|
-
|
|
3826
|
-
|
|
3827
|
-
|
|
4031
|
+
const specificKey = await this.generateCacheKey(resource, method, { id: data.id });
|
|
4032
|
+
const [ok2, err2] = await this.clearCacheWithRetry(resource.cache, specificKey);
|
|
4033
|
+
if (!ok2) {
|
|
4034
|
+
this.emit("cache_clear_error", {
|
|
4035
|
+
resource: resource.name,
|
|
4036
|
+
method,
|
|
4037
|
+
id: data.id,
|
|
4038
|
+
error: err2.message
|
|
4039
|
+
});
|
|
4040
|
+
if (this.config.verbose) {
|
|
4041
|
+
console.warn(`[CachePlugin] Failed to clear ${method} cache for ${resource.name}:${data.id}:`, err2.message);
|
|
4042
|
+
}
|
|
3828
4043
|
}
|
|
3829
4044
|
}
|
|
3830
4045
|
if (this.config.includePartitions === true && resource.config?.partitions && Object.keys(resource.config.partitions).length > 0) {
|
|
3831
4046
|
const partitionValues = this.getPartitionValues(data, resource);
|
|
3832
4047
|
for (const [partitionName, values] of Object.entries(partitionValues)) {
|
|
3833
4048
|
if (values && Object.keys(values).length > 0 && Object.values(values).some((v) => v !== null && v !== void 0)) {
|
|
3834
|
-
|
|
3835
|
-
|
|
3836
|
-
|
|
3837
|
-
|
|
4049
|
+
const partitionKeyPrefix = path.join(keyPrefix, `partition=${partitionName}`);
|
|
4050
|
+
const [ok2, err2] = await this.clearCacheWithRetry(resource.cache, partitionKeyPrefix);
|
|
4051
|
+
if (!ok2) {
|
|
4052
|
+
this.emit("cache_clear_error", {
|
|
4053
|
+
resource: resource.name,
|
|
4054
|
+
partition: partitionName,
|
|
4055
|
+
error: err2.message
|
|
4056
|
+
});
|
|
4057
|
+
if (this.config.verbose) {
|
|
4058
|
+
console.warn(`[CachePlugin] Failed to clear partition cache for ${resource.name}/${partitionName}:`, err2.message);
|
|
4059
|
+
}
|
|
3838
4060
|
}
|
|
3839
4061
|
}
|
|
3840
4062
|
}
|
|
3841
4063
|
}
|
|
3842
4064
|
}
|
|
3843
|
-
|
|
3844
|
-
|
|
3845
|
-
|
|
4065
|
+
const [ok, err] = await this.clearCacheWithRetry(resource.cache, keyPrefix);
|
|
4066
|
+
if (!ok) {
|
|
4067
|
+
this.emit("cache_clear_error", {
|
|
4068
|
+
resource: resource.name,
|
|
4069
|
+
type: "broad",
|
|
4070
|
+
error: err.message
|
|
4071
|
+
});
|
|
4072
|
+
if (this.config.verbose) {
|
|
4073
|
+
console.warn(`[CachePlugin] Failed to clear broad cache for ${resource.name}, trying specific methods:`, err.message);
|
|
4074
|
+
}
|
|
3846
4075
|
const aggregateMethods = ["count", "list", "listIds", "getAll", "page", "query"];
|
|
3847
4076
|
for (const method of aggregateMethods) {
|
|
3848
|
-
|
|
3849
|
-
|
|
3850
|
-
|
|
3851
|
-
|
|
3852
|
-
|
|
4077
|
+
await this.clearCacheWithRetry(resource.cache, `${keyPrefix}/action=${method}`);
|
|
4078
|
+
await this.clearCacheWithRetry(resource.cache, `resource=${resource.name}/action=${method}`);
|
|
4079
|
+
}
|
|
4080
|
+
}
|
|
4081
|
+
}
|
|
4082
|
+
async clearCacheWithRetry(cache, key) {
|
|
4083
|
+
let lastError;
|
|
4084
|
+
for (let attempt = 0; attempt < this.config.retryAttempts; attempt++) {
|
|
4085
|
+
const [ok, err] = await tryFn(() => cache.clear(key));
|
|
4086
|
+
if (ok) {
|
|
4087
|
+
return [true, null];
|
|
4088
|
+
}
|
|
4089
|
+
lastError = err;
|
|
4090
|
+
if (err.name === "NoSuchKey" || err.code === "NoSuchKey") {
|
|
4091
|
+
return [true, null];
|
|
4092
|
+
}
|
|
4093
|
+
if (attempt < this.config.retryAttempts - 1) {
|
|
4094
|
+
const delay = this.config.retryDelay * Math.pow(2, attempt);
|
|
4095
|
+
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
3853
4096
|
}
|
|
3854
4097
|
}
|
|
4098
|
+
return [false, lastError];
|
|
3855
4099
|
}
|
|
3856
4100
|
async generateCacheKey(resource, action, params = {}, partition = null, partitionValues = null) {
|
|
3857
4101
|
const keyParts = [
|
|
@@ -3867,14 +4111,14 @@ class CachePlugin extends Plugin {
|
|
|
3867
4111
|
}
|
|
3868
4112
|
}
|
|
3869
4113
|
if (Object.keys(params).length > 0) {
|
|
3870
|
-
const paramsHash =
|
|
4114
|
+
const paramsHash = this.hashParams(params);
|
|
3871
4115
|
keyParts.push(paramsHash);
|
|
3872
4116
|
}
|
|
3873
4117
|
return path.join(...keyParts) + ".json.gz";
|
|
3874
4118
|
}
|
|
3875
|
-
|
|
3876
|
-
const
|
|
3877
|
-
return
|
|
4119
|
+
hashParams(params) {
|
|
4120
|
+
const serialized = jsonStableStringify(params) || "empty";
|
|
4121
|
+
return crypto.createHash("md5").update(serialized).digest("hex").substring(0, 16);
|
|
3878
4122
|
}
|
|
3879
4123
|
// Utility methods
|
|
3880
4124
|
async getCacheStats() {
|
|
@@ -3899,50 +4143,48 @@ class CachePlugin extends Plugin {
|
|
|
3899
4143
|
if (!resource) {
|
|
3900
4144
|
throw new Error(`Resource '${resourceName}' not found`);
|
|
3901
4145
|
}
|
|
3902
|
-
const { includePartitions = true } = options;
|
|
4146
|
+
const { includePartitions = true, sampleSize = 100 } = options;
|
|
3903
4147
|
if (this.driver instanceof PartitionAwareFilesystemCache && resource.warmPartitionCache) {
|
|
3904
4148
|
const partitionNames = resource.config.partitions ? Object.keys(resource.config.partitions) : [];
|
|
3905
4149
|
return await resource.warmPartitionCache(partitionNames, options);
|
|
3906
4150
|
}
|
|
3907
|
-
|
|
3908
|
-
|
|
4151
|
+
let offset = 0;
|
|
4152
|
+
const pageSize = 100;
|
|
4153
|
+
const sampledRecords = [];
|
|
4154
|
+
while (sampledRecords.length < sampleSize) {
|
|
4155
|
+
const [ok, err, pageResult] = await tryFn(() => resource.page({ offset, size: pageSize }));
|
|
4156
|
+
if (!ok || !pageResult) {
|
|
4157
|
+
break;
|
|
4158
|
+
}
|
|
4159
|
+
const pageItems = Array.isArray(pageResult) ? pageResult : pageResult.items || [];
|
|
4160
|
+
if (pageItems.length === 0) {
|
|
4161
|
+
break;
|
|
4162
|
+
}
|
|
4163
|
+
sampledRecords.push(...pageItems);
|
|
4164
|
+
offset += pageSize;
|
|
4165
|
+
}
|
|
4166
|
+
if (includePartitions && resource.config.partitions && sampledRecords.length > 0) {
|
|
3909
4167
|
for (const [partitionName, partitionDef] of Object.entries(resource.config.partitions)) {
|
|
3910
4168
|
if (partitionDef.fields) {
|
|
3911
|
-
const
|
|
3912
|
-
const
|
|
3913
|
-
const partitionValues = /* @__PURE__ */ new Set();
|
|
3914
|
-
for (const record of recordsArray.slice(0, 10)) {
|
|
4169
|
+
const partitionValuesSet = /* @__PURE__ */ new Set();
|
|
4170
|
+
for (const record of sampledRecords) {
|
|
3915
4171
|
const values = this.getPartitionValues(record, resource);
|
|
3916
4172
|
if (values[partitionName]) {
|
|
3917
|
-
|
|
4173
|
+
partitionValuesSet.add(JSON.stringify(values[partitionName]));
|
|
3918
4174
|
}
|
|
3919
4175
|
}
|
|
3920
|
-
for (const partitionValueStr of
|
|
3921
|
-
const
|
|
3922
|
-
await resource.list({ partition: partitionName, partitionValues
|
|
4176
|
+
for (const partitionValueStr of partitionValuesSet) {
|
|
4177
|
+
const partitionValues = JSON.parse(partitionValueStr);
|
|
4178
|
+
await tryFn(() => resource.list({ partition: partitionName, partitionValues }));
|
|
3923
4179
|
}
|
|
3924
4180
|
}
|
|
3925
4181
|
}
|
|
3926
4182
|
}
|
|
3927
|
-
|
|
3928
|
-
|
|
3929
|
-
|
|
3930
|
-
|
|
3931
|
-
|
|
3932
|
-
}
|
|
3933
|
-
return await this.driver.getPartitionStats(resourceName, partition);
|
|
3934
|
-
}
|
|
3935
|
-
async getCacheRecommendations(resourceName) {
|
|
3936
|
-
if (!(this.driver instanceof PartitionAwareFilesystemCache)) {
|
|
3937
|
-
throw new Error("Cache recommendations are only available with PartitionAwareFilesystemCache");
|
|
3938
|
-
}
|
|
3939
|
-
return await this.driver.getCacheRecommendations(resourceName);
|
|
3940
|
-
}
|
|
3941
|
-
async clearPartitionCache(resourceName, partition, partitionValues = {}) {
|
|
3942
|
-
if (!(this.driver instanceof PartitionAwareFilesystemCache)) {
|
|
3943
|
-
throw new Error("Partition cache clearing is only available with PartitionAwareFilesystemCache");
|
|
3944
|
-
}
|
|
3945
|
-
return await this.driver.clearPartition(resourceName, partition, partitionValues);
|
|
4183
|
+
return {
|
|
4184
|
+
resourceName,
|
|
4185
|
+
recordsSampled: sampledRecords.length,
|
|
4186
|
+
partitionsWarmed: includePartitions && resource.config.partitions ? Object.keys(resource.config.partitions).length : 0
|
|
4187
|
+
};
|
|
3946
4188
|
}
|
|
3947
4189
|
async analyzeCacheUsage() {
|
|
3948
4190
|
if (!(this.driver instanceof PartitionAwareFilesystemCache)) {
|
|
@@ -3959,6 +4201,9 @@ class CachePlugin extends Plugin {
|
|
|
3959
4201
|
}
|
|
3960
4202
|
};
|
|
3961
4203
|
for (const [resourceName, resource] of Object.entries(this.database.resources)) {
|
|
4204
|
+
if (!this.shouldCacheResource(resourceName)) {
|
|
4205
|
+
continue;
|
|
4206
|
+
}
|
|
3962
4207
|
try {
|
|
3963
4208
|
analysis.resourceStats[resourceName] = await this.driver.getPartitionStats(resourceName);
|
|
3964
4209
|
analysis.recommendations[resourceName] = await this.driver.getCacheRecommendations(resourceName);
|
|
@@ -4050,62 +4295,807 @@ const CostsPlugin = {
|
|
|
4050
4295
|
}
|
|
4051
4296
|
};
|
|
4052
4297
|
|
|
4053
|
-
class
|
|
4298
|
+
class EventualConsistencyPlugin extends Plugin {
|
|
4054
4299
|
constructor(options = {}) {
|
|
4055
|
-
super();
|
|
4056
|
-
|
|
4300
|
+
super(options);
|
|
4301
|
+
if (!options.resource) {
|
|
4302
|
+
throw new Error("EventualConsistencyPlugin requires 'resource' option");
|
|
4303
|
+
}
|
|
4304
|
+
if (!options.field) {
|
|
4305
|
+
throw new Error("EventualConsistencyPlugin requires 'field' option");
|
|
4306
|
+
}
|
|
4307
|
+
const detectedTimezone = this._detectTimezone();
|
|
4057
4308
|
this.config = {
|
|
4058
|
-
|
|
4059
|
-
|
|
4060
|
-
|
|
4309
|
+
resource: options.resource,
|
|
4310
|
+
field: options.field,
|
|
4311
|
+
cohort: {
|
|
4312
|
+
timezone: options.cohort?.timezone || detectedTimezone
|
|
4313
|
+
},
|
|
4314
|
+
reducer: options.reducer || ((transactions) => {
|
|
4315
|
+
let baseValue = 0;
|
|
4316
|
+
for (const t of transactions) {
|
|
4317
|
+
if (t.operation === "set") {
|
|
4318
|
+
baseValue = t.value;
|
|
4319
|
+
} else if (t.operation === "add") {
|
|
4320
|
+
baseValue += t.value;
|
|
4321
|
+
} else if (t.operation === "sub") {
|
|
4322
|
+
baseValue -= t.value;
|
|
4323
|
+
}
|
|
4324
|
+
}
|
|
4325
|
+
return baseValue;
|
|
4326
|
+
}),
|
|
4327
|
+
consolidationInterval: options.consolidationInterval ?? 300,
|
|
4328
|
+
// 5 minutes (in seconds)
|
|
4329
|
+
consolidationConcurrency: options.consolidationConcurrency || 5,
|
|
4330
|
+
consolidationWindow: options.consolidationWindow || 24,
|
|
4331
|
+
// Hours to look back for pending transactions (watermark)
|
|
4332
|
+
autoConsolidate: options.autoConsolidate !== false,
|
|
4333
|
+
lateArrivalStrategy: options.lateArrivalStrategy || "warn",
|
|
4334
|
+
// 'ignore', 'warn', 'process'
|
|
4335
|
+
batchTransactions: options.batchTransactions || false,
|
|
4336
|
+
// CAUTION: Not safe in distributed environments! Loses data on container crash
|
|
4337
|
+
batchSize: options.batchSize || 100,
|
|
4338
|
+
mode: options.mode || "async",
|
|
4339
|
+
// 'async' or 'sync'
|
|
4340
|
+
lockTimeout: options.lockTimeout || 300,
|
|
4341
|
+
// 5 minutes (in seconds, configurable)
|
|
4342
|
+
transactionRetention: options.transactionRetention || 30,
|
|
4343
|
+
// Days to keep applied transactions
|
|
4344
|
+
gcInterval: options.gcInterval || 86400,
|
|
4345
|
+
// 24 hours (in seconds)
|
|
4346
|
+
verbose: options.verbose || false
|
|
4061
4347
|
};
|
|
4062
|
-
this.
|
|
4063
|
-
|
|
4064
|
-
|
|
4065
|
-
this.
|
|
4066
|
-
|
|
4067
|
-
|
|
4068
|
-
|
|
4069
|
-
|
|
4070
|
-
|
|
4071
|
-
|
|
4072
|
-
|
|
4073
|
-
|
|
4074
|
-
|
|
4075
|
-
|
|
4076
|
-
|
|
4077
|
-
}
|
|
4078
|
-
}));
|
|
4079
|
-
this.indexResource = ok ? indexResource : database.resources.fulltext_indexes;
|
|
4080
|
-
await this.loadIndexes();
|
|
4081
|
-
this.installDatabaseHooks();
|
|
4082
|
-
this.installIndexingHooks();
|
|
4083
|
-
}
|
|
4084
|
-
async start() {
|
|
4348
|
+
this.transactionResource = null;
|
|
4349
|
+
this.targetResource = null;
|
|
4350
|
+
this.consolidationTimer = null;
|
|
4351
|
+
this.gcTimer = null;
|
|
4352
|
+
this.pendingTransactions = /* @__PURE__ */ new Map();
|
|
4353
|
+
if (this.config.batchTransactions && !this.config.verbose) {
|
|
4354
|
+
console.warn(
|
|
4355
|
+
`[EventualConsistency] WARNING: batchTransactions is enabled. This stores transactions in memory and will lose data if container crashes. Not recommended for distributed/production environments. Set verbose: true to suppress this warning.`
|
|
4356
|
+
);
|
|
4357
|
+
}
|
|
4358
|
+
if (this.config.verbose && !options.cohort?.timezone) {
|
|
4359
|
+
console.log(
|
|
4360
|
+
`[EventualConsistency] Auto-detected timezone: ${this.config.cohort.timezone} (from ${process.env.TZ ? "TZ env var" : "system Intl API"})`
|
|
4361
|
+
);
|
|
4362
|
+
}
|
|
4085
4363
|
}
|
|
4086
|
-
async
|
|
4087
|
-
|
|
4088
|
-
this.
|
|
4364
|
+
async onSetup() {
|
|
4365
|
+
this.targetResource = this.database.resources[this.config.resource];
|
|
4366
|
+
if (!this.targetResource) {
|
|
4367
|
+
this.deferredSetup = true;
|
|
4368
|
+
this.watchForResource();
|
|
4369
|
+
return;
|
|
4370
|
+
}
|
|
4371
|
+
await this.completeSetup();
|
|
4089
4372
|
}
|
|
4090
|
-
|
|
4091
|
-
|
|
4092
|
-
|
|
4093
|
-
|
|
4094
|
-
|
|
4095
|
-
|
|
4096
|
-
this.indexes.set(key, {
|
|
4097
|
-
recordIds: indexRecord.recordIds || [],
|
|
4098
|
-
count: indexRecord.count || 0
|
|
4099
|
-
});
|
|
4373
|
+
watchForResource() {
|
|
4374
|
+
const hookCallback = async ({ resource, config }) => {
|
|
4375
|
+
if (config.name === this.config.resource && this.deferredSetup) {
|
|
4376
|
+
this.targetResource = resource;
|
|
4377
|
+
this.deferredSetup = false;
|
|
4378
|
+
await this.completeSetup();
|
|
4100
4379
|
}
|
|
4380
|
+
};
|
|
4381
|
+
this.database.addHook("afterCreateResource", hookCallback);
|
|
4382
|
+
}
|
|
4383
|
+
async completeSetup() {
|
|
4384
|
+
if (!this.targetResource) return;
|
|
4385
|
+
const transactionResourceName = `${this.config.resource}_transactions_${this.config.field}`;
|
|
4386
|
+
const partitionConfig = this.createPartitionConfig();
|
|
4387
|
+
const [ok, err, transactionResource] = await tryFn(
|
|
4388
|
+
() => this.database.createResource({
|
|
4389
|
+
name: transactionResourceName,
|
|
4390
|
+
attributes: {
|
|
4391
|
+
id: "string|required",
|
|
4392
|
+
originalId: "string|required",
|
|
4393
|
+
field: "string|required",
|
|
4394
|
+
value: "number|required",
|
|
4395
|
+
operation: "string|required",
|
|
4396
|
+
// 'set', 'add', or 'sub'
|
|
4397
|
+
timestamp: "string|required",
|
|
4398
|
+
cohortDate: "string|required",
|
|
4399
|
+
// For daily partitioning
|
|
4400
|
+
cohortHour: "string|required",
|
|
4401
|
+
// For hourly partitioning
|
|
4402
|
+
cohortMonth: "string|optional",
|
|
4403
|
+
// For monthly partitioning
|
|
4404
|
+
source: "string|optional",
|
|
4405
|
+
applied: "boolean|optional"
|
|
4406
|
+
// Track if transaction was applied
|
|
4407
|
+
},
|
|
4408
|
+
behavior: "body-overflow",
|
|
4409
|
+
timestamps: true,
|
|
4410
|
+
partitions: partitionConfig,
|
|
4411
|
+
asyncPartitions: true
|
|
4412
|
+
// Use async partitions for better performance
|
|
4413
|
+
})
|
|
4414
|
+
);
|
|
4415
|
+
if (!ok && !this.database.resources[transactionResourceName]) {
|
|
4416
|
+
throw new Error(`Failed to create transaction resource: ${err?.message}`);
|
|
4417
|
+
}
|
|
4418
|
+
this.transactionResource = ok ? transactionResource : this.database.resources[transactionResourceName];
|
|
4419
|
+
const lockResourceName = `${this.config.resource}_consolidation_locks_${this.config.field}`;
|
|
4420
|
+
const [lockOk, lockErr, lockResource] = await tryFn(
|
|
4421
|
+
() => this.database.createResource({
|
|
4422
|
+
name: lockResourceName,
|
|
4423
|
+
attributes: {
|
|
4424
|
+
id: "string|required",
|
|
4425
|
+
lockedAt: "number|required",
|
|
4426
|
+
workerId: "string|optional"
|
|
4427
|
+
},
|
|
4428
|
+
behavior: "body-only",
|
|
4429
|
+
timestamps: false
|
|
4430
|
+
})
|
|
4431
|
+
);
|
|
4432
|
+
if (!lockOk && !this.database.resources[lockResourceName]) {
|
|
4433
|
+
throw new Error(`Failed to create lock resource: ${lockErr?.message}`);
|
|
4434
|
+
}
|
|
4435
|
+
this.lockResource = lockOk ? lockResource : this.database.resources[lockResourceName];
|
|
4436
|
+
this.addHelperMethods();
|
|
4437
|
+
if (this.config.autoConsolidate) {
|
|
4438
|
+
this.startConsolidationTimer();
|
|
4101
4439
|
}
|
|
4440
|
+
this.startGarbageCollectionTimer();
|
|
4102
4441
|
}
|
|
4103
|
-
async
|
|
4104
|
-
if (
|
|
4105
|
-
|
|
4106
|
-
|
|
4107
|
-
|
|
4108
|
-
|
|
4442
|
+
async onStart() {
|
|
4443
|
+
if (this.deferredSetup) {
|
|
4444
|
+
return;
|
|
4445
|
+
}
|
|
4446
|
+
this.emit("eventual-consistency.started", {
|
|
4447
|
+
resource: this.config.resource,
|
|
4448
|
+
field: this.config.field,
|
|
4449
|
+
cohort: this.config.cohort
|
|
4450
|
+
});
|
|
4451
|
+
}
|
|
4452
|
+
async onStop() {
|
|
4453
|
+
if (this.consolidationTimer) {
|
|
4454
|
+
clearInterval(this.consolidationTimer);
|
|
4455
|
+
this.consolidationTimer = null;
|
|
4456
|
+
}
|
|
4457
|
+
if (this.gcTimer) {
|
|
4458
|
+
clearInterval(this.gcTimer);
|
|
4459
|
+
this.gcTimer = null;
|
|
4460
|
+
}
|
|
4461
|
+
await this.flushPendingTransactions();
|
|
4462
|
+
this.emit("eventual-consistency.stopped", {
|
|
4463
|
+
resource: this.config.resource,
|
|
4464
|
+
field: this.config.field
|
|
4465
|
+
});
|
|
4466
|
+
}
|
|
4467
|
+
createPartitionConfig() {
|
|
4468
|
+
const partitions = {
|
|
4469
|
+
byHour: {
|
|
4470
|
+
fields: {
|
|
4471
|
+
cohortHour: "string"
|
|
4472
|
+
}
|
|
4473
|
+
},
|
|
4474
|
+
byDay: {
|
|
4475
|
+
fields: {
|
|
4476
|
+
cohortDate: "string"
|
|
4477
|
+
}
|
|
4478
|
+
},
|
|
4479
|
+
byMonth: {
|
|
4480
|
+
fields: {
|
|
4481
|
+
cohortMonth: "string"
|
|
4482
|
+
}
|
|
4483
|
+
}
|
|
4484
|
+
};
|
|
4485
|
+
return partitions;
|
|
4486
|
+
}
|
|
4487
|
+
/**
|
|
4488
|
+
* Auto-detect timezone from environment or system
|
|
4489
|
+
* @private
|
|
4490
|
+
*/
|
|
4491
|
+
_detectTimezone() {
|
|
4492
|
+
if (process.env.TZ) {
|
|
4493
|
+
return process.env.TZ;
|
|
4494
|
+
}
|
|
4495
|
+
try {
|
|
4496
|
+
const systemTimezone = Intl.DateTimeFormat().resolvedOptions().timeZone;
|
|
4497
|
+
if (systemTimezone) {
|
|
4498
|
+
return systemTimezone;
|
|
4499
|
+
}
|
|
4500
|
+
} catch (err) {
|
|
4501
|
+
}
|
|
4502
|
+
return "UTC";
|
|
4503
|
+
}
|
|
4504
|
+
/**
|
|
4505
|
+
* Helper method to resolve field and plugin from arguments
|
|
4506
|
+
* Supports both single-field (field, value) and multi-field (field, value) signatures
|
|
4507
|
+
* @private
|
|
4508
|
+
*/
|
|
4509
|
+
_resolveFieldAndPlugin(resource, fieldOrValue, value) {
|
|
4510
|
+
const hasMultipleFields = Object.keys(resource._eventualConsistencyPlugins).length > 1;
|
|
4511
|
+
if (hasMultipleFields && value === void 0) {
|
|
4512
|
+
throw new Error(`Multiple fields have eventual consistency. Please specify the field explicitly.`);
|
|
4513
|
+
}
|
|
4514
|
+
const field = value !== void 0 ? fieldOrValue : this.config.field;
|
|
4515
|
+
const actualValue = value !== void 0 ? value : fieldOrValue;
|
|
4516
|
+
const fieldPlugin = resource._eventualConsistencyPlugins[field];
|
|
4517
|
+
if (!fieldPlugin) {
|
|
4518
|
+
throw new Error(`No eventual consistency plugin found for field "${field}"`);
|
|
4519
|
+
}
|
|
4520
|
+
return { field, value: actualValue, plugin: fieldPlugin };
|
|
4521
|
+
}
|
|
4522
|
+
/**
|
|
4523
|
+
* Helper method to perform atomic consolidation in sync mode
|
|
4524
|
+
* @private
|
|
4525
|
+
*/
|
|
4526
|
+
async _syncModeConsolidate(id, field) {
|
|
4527
|
+
const consolidatedValue = await this.consolidateRecord(id);
|
|
4528
|
+
await this.targetResource.update(id, {
|
|
4529
|
+
[field]: consolidatedValue
|
|
4530
|
+
});
|
|
4531
|
+
return consolidatedValue;
|
|
4532
|
+
}
|
|
4533
|
+
/**
|
|
4534
|
+
* Create synthetic 'set' transaction from current value
|
|
4535
|
+
* @private
|
|
4536
|
+
*/
|
|
4537
|
+
_createSyntheticSetTransaction(currentValue) {
|
|
4538
|
+
return {
|
|
4539
|
+
id: "__synthetic__",
|
|
4540
|
+
operation: "set",
|
|
4541
|
+
value: currentValue,
|
|
4542
|
+
timestamp: (/* @__PURE__ */ new Date(0)).toISOString(),
|
|
4543
|
+
synthetic: true
|
|
4544
|
+
};
|
|
4545
|
+
}
|
|
4546
|
+
addHelperMethods() {
|
|
4547
|
+
const resource = this.targetResource;
|
|
4548
|
+
const defaultField = this.config.field;
|
|
4549
|
+
const plugin = this;
|
|
4550
|
+
if (!resource._eventualConsistencyPlugins) {
|
|
4551
|
+
resource._eventualConsistencyPlugins = {};
|
|
4552
|
+
}
|
|
4553
|
+
resource._eventualConsistencyPlugins[defaultField] = plugin;
|
|
4554
|
+
resource.set = async (id, fieldOrValue, value) => {
|
|
4555
|
+
const { field, value: actualValue, plugin: fieldPlugin } = plugin._resolveFieldAndPlugin(resource, fieldOrValue, value);
|
|
4556
|
+
await fieldPlugin.createTransaction({
|
|
4557
|
+
originalId: id,
|
|
4558
|
+
operation: "set",
|
|
4559
|
+
value: actualValue,
|
|
4560
|
+
source: "set"
|
|
4561
|
+
});
|
|
4562
|
+
if (fieldPlugin.config.mode === "sync") {
|
|
4563
|
+
return await fieldPlugin._syncModeConsolidate(id, field);
|
|
4564
|
+
}
|
|
4565
|
+
return actualValue;
|
|
4566
|
+
};
|
|
4567
|
+
resource.add = async (id, fieldOrAmount, amount) => {
|
|
4568
|
+
const { field, value: actualAmount, plugin: fieldPlugin } = plugin._resolveFieldAndPlugin(resource, fieldOrAmount, amount);
|
|
4569
|
+
await fieldPlugin.createTransaction({
|
|
4570
|
+
originalId: id,
|
|
4571
|
+
operation: "add",
|
|
4572
|
+
value: actualAmount,
|
|
4573
|
+
source: "add"
|
|
4574
|
+
});
|
|
4575
|
+
if (fieldPlugin.config.mode === "sync") {
|
|
4576
|
+
return await fieldPlugin._syncModeConsolidate(id, field);
|
|
4577
|
+
}
|
|
4578
|
+
const currentValue = await fieldPlugin.getConsolidatedValue(id);
|
|
4579
|
+
return currentValue + actualAmount;
|
|
4580
|
+
};
|
|
4581
|
+
resource.sub = async (id, fieldOrAmount, amount) => {
|
|
4582
|
+
const { field, value: actualAmount, plugin: fieldPlugin } = plugin._resolveFieldAndPlugin(resource, fieldOrAmount, amount);
|
|
4583
|
+
await fieldPlugin.createTransaction({
|
|
4584
|
+
originalId: id,
|
|
4585
|
+
operation: "sub",
|
|
4586
|
+
value: actualAmount,
|
|
4587
|
+
source: "sub"
|
|
4588
|
+
});
|
|
4589
|
+
if (fieldPlugin.config.mode === "sync") {
|
|
4590
|
+
return await fieldPlugin._syncModeConsolidate(id, field);
|
|
4591
|
+
}
|
|
4592
|
+
const currentValue = await fieldPlugin.getConsolidatedValue(id);
|
|
4593
|
+
return currentValue - actualAmount;
|
|
4594
|
+
};
|
|
4595
|
+
resource.consolidate = async (id, field) => {
|
|
4596
|
+
const hasMultipleFields = Object.keys(resource._eventualConsistencyPlugins).length > 1;
|
|
4597
|
+
if (hasMultipleFields && !field) {
|
|
4598
|
+
throw new Error(`Multiple fields have eventual consistency. Please specify the field: consolidate(id, field)`);
|
|
4599
|
+
}
|
|
4600
|
+
const actualField = field || defaultField;
|
|
4601
|
+
const fieldPlugin = resource._eventualConsistencyPlugins[actualField];
|
|
4602
|
+
if (!fieldPlugin) {
|
|
4603
|
+
throw new Error(`No eventual consistency plugin found for field "${actualField}"`);
|
|
4604
|
+
}
|
|
4605
|
+
return await fieldPlugin.consolidateRecord(id);
|
|
4606
|
+
};
|
|
4607
|
+
resource.getConsolidatedValue = async (id, fieldOrOptions, options) => {
|
|
4608
|
+
if (typeof fieldOrOptions === "string") {
|
|
4609
|
+
const field = fieldOrOptions;
|
|
4610
|
+
const fieldPlugin = resource._eventualConsistencyPlugins[field] || plugin;
|
|
4611
|
+
return await fieldPlugin.getConsolidatedValue(id, options || {});
|
|
4612
|
+
} else {
|
|
4613
|
+
return await plugin.getConsolidatedValue(id, fieldOrOptions || {});
|
|
4614
|
+
}
|
|
4615
|
+
};
|
|
4616
|
+
}
|
|
4617
|
+
async createTransaction(data) {
|
|
4618
|
+
const now = /* @__PURE__ */ new Date();
|
|
4619
|
+
const cohortInfo = this.getCohortInfo(now);
|
|
4620
|
+
const watermarkMs = this.config.consolidationWindow * 60 * 60 * 1e3;
|
|
4621
|
+
const watermarkTime = now.getTime() - watermarkMs;
|
|
4622
|
+
const cohortHourDate = /* @__PURE__ */ new Date(cohortInfo.hour + ":00:00Z");
|
|
4623
|
+
if (cohortHourDate.getTime() < watermarkTime) {
|
|
4624
|
+
const hoursLate = Math.floor((now.getTime() - cohortHourDate.getTime()) / (60 * 60 * 1e3));
|
|
4625
|
+
if (this.config.lateArrivalStrategy === "ignore") {
|
|
4626
|
+
if (this.config.verbose) {
|
|
4627
|
+
console.warn(
|
|
4628
|
+
`[EventualConsistency] Late arrival ignored: transaction for ${cohortInfo.hour} is ${hoursLate}h late (watermark: ${this.config.consolidationWindow}h)`
|
|
4629
|
+
);
|
|
4630
|
+
}
|
|
4631
|
+
return null;
|
|
4632
|
+
} else if (this.config.lateArrivalStrategy === "warn") {
|
|
4633
|
+
console.warn(
|
|
4634
|
+
`[EventualConsistency] Late arrival detected: transaction for ${cohortInfo.hour} is ${hoursLate}h late (watermark: ${this.config.consolidationWindow}h). Processing anyway, but consolidation may not pick it up.`
|
|
4635
|
+
);
|
|
4636
|
+
}
|
|
4637
|
+
}
|
|
4638
|
+
const transaction = {
|
|
4639
|
+
id: idGenerator(),
|
|
4640
|
+
// Use nanoid for guaranteed uniqueness
|
|
4641
|
+
originalId: data.originalId,
|
|
4642
|
+
field: this.config.field,
|
|
4643
|
+
value: data.value || 0,
|
|
4644
|
+
operation: data.operation || "set",
|
|
4645
|
+
timestamp: now.toISOString(),
|
|
4646
|
+
cohortDate: cohortInfo.date,
|
|
4647
|
+
cohortHour: cohortInfo.hour,
|
|
4648
|
+
cohortMonth: cohortInfo.month,
|
|
4649
|
+
source: data.source || "unknown",
|
|
4650
|
+
applied: false
|
|
4651
|
+
};
|
|
4652
|
+
if (this.config.batchTransactions) {
|
|
4653
|
+
this.pendingTransactions.set(transaction.id, transaction);
|
|
4654
|
+
if (this.pendingTransactions.size >= this.config.batchSize) {
|
|
4655
|
+
await this.flushPendingTransactions();
|
|
4656
|
+
}
|
|
4657
|
+
} else {
|
|
4658
|
+
await this.transactionResource.insert(transaction);
|
|
4659
|
+
}
|
|
4660
|
+
return transaction;
|
|
4661
|
+
}
|
|
4662
|
+
async flushPendingTransactions() {
|
|
4663
|
+
if (this.pendingTransactions.size === 0) return;
|
|
4664
|
+
const transactions = Array.from(this.pendingTransactions.values());
|
|
4665
|
+
try {
|
|
4666
|
+
await Promise.all(
|
|
4667
|
+
transactions.map(
|
|
4668
|
+
(transaction) => this.transactionResource.insert(transaction)
|
|
4669
|
+
)
|
|
4670
|
+
);
|
|
4671
|
+
this.pendingTransactions.clear();
|
|
4672
|
+
} catch (error) {
|
|
4673
|
+
console.error("Failed to flush pending transactions:", error);
|
|
4674
|
+
throw error;
|
|
4675
|
+
}
|
|
4676
|
+
}
|
|
4677
|
+
getCohortInfo(date) {
|
|
4678
|
+
const tz = this.config.cohort.timezone;
|
|
4679
|
+
const offset = this.getTimezoneOffset(tz);
|
|
4680
|
+
const localDate = new Date(date.getTime() + offset);
|
|
4681
|
+
const year = localDate.getFullYear();
|
|
4682
|
+
const month = String(localDate.getMonth() + 1).padStart(2, "0");
|
|
4683
|
+
const day = String(localDate.getDate()).padStart(2, "0");
|
|
4684
|
+
const hour = String(localDate.getHours()).padStart(2, "0");
|
|
4685
|
+
return {
|
|
4686
|
+
date: `${year}-${month}-${day}`,
|
|
4687
|
+
hour: `${year}-${month}-${day}T${hour}`,
|
|
4688
|
+
// ISO-like format for hour partition
|
|
4689
|
+
month: `${year}-${month}`
|
|
4690
|
+
};
|
|
4691
|
+
}
|
|
4692
|
+
getTimezoneOffset(timezone) {
|
|
4693
|
+
try {
|
|
4694
|
+
const now = /* @__PURE__ */ new Date();
|
|
4695
|
+
const utcDate = new Date(now.toLocaleString("en-US", { timeZone: "UTC" }));
|
|
4696
|
+
const tzDate = new Date(now.toLocaleString("en-US", { timeZone: timezone }));
|
|
4697
|
+
return tzDate.getTime() - utcDate.getTime();
|
|
4698
|
+
} catch (err) {
|
|
4699
|
+
const offsets = {
|
|
4700
|
+
"UTC": 0,
|
|
4701
|
+
"America/New_York": -5 * 36e5,
|
|
4702
|
+
"America/Chicago": -6 * 36e5,
|
|
4703
|
+
"America/Denver": -7 * 36e5,
|
|
4704
|
+
"America/Los_Angeles": -8 * 36e5,
|
|
4705
|
+
"America/Sao_Paulo": -3 * 36e5,
|
|
4706
|
+
"Europe/London": 0,
|
|
4707
|
+
"Europe/Paris": 1 * 36e5,
|
|
4708
|
+
"Europe/Berlin": 1 * 36e5,
|
|
4709
|
+
"Asia/Tokyo": 9 * 36e5,
|
|
4710
|
+
"Asia/Shanghai": 8 * 36e5,
|
|
4711
|
+
"Australia/Sydney": 10 * 36e5
|
|
4712
|
+
};
|
|
4713
|
+
if (this.config.verbose && !offsets[timezone]) {
|
|
4714
|
+
console.warn(
|
|
4715
|
+
`[EventualConsistency] Unknown timezone '${timezone}', using UTC. Consider using a valid IANA timezone (e.g., 'America/New_York')`
|
|
4716
|
+
);
|
|
4717
|
+
}
|
|
4718
|
+
return offsets[timezone] || 0;
|
|
4719
|
+
}
|
|
4720
|
+
}
|
|
4721
|
+
startConsolidationTimer() {
|
|
4722
|
+
const intervalMs = this.config.consolidationInterval * 1e3;
|
|
4723
|
+
this.consolidationTimer = setInterval(async () => {
|
|
4724
|
+
await this.runConsolidation();
|
|
4725
|
+
}, intervalMs);
|
|
4726
|
+
}
|
|
4727
|
+
async runConsolidation() {
|
|
4728
|
+
try {
|
|
4729
|
+
const now = /* @__PURE__ */ new Date();
|
|
4730
|
+
const hoursToCheck = this.config.consolidationWindow || 24;
|
|
4731
|
+
const cohortHours = [];
|
|
4732
|
+
for (let i = 0; i < hoursToCheck; i++) {
|
|
4733
|
+
const date = new Date(now.getTime() - i * 60 * 60 * 1e3);
|
|
4734
|
+
const cohortInfo = this.getCohortInfo(date);
|
|
4735
|
+
cohortHours.push(cohortInfo.hour);
|
|
4736
|
+
}
|
|
4737
|
+
const transactionsByHour = await Promise.all(
|
|
4738
|
+
cohortHours.map(async (cohortHour) => {
|
|
4739
|
+
const [ok, err, txns] = await tryFn(
|
|
4740
|
+
() => this.transactionResource.query({
|
|
4741
|
+
cohortHour,
|
|
4742
|
+
applied: false
|
|
4743
|
+
})
|
|
4744
|
+
);
|
|
4745
|
+
return ok ? txns : [];
|
|
4746
|
+
})
|
|
4747
|
+
);
|
|
4748
|
+
const transactions = transactionsByHour.flat();
|
|
4749
|
+
if (transactions.length === 0) {
|
|
4750
|
+
if (this.config.verbose) {
|
|
4751
|
+
console.log(`[EventualConsistency] No pending transactions to consolidate`);
|
|
4752
|
+
}
|
|
4753
|
+
return;
|
|
4754
|
+
}
|
|
4755
|
+
const uniqueIds = [...new Set(transactions.map((t) => t.originalId))];
|
|
4756
|
+
const { results, errors } = await promisePool.PromisePool.for(uniqueIds).withConcurrency(this.config.consolidationConcurrency).process(async (id) => {
|
|
4757
|
+
return await this.consolidateRecord(id);
|
|
4758
|
+
});
|
|
4759
|
+
if (errors && errors.length > 0) {
|
|
4760
|
+
console.error(`Consolidation completed with ${errors.length} errors:`, errors);
|
|
4761
|
+
}
|
|
4762
|
+
this.emit("eventual-consistency.consolidated", {
|
|
4763
|
+
resource: this.config.resource,
|
|
4764
|
+
field: this.config.field,
|
|
4765
|
+
recordCount: uniqueIds.length,
|
|
4766
|
+
successCount: results.length,
|
|
4767
|
+
errorCount: errors.length
|
|
4768
|
+
});
|
|
4769
|
+
} catch (error) {
|
|
4770
|
+
console.error("Consolidation error:", error);
|
|
4771
|
+
this.emit("eventual-consistency.consolidation-error", error);
|
|
4772
|
+
}
|
|
4773
|
+
}
|
|
4774
|
+
async consolidateRecord(originalId) {
|
|
4775
|
+
await this.cleanupStaleLocks();
|
|
4776
|
+
const lockId = `lock-${originalId}`;
|
|
4777
|
+
const [lockAcquired, lockErr, lock] = await tryFn(
|
|
4778
|
+
() => this.lockResource.insert({
|
|
4779
|
+
id: lockId,
|
|
4780
|
+
lockedAt: Date.now(),
|
|
4781
|
+
workerId: process.pid ? String(process.pid) : "unknown"
|
|
4782
|
+
})
|
|
4783
|
+
);
|
|
4784
|
+
if (!lockAcquired) {
|
|
4785
|
+
if (this.config.verbose) {
|
|
4786
|
+
console.log(`[EventualConsistency] Lock for ${originalId} already held, skipping`);
|
|
4787
|
+
}
|
|
4788
|
+
const [recordOk, recordErr, record] = await tryFn(
|
|
4789
|
+
() => this.targetResource.get(originalId)
|
|
4790
|
+
);
|
|
4791
|
+
return recordOk && record ? record[this.config.field] || 0 : 0;
|
|
4792
|
+
}
|
|
4793
|
+
try {
|
|
4794
|
+
const [recordOk, recordErr, record] = await tryFn(
|
|
4795
|
+
() => this.targetResource.get(originalId)
|
|
4796
|
+
);
|
|
4797
|
+
const currentValue = recordOk && record ? record[this.config.field] || 0 : 0;
|
|
4798
|
+
const [ok, err, transactions] = await tryFn(
|
|
4799
|
+
() => this.transactionResource.query({
|
|
4800
|
+
originalId,
|
|
4801
|
+
applied: false
|
|
4802
|
+
})
|
|
4803
|
+
);
|
|
4804
|
+
if (!ok || !transactions || transactions.length === 0) {
|
|
4805
|
+
return currentValue;
|
|
4806
|
+
}
|
|
4807
|
+
transactions.sort(
|
|
4808
|
+
(a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()
|
|
4809
|
+
);
|
|
4810
|
+
const hasSetOperation = transactions.some((t) => t.operation === "set");
|
|
4811
|
+
if (currentValue !== 0 && !hasSetOperation) {
|
|
4812
|
+
transactions.unshift(this._createSyntheticSetTransaction(currentValue));
|
|
4813
|
+
}
|
|
4814
|
+
const consolidatedValue = this.config.reducer(transactions);
|
|
4815
|
+
const [updateOk, updateErr] = await tryFn(
|
|
4816
|
+
() => this.targetResource.update(originalId, {
|
|
4817
|
+
[this.config.field]: consolidatedValue
|
|
4818
|
+
})
|
|
4819
|
+
);
|
|
4820
|
+
if (updateOk) {
|
|
4821
|
+
const transactionsToUpdate = transactions.filter((txn) => txn.id !== "__synthetic__");
|
|
4822
|
+
const { results, errors } = await promisePool.PromisePool.for(transactionsToUpdate).withConcurrency(10).process(async (txn) => {
|
|
4823
|
+
const [ok2, err2] = await tryFn(
|
|
4824
|
+
() => this.transactionResource.update(txn.id, { applied: true })
|
|
4825
|
+
);
|
|
4826
|
+
if (!ok2 && this.config.verbose) {
|
|
4827
|
+
console.warn(`[EventualConsistency] Failed to mark transaction ${txn.id} as applied:`, err2?.message);
|
|
4828
|
+
}
|
|
4829
|
+
return ok2;
|
|
4830
|
+
});
|
|
4831
|
+
if (errors && errors.length > 0 && this.config.verbose) {
|
|
4832
|
+
console.warn(`[EventualConsistency] ${errors.length} transactions failed to mark as applied`);
|
|
4833
|
+
}
|
|
4834
|
+
}
|
|
4835
|
+
return consolidatedValue;
|
|
4836
|
+
} finally {
|
|
4837
|
+
const [lockReleased, lockReleaseErr] = await tryFn(() => this.lockResource.delete(lockId));
|
|
4838
|
+
if (!lockReleased && this.config.verbose) {
|
|
4839
|
+
console.warn(`[EventualConsistency] Failed to release lock ${lockId}:`, lockReleaseErr?.message);
|
|
4840
|
+
}
|
|
4841
|
+
}
|
|
4842
|
+
}
|
|
4843
|
+
async getConsolidatedValue(originalId, options = {}) {
|
|
4844
|
+
const includeApplied = options.includeApplied || false;
|
|
4845
|
+
const startDate = options.startDate;
|
|
4846
|
+
const endDate = options.endDate;
|
|
4847
|
+
const query = { originalId };
|
|
4848
|
+
if (!includeApplied) {
|
|
4849
|
+
query.applied = false;
|
|
4850
|
+
}
|
|
4851
|
+
const [ok, err, transactions] = await tryFn(
|
|
4852
|
+
() => this.transactionResource.query(query)
|
|
4853
|
+
);
|
|
4854
|
+
if (!ok || !transactions || transactions.length === 0) {
|
|
4855
|
+
const [recordOk2, recordErr2, record2] = await tryFn(
|
|
4856
|
+
() => this.targetResource.get(originalId)
|
|
4857
|
+
);
|
|
4858
|
+
if (recordOk2 && record2) {
|
|
4859
|
+
return record2[this.config.field] || 0;
|
|
4860
|
+
}
|
|
4861
|
+
return 0;
|
|
4862
|
+
}
|
|
4863
|
+
let filtered = transactions;
|
|
4864
|
+
if (startDate || endDate) {
|
|
4865
|
+
filtered = transactions.filter((t) => {
|
|
4866
|
+
const timestamp = new Date(t.timestamp);
|
|
4867
|
+
if (startDate && timestamp < new Date(startDate)) return false;
|
|
4868
|
+
if (endDate && timestamp > new Date(endDate)) return false;
|
|
4869
|
+
return true;
|
|
4870
|
+
});
|
|
4871
|
+
}
|
|
4872
|
+
const [recordOk, recordErr, record] = await tryFn(
|
|
4873
|
+
() => this.targetResource.get(originalId)
|
|
4874
|
+
);
|
|
4875
|
+
const currentValue = recordOk && record ? record[this.config.field] || 0 : 0;
|
|
4876
|
+
const hasSetOperation = filtered.some((t) => t.operation === "set");
|
|
4877
|
+
if (currentValue !== 0 && !hasSetOperation) {
|
|
4878
|
+
filtered.unshift(this._createSyntheticSetTransaction(currentValue));
|
|
4879
|
+
}
|
|
4880
|
+
filtered.sort(
|
|
4881
|
+
(a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()
|
|
4882
|
+
);
|
|
4883
|
+
return this.config.reducer(filtered);
|
|
4884
|
+
}
|
|
4885
|
+
// Helper method to get cohort statistics
|
|
4886
|
+
async getCohortStats(cohortDate) {
|
|
4887
|
+
const [ok, err, transactions] = await tryFn(
|
|
4888
|
+
() => this.transactionResource.query({
|
|
4889
|
+
cohortDate
|
|
4890
|
+
})
|
|
4891
|
+
);
|
|
4892
|
+
if (!ok) return null;
|
|
4893
|
+
const stats = {
|
|
4894
|
+
date: cohortDate,
|
|
4895
|
+
transactionCount: transactions.length,
|
|
4896
|
+
totalValue: 0,
|
|
4897
|
+
byOperation: { set: 0, add: 0, sub: 0 },
|
|
4898
|
+
byOriginalId: {}
|
|
4899
|
+
};
|
|
4900
|
+
for (const txn of transactions) {
|
|
4901
|
+
stats.totalValue += txn.value || 0;
|
|
4902
|
+
stats.byOperation[txn.operation] = (stats.byOperation[txn.operation] || 0) + 1;
|
|
4903
|
+
if (!stats.byOriginalId[txn.originalId]) {
|
|
4904
|
+
stats.byOriginalId[txn.originalId] = {
|
|
4905
|
+
count: 0,
|
|
4906
|
+
value: 0
|
|
4907
|
+
};
|
|
4908
|
+
}
|
|
4909
|
+
stats.byOriginalId[txn.originalId].count++;
|
|
4910
|
+
stats.byOriginalId[txn.originalId].value += txn.value || 0;
|
|
4911
|
+
}
|
|
4912
|
+
return stats;
|
|
4913
|
+
}
|
|
4914
|
+
/**
|
|
4915
|
+
* Clean up stale locks that exceed the configured timeout
|
|
4916
|
+
* Uses distributed locking to prevent multiple containers from cleaning simultaneously
|
|
4917
|
+
*/
|
|
4918
|
+
async cleanupStaleLocks() {
|
|
4919
|
+
const now = Date.now();
|
|
4920
|
+
const lockTimeoutMs = this.config.lockTimeout * 1e3;
|
|
4921
|
+
const cutoffTime = now - lockTimeoutMs;
|
|
4922
|
+
const cleanupLockId = `lock-cleanup-${this.config.resource}-${this.config.field}`;
|
|
4923
|
+
const [lockAcquired] = await tryFn(
|
|
4924
|
+
() => this.lockResource.insert({
|
|
4925
|
+
id: cleanupLockId,
|
|
4926
|
+
lockedAt: Date.now(),
|
|
4927
|
+
workerId: process.pid ? String(process.pid) : "unknown"
|
|
4928
|
+
})
|
|
4929
|
+
);
|
|
4930
|
+
if (!lockAcquired) {
|
|
4931
|
+
if (this.config.verbose) {
|
|
4932
|
+
console.log(`[EventualConsistency] Lock cleanup already running in another container`);
|
|
4933
|
+
}
|
|
4934
|
+
return;
|
|
4935
|
+
}
|
|
4936
|
+
try {
|
|
4937
|
+
const [ok, err, locks] = await tryFn(() => this.lockResource.list());
|
|
4938
|
+
if (!ok || !locks || locks.length === 0) return;
|
|
4939
|
+
const staleLocks = locks.filter(
|
|
4940
|
+
(lock) => lock.id !== cleanupLockId && lock.lockedAt < cutoffTime
|
|
4941
|
+
);
|
|
4942
|
+
if (staleLocks.length === 0) return;
|
|
4943
|
+
if (this.config.verbose) {
|
|
4944
|
+
console.log(`[EventualConsistency] Cleaning up ${staleLocks.length} stale locks`);
|
|
4945
|
+
}
|
|
4946
|
+
const { results, errors } = await promisePool.PromisePool.for(staleLocks).withConcurrency(5).process(async (lock) => {
|
|
4947
|
+
const [deleted] = await tryFn(() => this.lockResource.delete(lock.id));
|
|
4948
|
+
return deleted;
|
|
4949
|
+
});
|
|
4950
|
+
if (errors && errors.length > 0 && this.config.verbose) {
|
|
4951
|
+
console.warn(`[EventualConsistency] ${errors.length} stale locks failed to delete`);
|
|
4952
|
+
}
|
|
4953
|
+
} catch (error) {
|
|
4954
|
+
if (this.config.verbose) {
|
|
4955
|
+
console.warn(`[EventualConsistency] Error cleaning up stale locks:`, error.message);
|
|
4956
|
+
}
|
|
4957
|
+
} finally {
|
|
4958
|
+
await tryFn(() => this.lockResource.delete(cleanupLockId));
|
|
4959
|
+
}
|
|
4960
|
+
}
|
|
4961
|
+
/**
|
|
4962
|
+
* Start garbage collection timer for old applied transactions
|
|
4963
|
+
*/
|
|
4964
|
+
startGarbageCollectionTimer() {
|
|
4965
|
+
const gcIntervalMs = this.config.gcInterval * 1e3;
|
|
4966
|
+
this.gcTimer = setInterval(async () => {
|
|
4967
|
+
await this.runGarbageCollection();
|
|
4968
|
+
}, gcIntervalMs);
|
|
4969
|
+
}
|
|
4970
|
+
/**
|
|
4971
|
+
* Delete old applied transactions based on retention policy
|
|
4972
|
+
* Uses distributed locking to prevent multiple containers from running GC simultaneously
|
|
4973
|
+
*/
|
|
4974
|
+
async runGarbageCollection() {
|
|
4975
|
+
const gcLockId = `lock-gc-${this.config.resource}-${this.config.field}`;
|
|
4976
|
+
const [lockAcquired] = await tryFn(
|
|
4977
|
+
() => this.lockResource.insert({
|
|
4978
|
+
id: gcLockId,
|
|
4979
|
+
lockedAt: Date.now(),
|
|
4980
|
+
workerId: process.pid ? String(process.pid) : "unknown"
|
|
4981
|
+
})
|
|
4982
|
+
);
|
|
4983
|
+
if (!lockAcquired) {
|
|
4984
|
+
if (this.config.verbose) {
|
|
4985
|
+
console.log(`[EventualConsistency] GC already running in another container`);
|
|
4986
|
+
}
|
|
4987
|
+
return;
|
|
4988
|
+
}
|
|
4989
|
+
try {
|
|
4990
|
+
const now = Date.now();
|
|
4991
|
+
const retentionMs = this.config.transactionRetention * 24 * 60 * 60 * 1e3;
|
|
4992
|
+
const cutoffDate = new Date(now - retentionMs);
|
|
4993
|
+
const cutoffIso = cutoffDate.toISOString();
|
|
4994
|
+
if (this.config.verbose) {
|
|
4995
|
+
console.log(`[EventualConsistency] Running GC for transactions older than ${cutoffIso} (${this.config.transactionRetention} days)`);
|
|
4996
|
+
}
|
|
4997
|
+
const cutoffMonth = cutoffDate.toISOString().substring(0, 7);
|
|
4998
|
+
const [ok, err, oldTransactions] = await tryFn(
|
|
4999
|
+
() => this.transactionResource.query({
|
|
5000
|
+
applied: true,
|
|
5001
|
+
timestamp: { "<": cutoffIso }
|
|
5002
|
+
})
|
|
5003
|
+
);
|
|
5004
|
+
if (!ok) {
|
|
5005
|
+
if (this.config.verbose) {
|
|
5006
|
+
console.warn(`[EventualConsistency] GC failed to query transactions:`, err?.message);
|
|
5007
|
+
}
|
|
5008
|
+
return;
|
|
5009
|
+
}
|
|
5010
|
+
if (!oldTransactions || oldTransactions.length === 0) {
|
|
5011
|
+
if (this.config.verbose) {
|
|
5012
|
+
console.log(`[EventualConsistency] No old transactions to clean up`);
|
|
5013
|
+
}
|
|
5014
|
+
return;
|
|
5015
|
+
}
|
|
5016
|
+
if (this.config.verbose) {
|
|
5017
|
+
console.log(`[EventualConsistency] Deleting ${oldTransactions.length} old transactions`);
|
|
5018
|
+
}
|
|
5019
|
+
const { results, errors } = await promisePool.PromisePool.for(oldTransactions).withConcurrency(10).process(async (txn) => {
|
|
5020
|
+
const [deleted] = await tryFn(() => this.transactionResource.delete(txn.id));
|
|
5021
|
+
return deleted;
|
|
5022
|
+
});
|
|
5023
|
+
if (this.config.verbose) {
|
|
5024
|
+
console.log(`[EventualConsistency] GC completed: ${results.length} deleted, ${errors.length} errors`);
|
|
5025
|
+
}
|
|
5026
|
+
this.emit("eventual-consistency.gc-completed", {
|
|
5027
|
+
resource: this.config.resource,
|
|
5028
|
+
field: this.config.field,
|
|
5029
|
+
deletedCount: results.length,
|
|
5030
|
+
errorCount: errors.length
|
|
5031
|
+
});
|
|
5032
|
+
} catch (error) {
|
|
5033
|
+
if (this.config.verbose) {
|
|
5034
|
+
console.warn(`[EventualConsistency] GC error:`, error.message);
|
|
5035
|
+
}
|
|
5036
|
+
this.emit("eventual-consistency.gc-error", error);
|
|
5037
|
+
} finally {
|
|
5038
|
+
await tryFn(() => this.lockResource.delete(gcLockId));
|
|
5039
|
+
}
|
|
5040
|
+
}
|
|
5041
|
+
}
|
|
5042
|
+
|
|
5043
|
+
class FullTextPlugin extends Plugin {
|
|
5044
|
+
constructor(options = {}) {
|
|
5045
|
+
super();
|
|
5046
|
+
this.indexResource = null;
|
|
5047
|
+
this.config = {
|
|
5048
|
+
minWordLength: options.minWordLength || 3,
|
|
5049
|
+
maxResults: options.maxResults || 100,
|
|
5050
|
+
...options
|
|
5051
|
+
};
|
|
5052
|
+
this.indexes = /* @__PURE__ */ new Map();
|
|
5053
|
+
}
|
|
5054
|
+
async setup(database) {
|
|
5055
|
+
this.database = database;
|
|
5056
|
+
const [ok, err, indexResource] = await tryFn(() => database.createResource({
|
|
5057
|
+
name: "plg_fulltext_indexes",
|
|
5058
|
+
attributes: {
|
|
5059
|
+
id: "string|required",
|
|
5060
|
+
resourceName: "string|required",
|
|
5061
|
+
fieldName: "string|required",
|
|
5062
|
+
word: "string|required",
|
|
5063
|
+
recordIds: "json|required",
|
|
5064
|
+
// Array of record IDs containing this word
|
|
5065
|
+
count: "number|required",
|
|
5066
|
+
lastUpdated: "string|required"
|
|
5067
|
+
}
|
|
5068
|
+
}));
|
|
5069
|
+
this.indexResource = ok ? indexResource : database.resources.fulltext_indexes;
|
|
5070
|
+
await this.loadIndexes();
|
|
5071
|
+
this.installDatabaseHooks();
|
|
5072
|
+
this.installIndexingHooks();
|
|
5073
|
+
}
|
|
5074
|
+
async start() {
|
|
5075
|
+
}
|
|
5076
|
+
async stop() {
|
|
5077
|
+
await this.saveIndexes();
|
|
5078
|
+
this.removeDatabaseHooks();
|
|
5079
|
+
}
|
|
5080
|
+
async loadIndexes() {
|
|
5081
|
+
if (!this.indexResource) return;
|
|
5082
|
+
const [ok, err, allIndexes] = await tryFn(() => this.indexResource.getAll());
|
|
5083
|
+
if (ok) {
|
|
5084
|
+
for (const indexRecord of allIndexes) {
|
|
5085
|
+
const key = `${indexRecord.resourceName}:${indexRecord.fieldName}:${indexRecord.word}`;
|
|
5086
|
+
this.indexes.set(key, {
|
|
5087
|
+
recordIds: indexRecord.recordIds || [],
|
|
5088
|
+
count: indexRecord.count || 0
|
|
5089
|
+
});
|
|
5090
|
+
}
|
|
5091
|
+
}
|
|
5092
|
+
}
|
|
5093
|
+
async saveIndexes() {
|
|
5094
|
+
if (!this.indexResource) return;
|
|
5095
|
+
const [ok, err] = await tryFn(async () => {
|
|
5096
|
+
const existingIndexes = await this.indexResource.getAll();
|
|
5097
|
+
for (const index of existingIndexes) {
|
|
5098
|
+
await this.indexResource.delete(index.id);
|
|
4109
5099
|
}
|
|
4110
5100
|
for (const [key, data] of this.indexes.entries()) {
|
|
4111
5101
|
const [resourceName, fieldName, word] = key.split(":");
|
|
@@ -4123,7 +5113,7 @@ class FullTextPlugin extends Plugin {
|
|
|
4123
5113
|
}
|
|
4124
5114
|
installDatabaseHooks() {
|
|
4125
5115
|
this.database.addHook("afterCreateResource", (resource) => {
|
|
4126
|
-
if (resource.name !== "
|
|
5116
|
+
if (resource.name !== "plg_fulltext_indexes") {
|
|
4127
5117
|
this.installResourceHooks(resource);
|
|
4128
5118
|
}
|
|
4129
5119
|
});
|
|
@@ -4137,14 +5127,14 @@ class FullTextPlugin extends Plugin {
|
|
|
4137
5127
|
}
|
|
4138
5128
|
this.database.plugins.fulltext = this;
|
|
4139
5129
|
for (const resource of Object.values(this.database.resources)) {
|
|
4140
|
-
if (resource.name === "
|
|
5130
|
+
if (resource.name === "plg_fulltext_indexes") continue;
|
|
4141
5131
|
this.installResourceHooks(resource);
|
|
4142
5132
|
}
|
|
4143
5133
|
if (!this.database._fulltextProxyInstalled) {
|
|
4144
5134
|
this.database._previousCreateResourceForFullText = this.database.createResource;
|
|
4145
5135
|
this.database.createResource = async function(...args) {
|
|
4146
5136
|
const resource = await this._previousCreateResourceForFullText(...args);
|
|
4147
|
-
if (this.plugins?.fulltext && resource.name !== "
|
|
5137
|
+
if (this.plugins?.fulltext && resource.name !== "plg_fulltext_indexes") {
|
|
4148
5138
|
this.plugins.fulltext.installResourceHooks(resource);
|
|
4149
5139
|
}
|
|
4150
5140
|
return resource;
|
|
@@ -4152,7 +5142,7 @@ class FullTextPlugin extends Plugin {
|
|
|
4152
5142
|
this.database._fulltextProxyInstalled = true;
|
|
4153
5143
|
}
|
|
4154
5144
|
for (const resource of Object.values(this.database.resources)) {
|
|
4155
|
-
if (resource.name !== "
|
|
5145
|
+
if (resource.name !== "plg_fulltext_indexes") {
|
|
4156
5146
|
this.installResourceHooks(resource);
|
|
4157
5147
|
}
|
|
4158
5148
|
}
|
|
@@ -4395,7 +5385,7 @@ class FullTextPlugin extends Plugin {
|
|
|
4395
5385
|
return this._rebuildAllIndexesInternal();
|
|
4396
5386
|
}
|
|
4397
5387
|
async _rebuildAllIndexesInternal() {
|
|
4398
|
-
const resourceNames = Object.keys(this.database.resources).filter((name) => name !== "
|
|
5388
|
+
const resourceNames = Object.keys(this.database.resources).filter((name) => name !== "plg_fulltext_indexes");
|
|
4399
5389
|
for (const resourceName of resourceNames) {
|
|
4400
5390
|
const [ok, err] = await tryFn(() => this.rebuildIndex(resourceName));
|
|
4401
5391
|
}
|
|
@@ -4447,7 +5437,7 @@ class MetricsPlugin extends Plugin {
|
|
|
4447
5437
|
if (typeof process !== "undefined" && process.env.NODE_ENV === "test") return;
|
|
4448
5438
|
const [ok, err] = await tryFn(async () => {
|
|
4449
5439
|
const [ok1, err1, metricsResource] = await tryFn(() => database.createResource({
|
|
4450
|
-
name: "
|
|
5440
|
+
name: "plg_metrics",
|
|
4451
5441
|
attributes: {
|
|
4452
5442
|
id: "string|required",
|
|
4453
5443
|
type: "string|required",
|
|
@@ -4462,9 +5452,9 @@ class MetricsPlugin extends Plugin {
|
|
|
4462
5452
|
metadata: "json"
|
|
4463
5453
|
}
|
|
4464
5454
|
}));
|
|
4465
|
-
this.metricsResource = ok1 ? metricsResource : database.resources.
|
|
5455
|
+
this.metricsResource = ok1 ? metricsResource : database.resources.plg_metrics;
|
|
4466
5456
|
const [ok2, err2, errorsResource] = await tryFn(() => database.createResource({
|
|
4467
|
-
name: "
|
|
5457
|
+
name: "plg_error_logs",
|
|
4468
5458
|
attributes: {
|
|
4469
5459
|
id: "string|required",
|
|
4470
5460
|
resourceName: "string|required",
|
|
@@ -4474,9 +5464,9 @@ class MetricsPlugin extends Plugin {
|
|
|
4474
5464
|
metadata: "json"
|
|
4475
5465
|
}
|
|
4476
5466
|
}));
|
|
4477
|
-
this.errorsResource = ok2 ? errorsResource : database.resources.
|
|
5467
|
+
this.errorsResource = ok2 ? errorsResource : database.resources.plg_error_logs;
|
|
4478
5468
|
const [ok3, err3, performanceResource] = await tryFn(() => database.createResource({
|
|
4479
|
-
name: "
|
|
5469
|
+
name: "plg_performance_logs",
|
|
4480
5470
|
attributes: {
|
|
4481
5471
|
id: "string|required",
|
|
4482
5472
|
resourceName: "string|required",
|
|
@@ -4486,12 +5476,12 @@ class MetricsPlugin extends Plugin {
|
|
|
4486
5476
|
metadata: "json"
|
|
4487
5477
|
}
|
|
4488
5478
|
}));
|
|
4489
|
-
this.performanceResource = ok3 ? performanceResource : database.resources.
|
|
5479
|
+
this.performanceResource = ok3 ? performanceResource : database.resources.plg_performance_logs;
|
|
4490
5480
|
});
|
|
4491
5481
|
if (!ok) {
|
|
4492
|
-
this.metricsResource = database.resources.
|
|
4493
|
-
this.errorsResource = database.resources.
|
|
4494
|
-
this.performanceResource = database.resources.
|
|
5482
|
+
this.metricsResource = database.resources.plg_metrics;
|
|
5483
|
+
this.errorsResource = database.resources.plg_error_logs;
|
|
5484
|
+
this.performanceResource = database.resources.plg_performance_logs;
|
|
4495
5485
|
}
|
|
4496
5486
|
this.installDatabaseHooks();
|
|
4497
5487
|
this.installMetricsHooks();
|
|
@@ -4510,7 +5500,7 @@ class MetricsPlugin extends Plugin {
|
|
|
4510
5500
|
}
|
|
4511
5501
|
installDatabaseHooks() {
|
|
4512
5502
|
this.database.addHook("afterCreateResource", (resource) => {
|
|
4513
|
-
if (resource.name !== "
|
|
5503
|
+
if (resource.name !== "plg_metrics" && resource.name !== "plg_error_logs" && resource.name !== "plg_performance_logs") {
|
|
4514
5504
|
this.installResourceHooks(resource);
|
|
4515
5505
|
}
|
|
4516
5506
|
});
|
|
@@ -4520,7 +5510,7 @@ class MetricsPlugin extends Plugin {
|
|
|
4520
5510
|
}
|
|
4521
5511
|
installMetricsHooks() {
|
|
4522
5512
|
for (const resource of Object.values(this.database.resources)) {
|
|
4523
|
-
if (["
|
|
5513
|
+
if (["plg_metrics", "plg_error_logs", "plg_performance_logs"].includes(resource.name)) {
|
|
4524
5514
|
continue;
|
|
4525
5515
|
}
|
|
4526
5516
|
this.installResourceHooks(resource);
|
|
@@ -4528,7 +5518,7 @@ class MetricsPlugin extends Plugin {
|
|
|
4528
5518
|
this.database._createResource = this.database.createResource;
|
|
4529
5519
|
this.database.createResource = async function(...args) {
|
|
4530
5520
|
const resource = await this._createResource(...args);
|
|
4531
|
-
if (this.plugins?.metrics && !["
|
|
5521
|
+
if (this.plugins?.metrics && !["plg_metrics", "plg_error_logs", "plg_performance_logs"].includes(resource.name)) {
|
|
4532
5522
|
this.plugins.metrics.installResourceHooks(resource);
|
|
4533
5523
|
}
|
|
4534
5524
|
return resource;
|
|
@@ -5834,10 +6824,10 @@ class Client extends EventEmitter {
|
|
|
5834
6824
|
// Enabled for better performance
|
|
5835
6825
|
keepAliveMsecs: 1e3,
|
|
5836
6826
|
// 1 second keep-alive
|
|
5837
|
-
maxSockets:
|
|
5838
|
-
//
|
|
5839
|
-
maxFreeSockets:
|
|
5840
|
-
//
|
|
6827
|
+
maxSockets: httpClientOptions.maxSockets || 500,
|
|
6828
|
+
// High concurrency support
|
|
6829
|
+
maxFreeSockets: httpClientOptions.maxFreeSockets || 100,
|
|
6830
|
+
// Better connection reuse
|
|
5841
6831
|
timeout: 6e4,
|
|
5842
6832
|
// 60 second timeout
|
|
5843
6833
|
...httpClientOptions
|
|
@@ -5899,7 +6889,7 @@ class Client extends EventEmitter {
|
|
|
5899
6889
|
this.emit("command.response", command.constructor.name, response, command.input);
|
|
5900
6890
|
return response;
|
|
5901
6891
|
}
|
|
5902
|
-
async putObject({ key, metadata, contentType, body, contentEncoding, contentLength }) {
|
|
6892
|
+
async putObject({ key, metadata, contentType, body, contentEncoding, contentLength, ifMatch }) {
|
|
5903
6893
|
const keyPrefix = typeof this.config.keyPrefix === "string" ? this.config.keyPrefix : "";
|
|
5904
6894
|
keyPrefix ? path.join(keyPrefix, key) : key;
|
|
5905
6895
|
const stringMetadata = {};
|
|
@@ -5919,6 +6909,7 @@ class Client extends EventEmitter {
|
|
|
5919
6909
|
if (contentType !== void 0) options.ContentType = contentType;
|
|
5920
6910
|
if (contentEncoding !== void 0) options.ContentEncoding = contentEncoding;
|
|
5921
6911
|
if (contentLength !== void 0) options.ContentLength = contentLength;
|
|
6912
|
+
if (ifMatch !== void 0) options.IfMatch = ifMatch;
|
|
5922
6913
|
let response, error;
|
|
5923
6914
|
try {
|
|
5924
6915
|
response = await this.sendCommand(new clientS3.PutObjectCommand(options));
|
|
@@ -8082,6 +9073,7 @@ ${errorDetails}`,
|
|
|
8082
9073
|
data._lastModified = request.LastModified;
|
|
8083
9074
|
data._hasContent = request.ContentLength > 0;
|
|
8084
9075
|
data._mimeType = request.ContentType || null;
|
|
9076
|
+
data._etag = request.ETag;
|
|
8085
9077
|
data._v = objectVersion;
|
|
8086
9078
|
if (request.VersionId) data._versionId = request.VersionId;
|
|
8087
9079
|
if (request.Expiration) data._expiresAt = request.Expiration;
|
|
@@ -8293,6 +9285,172 @@ ${errorDetails}`,
|
|
|
8293
9285
|
return finalResult;
|
|
8294
9286
|
}
|
|
8295
9287
|
}
|
|
9288
|
+
/**
|
|
9289
|
+
* Update with conditional check (If-Match ETag)
|
|
9290
|
+
* @param {string} id - Resource ID
|
|
9291
|
+
* @param {Object} attributes - Attributes to update
|
|
9292
|
+
* @param {Object} options - Options including ifMatch (ETag)
|
|
9293
|
+
* @returns {Promise<Object>} { success: boolean, data?: Object, etag?: string, error?: string }
|
|
9294
|
+
* @example
|
|
9295
|
+
* const msg = await resource.get('msg-123');
|
|
9296
|
+
* const result = await resource.updateConditional('msg-123', { status: 'processing' }, { ifMatch: msg._etag });
|
|
9297
|
+
* if (!result.success) {
|
|
9298
|
+
* console.log('Update failed - object was modified by another process');
|
|
9299
|
+
* }
|
|
9300
|
+
*/
|
|
9301
|
+
async updateConditional(id, attributes, options = {}) {
|
|
9302
|
+
if (lodashEs.isEmpty(id)) {
|
|
9303
|
+
throw new Error("id cannot be empty");
|
|
9304
|
+
}
|
|
9305
|
+
const { ifMatch } = options;
|
|
9306
|
+
if (!ifMatch) {
|
|
9307
|
+
throw new Error("updateConditional requires ifMatch option with ETag value");
|
|
9308
|
+
}
|
|
9309
|
+
const exists = await this.exists(id);
|
|
9310
|
+
if (!exists) {
|
|
9311
|
+
return {
|
|
9312
|
+
success: false,
|
|
9313
|
+
error: `Resource with id '${id}' does not exist`
|
|
9314
|
+
};
|
|
9315
|
+
}
|
|
9316
|
+
const originalData = await this.get(id);
|
|
9317
|
+
const attributesClone = lodashEs.cloneDeep(attributes);
|
|
9318
|
+
let mergedData = lodashEs.cloneDeep(originalData);
|
|
9319
|
+
for (const [key2, value] of Object.entries(attributesClone)) {
|
|
9320
|
+
if (key2.includes(".")) {
|
|
9321
|
+
let ref = mergedData;
|
|
9322
|
+
const parts = key2.split(".");
|
|
9323
|
+
for (let i = 0; i < parts.length - 1; i++) {
|
|
9324
|
+
if (typeof ref[parts[i]] !== "object" || ref[parts[i]] === null) {
|
|
9325
|
+
ref[parts[i]] = {};
|
|
9326
|
+
}
|
|
9327
|
+
ref = ref[parts[i]];
|
|
9328
|
+
}
|
|
9329
|
+
ref[parts[parts.length - 1]] = lodashEs.cloneDeep(value);
|
|
9330
|
+
} else if (typeof value === "object" && value !== null && !Array.isArray(value)) {
|
|
9331
|
+
mergedData[key2] = lodashEs.merge({}, mergedData[key2], value);
|
|
9332
|
+
} else {
|
|
9333
|
+
mergedData[key2] = lodashEs.cloneDeep(value);
|
|
9334
|
+
}
|
|
9335
|
+
}
|
|
9336
|
+
if (this.config.timestamps) {
|
|
9337
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
9338
|
+
mergedData.updatedAt = now;
|
|
9339
|
+
if (!mergedData.metadata) mergedData.metadata = {};
|
|
9340
|
+
mergedData.metadata.updatedAt = now;
|
|
9341
|
+
}
|
|
9342
|
+
const preProcessedData = await this.executeHooks("beforeUpdate", lodashEs.cloneDeep(mergedData));
|
|
9343
|
+
const completeData = { ...originalData, ...preProcessedData, id };
|
|
9344
|
+
const { isValid, errors, data } = await this.validate(lodashEs.cloneDeep(completeData));
|
|
9345
|
+
if (!isValid) {
|
|
9346
|
+
return {
|
|
9347
|
+
success: false,
|
|
9348
|
+
error: "Validation failed: " + (errors && errors.length ? JSON.stringify(errors) : "unknown"),
|
|
9349
|
+
validationErrors: errors
|
|
9350
|
+
};
|
|
9351
|
+
}
|
|
9352
|
+
const { id: validatedId, ...validatedAttributes } = data;
|
|
9353
|
+
const mappedData = await this.schema.mapper(validatedAttributes);
|
|
9354
|
+
mappedData._v = String(this.version);
|
|
9355
|
+
const behaviorImpl = getBehavior(this.behavior);
|
|
9356
|
+
const { mappedData: processedMetadata, body } = await behaviorImpl.handleUpdate({
|
|
9357
|
+
resource: this,
|
|
9358
|
+
id,
|
|
9359
|
+
data: validatedAttributes,
|
|
9360
|
+
mappedData,
|
|
9361
|
+
originalData: { ...attributesClone, id }
|
|
9362
|
+
});
|
|
9363
|
+
const key = this.getResourceKey(id);
|
|
9364
|
+
let existingContentType = void 0;
|
|
9365
|
+
let finalBody = body;
|
|
9366
|
+
if (body === "" && this.behavior !== "body-overflow") {
|
|
9367
|
+
const [ok2, err2, existingObject] = await tryFn(() => this.client.getObject(key));
|
|
9368
|
+
if (ok2 && existingObject.ContentLength > 0) {
|
|
9369
|
+
const existingBodyBuffer = Buffer.from(await existingObject.Body.transformToByteArray());
|
|
9370
|
+
const existingBodyString = existingBodyBuffer.toString();
|
|
9371
|
+
const [okParse, errParse] = await tryFn(() => Promise.resolve(JSON.parse(existingBodyString)));
|
|
9372
|
+
if (!okParse) {
|
|
9373
|
+
finalBody = existingBodyBuffer;
|
|
9374
|
+
existingContentType = existingObject.ContentType;
|
|
9375
|
+
}
|
|
9376
|
+
}
|
|
9377
|
+
}
|
|
9378
|
+
let finalContentType = existingContentType;
|
|
9379
|
+
if (finalBody && finalBody !== "" && !finalContentType) {
|
|
9380
|
+
const [okParse, errParse] = await tryFn(() => Promise.resolve(JSON.parse(finalBody)));
|
|
9381
|
+
if (okParse) finalContentType = "application/json";
|
|
9382
|
+
}
|
|
9383
|
+
const [ok, err, response] = await tryFn(() => this.client.putObject({
|
|
9384
|
+
key,
|
|
9385
|
+
body: finalBody,
|
|
9386
|
+
contentType: finalContentType,
|
|
9387
|
+
metadata: processedMetadata,
|
|
9388
|
+
ifMatch
|
|
9389
|
+
// ← Conditional write with ETag
|
|
9390
|
+
}));
|
|
9391
|
+
if (!ok) {
|
|
9392
|
+
if (err.name === "PreconditionFailed" || err.$metadata?.httpStatusCode === 412) {
|
|
9393
|
+
return {
|
|
9394
|
+
success: false,
|
|
9395
|
+
error: "ETag mismatch - object was modified by another process"
|
|
9396
|
+
};
|
|
9397
|
+
}
|
|
9398
|
+
return {
|
|
9399
|
+
success: false,
|
|
9400
|
+
error: err.message || "Update failed"
|
|
9401
|
+
};
|
|
9402
|
+
}
|
|
9403
|
+
const updatedData = await this.composeFullObjectFromWrite({
|
|
9404
|
+
id,
|
|
9405
|
+
metadata: processedMetadata,
|
|
9406
|
+
body: finalBody,
|
|
9407
|
+
behavior: this.behavior
|
|
9408
|
+
});
|
|
9409
|
+
const oldData = { ...originalData, id };
|
|
9410
|
+
const newData = { ...validatedAttributes, id };
|
|
9411
|
+
if (this.config.asyncPartitions && this.config.partitions && Object.keys(this.config.partitions).length > 0) {
|
|
9412
|
+
setImmediate(() => {
|
|
9413
|
+
this.handlePartitionReferenceUpdates(oldData, newData).catch((err2) => {
|
|
9414
|
+
this.emit("partitionIndexError", {
|
|
9415
|
+
operation: "updateConditional",
|
|
9416
|
+
id,
|
|
9417
|
+
error: err2,
|
|
9418
|
+
message: err2.message
|
|
9419
|
+
});
|
|
9420
|
+
});
|
|
9421
|
+
});
|
|
9422
|
+
const nonPartitionHooks = this.hooks.afterUpdate.filter(
|
|
9423
|
+
(hook) => !hook.toString().includes("handlePartitionReferenceUpdates")
|
|
9424
|
+
);
|
|
9425
|
+
let finalResult = updatedData;
|
|
9426
|
+
for (const hook of nonPartitionHooks) {
|
|
9427
|
+
finalResult = await hook(finalResult);
|
|
9428
|
+
}
|
|
9429
|
+
this.emit("update", {
|
|
9430
|
+
...updatedData,
|
|
9431
|
+
$before: { ...originalData },
|
|
9432
|
+
$after: { ...finalResult }
|
|
9433
|
+
});
|
|
9434
|
+
return {
|
|
9435
|
+
success: true,
|
|
9436
|
+
data: finalResult,
|
|
9437
|
+
etag: response.ETag
|
|
9438
|
+
};
|
|
9439
|
+
} else {
|
|
9440
|
+
await this.handlePartitionReferenceUpdates(oldData, newData);
|
|
9441
|
+
const finalResult = await this.executeHooks("afterUpdate", updatedData);
|
|
9442
|
+
this.emit("update", {
|
|
9443
|
+
...updatedData,
|
|
9444
|
+
$before: { ...originalData },
|
|
9445
|
+
$after: { ...finalResult }
|
|
9446
|
+
});
|
|
9447
|
+
return {
|
|
9448
|
+
success: true,
|
|
9449
|
+
data: finalResult,
|
|
9450
|
+
etag: response.ETag
|
|
9451
|
+
};
|
|
9452
|
+
}
|
|
9453
|
+
}
|
|
8296
9454
|
/**
|
|
8297
9455
|
* Delete a resource object by ID
|
|
8298
9456
|
* @param {string} id - Resource ID
|
|
@@ -9700,7 +10858,7 @@ class Database extends EventEmitter {
|
|
|
9700
10858
|
this.id = idGenerator(7);
|
|
9701
10859
|
this.version = "1";
|
|
9702
10860
|
this.s3dbVersion = (() => {
|
|
9703
|
-
const [ok, err, version] = tryFn(() => true ? "
|
|
10861
|
+
const [ok, err, version] = tryFn(() => true ? "10.0.1" : "latest");
|
|
9704
10862
|
return ok ? version : "latest";
|
|
9705
10863
|
})();
|
|
9706
10864
|
this.resources = {};
|
|
@@ -10953,16 +12111,20 @@ class S3dbReplicator extends BaseReplicator {
|
|
|
10953
12111
|
return resource;
|
|
10954
12112
|
}
|
|
10955
12113
|
_getDestResourceObj(resource) {
|
|
10956
|
-
const
|
|
12114
|
+
const db = this.targetDatabase || this.client;
|
|
12115
|
+
const available = Object.keys(db.resources || {});
|
|
10957
12116
|
const norm = normalizeResourceName$1(resource);
|
|
10958
12117
|
const found = available.find((r) => normalizeResourceName$1(r) === norm);
|
|
10959
12118
|
if (!found) {
|
|
10960
12119
|
throw new Error(`[S3dbReplicator] Destination resource not found: ${resource}. Available: ${available.join(", ")}`);
|
|
10961
12120
|
}
|
|
10962
|
-
return
|
|
12121
|
+
return db.resources[found];
|
|
10963
12122
|
}
|
|
10964
12123
|
async replicateBatch(resourceName, records) {
|
|
10965
|
-
if (
|
|
12124
|
+
if (this.enabled === false) {
|
|
12125
|
+
return { skipped: true, reason: "replicator_disabled" };
|
|
12126
|
+
}
|
|
12127
|
+
if (!this.shouldReplicateResource(resourceName)) {
|
|
10966
12128
|
return { skipped: true, reason: "resource_not_included" };
|
|
10967
12129
|
}
|
|
10968
12130
|
const results = [];
|
|
@@ -11073,11 +12235,12 @@ class SqsReplicator extends BaseReplicator {
|
|
|
11073
12235
|
this.client = client;
|
|
11074
12236
|
this.queueUrl = config.queueUrl;
|
|
11075
12237
|
this.queues = config.queues || {};
|
|
11076
|
-
this.defaultQueue = config.defaultQueue || config.defaultQueueUrl || config.queueUrlDefault;
|
|
12238
|
+
this.defaultQueue = config.defaultQueue || config.defaultQueueUrl || config.queueUrlDefault || null;
|
|
11077
12239
|
this.region = config.region || "us-east-1";
|
|
11078
12240
|
this.sqsClient = client || null;
|
|
11079
12241
|
this.messageGroupId = config.messageGroupId;
|
|
11080
12242
|
this.deduplicationId = config.deduplicationId;
|
|
12243
|
+
this.resourceQueueMap = config.resourceQueueMap || null;
|
|
11081
12244
|
if (Array.isArray(resources)) {
|
|
11082
12245
|
this.resources = {};
|
|
11083
12246
|
for (const resource of resources) {
|
|
@@ -11208,7 +12371,10 @@ class SqsReplicator extends BaseReplicator {
|
|
|
11208
12371
|
}
|
|
11209
12372
|
}
|
|
11210
12373
|
async replicate(resource, operation, data, id, beforeData = null) {
|
|
11211
|
-
if (
|
|
12374
|
+
if (this.enabled === false) {
|
|
12375
|
+
return { skipped: true, reason: "replicator_disabled" };
|
|
12376
|
+
}
|
|
12377
|
+
if (!this.shouldReplicateResource(resource)) {
|
|
11212
12378
|
return { skipped: true, reason: "resource_not_included" };
|
|
11213
12379
|
}
|
|
11214
12380
|
const [ok, err, result] = await tryFn(async () => {
|
|
@@ -11252,7 +12418,10 @@ class SqsReplicator extends BaseReplicator {
|
|
|
11252
12418
|
return { success: false, error: err.message };
|
|
11253
12419
|
}
|
|
11254
12420
|
async replicateBatch(resource, records) {
|
|
11255
|
-
if (
|
|
12421
|
+
if (this.enabled === false) {
|
|
12422
|
+
return { skipped: true, reason: "replicator_disabled" };
|
|
12423
|
+
}
|
|
12424
|
+
if (!this.shouldReplicateResource(resource)) {
|
|
11256
12425
|
return { skipped: true, reason: "resource_not_included" };
|
|
11257
12426
|
}
|
|
11258
12427
|
const [ok, err, result] = await tryFn(async () => {
|
|
@@ -11406,22 +12575,23 @@ class ReplicatorPlugin extends Plugin {
|
|
|
11406
12575
|
replicators: options.replicators || [],
|
|
11407
12576
|
logErrors: options.logErrors !== false,
|
|
11408
12577
|
replicatorLogResource: options.replicatorLogResource || "replicator_log",
|
|
12578
|
+
persistReplicatorLog: options.persistReplicatorLog || false,
|
|
11409
12579
|
enabled: options.enabled !== false,
|
|
11410
12580
|
batchSize: options.batchSize || 100,
|
|
11411
12581
|
maxRetries: options.maxRetries || 3,
|
|
11412
12582
|
timeout: options.timeout || 3e4,
|
|
11413
|
-
verbose: options.verbose || false
|
|
11414
|
-
...options
|
|
12583
|
+
verbose: options.verbose || false
|
|
11415
12584
|
};
|
|
11416
12585
|
this.replicators = [];
|
|
11417
12586
|
this.database = null;
|
|
11418
12587
|
this.eventListenersInstalled = /* @__PURE__ */ new Set();
|
|
11419
|
-
|
|
11420
|
-
|
|
11421
|
-
|
|
11422
|
-
|
|
11423
|
-
|
|
11424
|
-
|
|
12588
|
+
this.eventHandlers = /* @__PURE__ */ new Map();
|
|
12589
|
+
this.stats = {
|
|
12590
|
+
totalReplications: 0,
|
|
12591
|
+
totalErrors: 0,
|
|
12592
|
+
lastSync: null
|
|
12593
|
+
};
|
|
12594
|
+
this._afterCreateResourceHook = null;
|
|
11425
12595
|
}
|
|
11426
12596
|
// Helper to filter out internal S3DB fields
|
|
11427
12597
|
filterInternalFields(obj) {
|
|
@@ -11442,7 +12612,7 @@ class ReplicatorPlugin extends Plugin {
|
|
|
11442
12612
|
if (!resource || this.eventListenersInstalled.has(resource.name) || resource.name === this.config.replicatorLogResource) {
|
|
11443
12613
|
return;
|
|
11444
12614
|
}
|
|
11445
|
-
|
|
12615
|
+
const insertHandler = async (data) => {
|
|
11446
12616
|
const [ok, error] = await tryFn(async () => {
|
|
11447
12617
|
const completeData = { ...data, createdAt: (/* @__PURE__ */ new Date()).toISOString() };
|
|
11448
12618
|
await plugin.processReplicatorEvent("insert", resource.name, completeData.id, completeData);
|
|
@@ -11453,8 +12623,8 @@ class ReplicatorPlugin extends Plugin {
|
|
|
11453
12623
|
}
|
|
11454
12624
|
this.emit("error", { operation: "insert", error: error.message, resource: resource.name });
|
|
11455
12625
|
}
|
|
11456
|
-
}
|
|
11457
|
-
|
|
12626
|
+
};
|
|
12627
|
+
const updateHandler = async (data, beforeData) => {
|
|
11458
12628
|
const [ok, error] = await tryFn(async () => {
|
|
11459
12629
|
const completeData = await plugin.getCompleteData(resource, data);
|
|
11460
12630
|
const dataWithTimestamp = { ...completeData, updatedAt: (/* @__PURE__ */ new Date()).toISOString() };
|
|
@@ -11466,8 +12636,8 @@ class ReplicatorPlugin extends Plugin {
|
|
|
11466
12636
|
}
|
|
11467
12637
|
this.emit("error", { operation: "update", error: error.message, resource: resource.name });
|
|
11468
12638
|
}
|
|
11469
|
-
}
|
|
11470
|
-
|
|
12639
|
+
};
|
|
12640
|
+
const deleteHandler = async (data) => {
|
|
11471
12641
|
const [ok, error] = await tryFn(async () => {
|
|
11472
12642
|
await plugin.processReplicatorEvent("delete", resource.name, data.id, data);
|
|
11473
12643
|
});
|
|
@@ -11477,14 +12647,22 @@ class ReplicatorPlugin extends Plugin {
|
|
|
11477
12647
|
}
|
|
11478
12648
|
this.emit("error", { operation: "delete", error: error.message, resource: resource.name });
|
|
11479
12649
|
}
|
|
11480
|
-
}
|
|
12650
|
+
};
|
|
12651
|
+
this.eventHandlers.set(resource.name, {
|
|
12652
|
+
insert: insertHandler,
|
|
12653
|
+
update: updateHandler,
|
|
12654
|
+
delete: deleteHandler
|
|
12655
|
+
});
|
|
12656
|
+
resource.on("insert", insertHandler);
|
|
12657
|
+
resource.on("update", updateHandler);
|
|
12658
|
+
resource.on("delete", deleteHandler);
|
|
11481
12659
|
this.eventListenersInstalled.add(resource.name);
|
|
11482
12660
|
}
|
|
11483
12661
|
async setup(database) {
|
|
11484
12662
|
this.database = database;
|
|
11485
12663
|
if (this.config.persistReplicatorLog) {
|
|
11486
12664
|
const [ok, err, logResource] = await tryFn(() => database.createResource({
|
|
11487
|
-
name: this.config.replicatorLogResource || "
|
|
12665
|
+
name: this.config.replicatorLogResource || "plg_replicator_logs",
|
|
11488
12666
|
attributes: {
|
|
11489
12667
|
id: "string|required",
|
|
11490
12668
|
resource: "string|required",
|
|
@@ -11498,13 +12676,13 @@ class ReplicatorPlugin extends Plugin {
|
|
|
11498
12676
|
if (ok) {
|
|
11499
12677
|
this.replicatorLogResource = logResource;
|
|
11500
12678
|
} else {
|
|
11501
|
-
this.replicatorLogResource = database.resources[this.config.replicatorLogResource || "
|
|
12679
|
+
this.replicatorLogResource = database.resources[this.config.replicatorLogResource || "plg_replicator_logs"];
|
|
11502
12680
|
}
|
|
11503
12681
|
}
|
|
11504
12682
|
await this.initializeReplicators(database);
|
|
11505
12683
|
this.installDatabaseHooks();
|
|
11506
12684
|
for (const resource of Object.values(database.resources)) {
|
|
11507
|
-
if (resource.name !== (this.config.replicatorLogResource || "
|
|
12685
|
+
if (resource.name !== (this.config.replicatorLogResource || "plg_replicator_logs")) {
|
|
11508
12686
|
this.installEventListeners(resource, database, this);
|
|
11509
12687
|
}
|
|
11510
12688
|
}
|
|
@@ -11520,14 +12698,18 @@ class ReplicatorPlugin extends Plugin {
|
|
|
11520
12698
|
this.removeDatabaseHooks();
|
|
11521
12699
|
}
|
|
11522
12700
|
installDatabaseHooks() {
|
|
11523
|
-
this.
|
|
11524
|
-
if (resource.name !== (this.config.replicatorLogResource || "
|
|
12701
|
+
this._afterCreateResourceHook = (resource) => {
|
|
12702
|
+
if (resource.name !== (this.config.replicatorLogResource || "plg_replicator_logs")) {
|
|
11525
12703
|
this.installEventListeners(resource, this.database, this);
|
|
11526
12704
|
}
|
|
11527
|
-
}
|
|
12705
|
+
};
|
|
12706
|
+
this.database.addHook("afterCreateResource", this._afterCreateResourceHook);
|
|
11528
12707
|
}
|
|
11529
12708
|
removeDatabaseHooks() {
|
|
11530
|
-
|
|
12709
|
+
if (this._afterCreateResourceHook) {
|
|
12710
|
+
this.database.removeHook("afterCreateResource", this._afterCreateResourceHook);
|
|
12711
|
+
this._afterCreateResourceHook = null;
|
|
12712
|
+
}
|
|
11531
12713
|
}
|
|
11532
12714
|
createReplicator(driver, config, resources, client) {
|
|
11533
12715
|
return createReplicator(driver, config, resources, client);
|
|
@@ -11552,9 +12734,9 @@ class ReplicatorPlugin extends Plugin {
|
|
|
11552
12734
|
async retryWithBackoff(operation, maxRetries = 3) {
|
|
11553
12735
|
let lastError;
|
|
11554
12736
|
for (let attempt = 1; attempt <= maxRetries; attempt++) {
|
|
11555
|
-
const [ok, error] = await tryFn(operation);
|
|
12737
|
+
const [ok, error, result] = await tryFn(operation);
|
|
11556
12738
|
if (ok) {
|
|
11557
|
-
return
|
|
12739
|
+
return result;
|
|
11558
12740
|
} else {
|
|
11559
12741
|
lastError = error;
|
|
11560
12742
|
if (this.config.verbose) {
|
|
@@ -11649,7 +12831,7 @@ class ReplicatorPlugin extends Plugin {
|
|
|
11649
12831
|
});
|
|
11650
12832
|
return Promise.allSettled(promises);
|
|
11651
12833
|
}
|
|
11652
|
-
async
|
|
12834
|
+
async processReplicatorItem(item) {
|
|
11653
12835
|
const applicableReplicators = this.replicators.filter((replicator) => {
|
|
11654
12836
|
const should = replicator.shouldReplicateResource && replicator.shouldReplicateResource(item.resourceName, item.operation);
|
|
11655
12837
|
return should;
|
|
@@ -11709,12 +12891,9 @@ class ReplicatorPlugin extends Plugin {
|
|
|
11709
12891
|
});
|
|
11710
12892
|
return Promise.allSettled(promises);
|
|
11711
12893
|
}
|
|
11712
|
-
async
|
|
12894
|
+
async logReplicator(item) {
|
|
11713
12895
|
const logRes = this.replicatorLog || this.database.resources[normalizeResourceName(this.config.replicatorLogResource)];
|
|
11714
12896
|
if (!logRes) {
|
|
11715
|
-
if (this.database) {
|
|
11716
|
-
if (this.database.options && this.database.options.connectionString) ;
|
|
11717
|
-
}
|
|
11718
12897
|
this.emit("replicator.log.failed", { error: "replicator log resource not found", item });
|
|
11719
12898
|
return;
|
|
11720
12899
|
}
|
|
@@ -11736,7 +12915,7 @@ class ReplicatorPlugin extends Plugin {
|
|
|
11736
12915
|
this.emit("replicator.log.failed", { error: err, item });
|
|
11737
12916
|
}
|
|
11738
12917
|
}
|
|
11739
|
-
async
|
|
12918
|
+
async updateReplicatorLog(logId, updates) {
|
|
11740
12919
|
if (!this.replicatorLog) return;
|
|
11741
12920
|
const [ok, err] = await tryFn(async () => {
|
|
11742
12921
|
await this.replicatorLog.update(logId, {
|
|
@@ -11749,7 +12928,7 @@ class ReplicatorPlugin extends Plugin {
|
|
|
11749
12928
|
}
|
|
11750
12929
|
}
|
|
11751
12930
|
// Utility methods
|
|
11752
|
-
async
|
|
12931
|
+
async getReplicatorStats() {
|
|
11753
12932
|
const replicatorStats = await Promise.all(
|
|
11754
12933
|
this.replicators.map(async (replicator) => {
|
|
11755
12934
|
const status = await replicator.getStatus();
|
|
@@ -11761,116 +12940,669 @@ class ReplicatorPlugin extends Plugin {
|
|
|
11761
12940
|
};
|
|
11762
12941
|
})
|
|
11763
12942
|
);
|
|
11764
|
-
return {
|
|
11765
|
-
replicators: replicatorStats,
|
|
11766
|
-
|
|
11767
|
-
|
|
11768
|
-
|
|
11769
|
-
|
|
11770
|
-
|
|
11771
|
-
|
|
11772
|
-
|
|
11773
|
-
|
|
11774
|
-
|
|
11775
|
-
|
|
11776
|
-
|
|
11777
|
-
|
|
11778
|
-
|
|
11779
|
-
|
|
11780
|
-
|
|
11781
|
-
|
|
11782
|
-
|
|
11783
|
-
|
|
11784
|
-
}
|
|
11785
|
-
|
|
11786
|
-
|
|
11787
|
-
|
|
12943
|
+
return {
|
|
12944
|
+
replicators: replicatorStats,
|
|
12945
|
+
stats: this.stats,
|
|
12946
|
+
lastSync: this.stats.lastSync
|
|
12947
|
+
};
|
|
12948
|
+
}
|
|
12949
|
+
async getReplicatorLogs(options = {}) {
|
|
12950
|
+
if (!this.replicatorLog) {
|
|
12951
|
+
return [];
|
|
12952
|
+
}
|
|
12953
|
+
const {
|
|
12954
|
+
resourceName,
|
|
12955
|
+
operation,
|
|
12956
|
+
status,
|
|
12957
|
+
limit = 100,
|
|
12958
|
+
offset = 0
|
|
12959
|
+
} = options;
|
|
12960
|
+
const filter = {};
|
|
12961
|
+
if (resourceName) {
|
|
12962
|
+
filter.resourceName = resourceName;
|
|
12963
|
+
}
|
|
12964
|
+
if (operation) {
|
|
12965
|
+
filter.operation = operation;
|
|
12966
|
+
}
|
|
12967
|
+
if (status) {
|
|
12968
|
+
filter.status = status;
|
|
12969
|
+
}
|
|
12970
|
+
const logs = await this.replicatorLog.query(filter, { limit, offset });
|
|
12971
|
+
return logs || [];
|
|
12972
|
+
}
|
|
12973
|
+
async retryFailedReplicators() {
|
|
12974
|
+
if (!this.replicatorLog) {
|
|
12975
|
+
return { retried: 0 };
|
|
12976
|
+
}
|
|
12977
|
+
const failedLogs = await this.replicatorLog.query({
|
|
12978
|
+
status: "failed"
|
|
12979
|
+
});
|
|
12980
|
+
let retried = 0;
|
|
12981
|
+
for (const log of failedLogs || []) {
|
|
12982
|
+
const [ok, err] = await tryFn(async () => {
|
|
12983
|
+
await this.processReplicatorEvent(
|
|
12984
|
+
log.operation,
|
|
12985
|
+
log.resourceName,
|
|
12986
|
+
log.recordId,
|
|
12987
|
+
log.data
|
|
12988
|
+
);
|
|
12989
|
+
});
|
|
12990
|
+
if (ok) {
|
|
12991
|
+
retried++;
|
|
12992
|
+
}
|
|
12993
|
+
}
|
|
12994
|
+
return { retried };
|
|
12995
|
+
}
|
|
12996
|
+
async syncAllData(replicatorId) {
|
|
12997
|
+
const replicator = this.replicators.find((r) => r.id === replicatorId);
|
|
12998
|
+
if (!replicator) {
|
|
12999
|
+
throw new Error(`Replicator not found: ${replicatorId}`);
|
|
13000
|
+
}
|
|
13001
|
+
this.stats.lastSync = (/* @__PURE__ */ new Date()).toISOString();
|
|
13002
|
+
for (const resourceName in this.database.resources) {
|
|
13003
|
+
if (normalizeResourceName(resourceName) === normalizeResourceName("plg_replicator_logs")) continue;
|
|
13004
|
+
if (replicator.shouldReplicateResource(resourceName)) {
|
|
13005
|
+
this.emit("replicator.sync.resource", { resourceName, replicatorId });
|
|
13006
|
+
const resource = this.database.resources[resourceName];
|
|
13007
|
+
let offset = 0;
|
|
13008
|
+
const pageSize = this.config.batchSize || 100;
|
|
13009
|
+
while (true) {
|
|
13010
|
+
const [ok, err, page] = await tryFn(() => resource.page({ offset, size: pageSize }));
|
|
13011
|
+
if (!ok || !page) break;
|
|
13012
|
+
const records = Array.isArray(page) ? page : page.items || [];
|
|
13013
|
+
if (records.length === 0) break;
|
|
13014
|
+
for (const record of records) {
|
|
13015
|
+
await replicator.replicate(resourceName, "insert", record, record.id);
|
|
13016
|
+
}
|
|
13017
|
+
offset += pageSize;
|
|
13018
|
+
}
|
|
13019
|
+
}
|
|
13020
|
+
}
|
|
13021
|
+
this.emit("replicator.sync.completed", { replicatorId, stats: this.stats });
|
|
13022
|
+
}
|
|
13023
|
+
async cleanup() {
|
|
13024
|
+
const [ok, error] = await tryFn(async () => {
|
|
13025
|
+
if (this.replicators && this.replicators.length > 0) {
|
|
13026
|
+
const cleanupPromises = this.replicators.map(async (replicator) => {
|
|
13027
|
+
const [replicatorOk, replicatorError] = await tryFn(async () => {
|
|
13028
|
+
if (replicator && typeof replicator.cleanup === "function") {
|
|
13029
|
+
await replicator.cleanup();
|
|
13030
|
+
}
|
|
13031
|
+
});
|
|
13032
|
+
if (!replicatorOk) {
|
|
13033
|
+
if (this.config.verbose) {
|
|
13034
|
+
console.warn(`[ReplicatorPlugin] Failed to cleanup replicator ${replicator.name || replicator.id}: ${replicatorError.message}`);
|
|
13035
|
+
}
|
|
13036
|
+
this.emit("replicator_cleanup_error", {
|
|
13037
|
+
replicator: replicator.name || replicator.id || "unknown",
|
|
13038
|
+
driver: replicator.driver || "unknown",
|
|
13039
|
+
error: replicatorError.message
|
|
13040
|
+
});
|
|
13041
|
+
}
|
|
13042
|
+
});
|
|
13043
|
+
await Promise.allSettled(cleanupPromises);
|
|
13044
|
+
}
|
|
13045
|
+
if (this.database && this.database.resources) {
|
|
13046
|
+
for (const resourceName of this.eventListenersInstalled) {
|
|
13047
|
+
const resource = this.database.resources[resourceName];
|
|
13048
|
+
const handlers = this.eventHandlers.get(resourceName);
|
|
13049
|
+
if (resource && handlers) {
|
|
13050
|
+
resource.off("insert", handlers.insert);
|
|
13051
|
+
resource.off("update", handlers.update);
|
|
13052
|
+
resource.off("delete", handlers.delete);
|
|
13053
|
+
}
|
|
13054
|
+
}
|
|
13055
|
+
}
|
|
13056
|
+
this.replicators = [];
|
|
13057
|
+
this.database = null;
|
|
13058
|
+
this.eventListenersInstalled.clear();
|
|
13059
|
+
this.eventHandlers.clear();
|
|
13060
|
+
this.removeAllListeners();
|
|
13061
|
+
});
|
|
13062
|
+
if (!ok) {
|
|
13063
|
+
if (this.config.verbose) {
|
|
13064
|
+
console.warn(`[ReplicatorPlugin] Failed to cleanup plugin: ${error.message}`);
|
|
13065
|
+
}
|
|
13066
|
+
this.emit("replicator_plugin_cleanup_error", {
|
|
13067
|
+
error: error.message
|
|
13068
|
+
});
|
|
13069
|
+
}
|
|
13070
|
+
}
|
|
13071
|
+
}
|
|
13072
|
+
|
|
13073
|
+
class S3QueuePlugin extends Plugin {
|
|
13074
|
+
constructor(options = {}) {
|
|
13075
|
+
super(options);
|
|
13076
|
+
if (!options.resource) {
|
|
13077
|
+
throw new Error('S3QueuePlugin requires "resource" option');
|
|
13078
|
+
}
|
|
13079
|
+
this.config = {
|
|
13080
|
+
resource: options.resource,
|
|
13081
|
+
visibilityTimeout: options.visibilityTimeout || 3e4,
|
|
13082
|
+
// 30 seconds
|
|
13083
|
+
pollInterval: options.pollInterval || 1e3,
|
|
13084
|
+
// 1 second
|
|
13085
|
+
maxAttempts: options.maxAttempts || 3,
|
|
13086
|
+
concurrency: options.concurrency || 1,
|
|
13087
|
+
deadLetterResource: options.deadLetterResource || null,
|
|
13088
|
+
autoStart: options.autoStart !== false,
|
|
13089
|
+
onMessage: options.onMessage,
|
|
13090
|
+
onError: options.onError,
|
|
13091
|
+
onComplete: options.onComplete,
|
|
13092
|
+
verbose: options.verbose || false,
|
|
13093
|
+
...options
|
|
13094
|
+
};
|
|
13095
|
+
this.queueResource = null;
|
|
13096
|
+
this.targetResource = null;
|
|
13097
|
+
this.deadLetterResourceObj = null;
|
|
13098
|
+
this.workers = [];
|
|
13099
|
+
this.isRunning = false;
|
|
13100
|
+
this.workerId = `worker-${Date.now()}-${Math.random().toString(36).slice(2, 9)}`;
|
|
13101
|
+
this.processedCache = /* @__PURE__ */ new Map();
|
|
13102
|
+
this.cacheCleanupInterval = null;
|
|
13103
|
+
this.lockCleanupInterval = null;
|
|
13104
|
+
}
|
|
13105
|
+
async onSetup() {
|
|
13106
|
+
this.targetResource = this.database.resources[this.config.resource];
|
|
13107
|
+
if (!this.targetResource) {
|
|
13108
|
+
throw new Error(`S3QueuePlugin: resource '${this.config.resource}' not found`);
|
|
13109
|
+
}
|
|
13110
|
+
const queueName = `${this.config.resource}_queue`;
|
|
13111
|
+
const [ok, err] = await tryFn(
|
|
13112
|
+
() => this.database.createResource({
|
|
13113
|
+
name: queueName,
|
|
13114
|
+
attributes: {
|
|
13115
|
+
id: "string|required",
|
|
13116
|
+
originalId: "string|required",
|
|
13117
|
+
// ID do registro original
|
|
13118
|
+
status: "string|required",
|
|
13119
|
+
// pending/processing/completed/failed/dead
|
|
13120
|
+
visibleAt: "number|required",
|
|
13121
|
+
// Timestamp de visibilidade
|
|
13122
|
+
claimedBy: "string|optional",
|
|
13123
|
+
// Worker que claimed
|
|
13124
|
+
claimedAt: "number|optional",
|
|
13125
|
+
// Timestamp do claim
|
|
13126
|
+
attempts: "number|default:0",
|
|
13127
|
+
maxAttempts: "number|default:3",
|
|
13128
|
+
error: "string|optional",
|
|
13129
|
+
result: "json|optional",
|
|
13130
|
+
createdAt: "string|required",
|
|
13131
|
+
completedAt: "number|optional"
|
|
13132
|
+
},
|
|
13133
|
+
behavior: "body-overflow",
|
|
13134
|
+
timestamps: true,
|
|
13135
|
+
asyncPartitions: true,
|
|
13136
|
+
partitions: {
|
|
13137
|
+
byStatus: { fields: { status: "string" } },
|
|
13138
|
+
byDate: { fields: { createdAt: "string|maxlength:10" } }
|
|
13139
|
+
}
|
|
13140
|
+
})
|
|
13141
|
+
);
|
|
13142
|
+
if (!ok && !this.database.resources[queueName]) {
|
|
13143
|
+
throw new Error(`Failed to create queue resource: ${err?.message}`);
|
|
13144
|
+
}
|
|
13145
|
+
this.queueResource = this.database.resources[queueName];
|
|
13146
|
+
const lockName = `${this.config.resource}_locks`;
|
|
13147
|
+
const [okLock, errLock] = await tryFn(
|
|
13148
|
+
() => this.database.createResource({
|
|
13149
|
+
name: lockName,
|
|
13150
|
+
attributes: {
|
|
13151
|
+
id: "string|required",
|
|
13152
|
+
workerId: "string|required",
|
|
13153
|
+
timestamp: "number|required",
|
|
13154
|
+
ttl: "number|default:5000"
|
|
13155
|
+
},
|
|
13156
|
+
behavior: "body-overflow",
|
|
13157
|
+
timestamps: false
|
|
13158
|
+
})
|
|
13159
|
+
);
|
|
13160
|
+
if (okLock || this.database.resources[lockName]) {
|
|
13161
|
+
this.lockResource = this.database.resources[lockName];
|
|
13162
|
+
} else {
|
|
13163
|
+
this.lockResource = null;
|
|
13164
|
+
if (this.config.verbose) {
|
|
13165
|
+
console.log(`[S3QueuePlugin] Lock resource creation failed, locking disabled: ${errLock?.message}`);
|
|
13166
|
+
}
|
|
13167
|
+
}
|
|
13168
|
+
this.addHelperMethods();
|
|
13169
|
+
if (this.config.deadLetterResource) {
|
|
13170
|
+
await this.createDeadLetterResource();
|
|
13171
|
+
}
|
|
13172
|
+
if (this.config.verbose) {
|
|
13173
|
+
console.log(`[S3QueuePlugin] Setup completed for resource '${this.config.resource}'`);
|
|
13174
|
+
}
|
|
13175
|
+
}
|
|
13176
|
+
async onStart() {
|
|
13177
|
+
if (this.config.autoStart && this.config.onMessage) {
|
|
13178
|
+
await this.startProcessing();
|
|
13179
|
+
}
|
|
13180
|
+
}
|
|
13181
|
+
async onStop() {
|
|
13182
|
+
await this.stopProcessing();
|
|
13183
|
+
}
|
|
13184
|
+
addHelperMethods() {
|
|
13185
|
+
const plugin = this;
|
|
13186
|
+
const resource = this.targetResource;
|
|
13187
|
+
resource.enqueue = async function(data, options = {}) {
|
|
13188
|
+
const recordData = {
|
|
13189
|
+
id: data.id || idGenerator(),
|
|
13190
|
+
...data
|
|
13191
|
+
};
|
|
13192
|
+
const record = await resource.insert(recordData);
|
|
13193
|
+
const queueEntry = {
|
|
13194
|
+
id: idGenerator(),
|
|
13195
|
+
originalId: record.id,
|
|
13196
|
+
status: "pending",
|
|
13197
|
+
visibleAt: Date.now(),
|
|
13198
|
+
attempts: 0,
|
|
13199
|
+
maxAttempts: options.maxAttempts || plugin.config.maxAttempts,
|
|
13200
|
+
createdAt: (/* @__PURE__ */ new Date()).toISOString().slice(0, 10)
|
|
13201
|
+
};
|
|
13202
|
+
await plugin.queueResource.insert(queueEntry);
|
|
13203
|
+
plugin.emit("message.enqueued", { id: record.id, queueId: queueEntry.id });
|
|
13204
|
+
return record;
|
|
13205
|
+
};
|
|
13206
|
+
resource.queueStats = async function() {
|
|
13207
|
+
return await plugin.getStats();
|
|
13208
|
+
};
|
|
13209
|
+
resource.startProcessing = async function(handler, options = {}) {
|
|
13210
|
+
return await plugin.startProcessing(handler, options);
|
|
13211
|
+
};
|
|
13212
|
+
resource.stopProcessing = async function() {
|
|
13213
|
+
return await plugin.stopProcessing();
|
|
13214
|
+
};
|
|
13215
|
+
}
|
|
13216
|
+
async startProcessing(handler = null, options = {}) {
|
|
13217
|
+
if (this.isRunning) {
|
|
13218
|
+
if (this.config.verbose) {
|
|
13219
|
+
console.log("[S3QueuePlugin] Already running");
|
|
13220
|
+
}
|
|
13221
|
+
return;
|
|
13222
|
+
}
|
|
13223
|
+
const messageHandler = handler || this.config.onMessage;
|
|
13224
|
+
if (!messageHandler) {
|
|
13225
|
+
throw new Error("S3QueuePlugin: onMessage handler required");
|
|
13226
|
+
}
|
|
13227
|
+
this.isRunning = true;
|
|
13228
|
+
const concurrency = options.concurrency || this.config.concurrency;
|
|
13229
|
+
this.cacheCleanupInterval = setInterval(() => {
|
|
13230
|
+
const now = Date.now();
|
|
13231
|
+
const maxAge = 3e4;
|
|
13232
|
+
for (const [queueId, timestamp] of this.processedCache.entries()) {
|
|
13233
|
+
if (now - timestamp > maxAge) {
|
|
13234
|
+
this.processedCache.delete(queueId);
|
|
13235
|
+
}
|
|
13236
|
+
}
|
|
13237
|
+
}, 5e3);
|
|
13238
|
+
this.lockCleanupInterval = setInterval(() => {
|
|
13239
|
+
this.cleanupStaleLocks().catch((err) => {
|
|
13240
|
+
if (this.config.verbose) {
|
|
13241
|
+
console.log(`[lockCleanup] Error: ${err.message}`);
|
|
13242
|
+
}
|
|
13243
|
+
});
|
|
13244
|
+
}, 1e4);
|
|
13245
|
+
for (let i = 0; i < concurrency; i++) {
|
|
13246
|
+
const worker = this.createWorker(messageHandler, i);
|
|
13247
|
+
this.workers.push(worker);
|
|
13248
|
+
}
|
|
13249
|
+
if (this.config.verbose) {
|
|
13250
|
+
console.log(`[S3QueuePlugin] Started ${concurrency} workers`);
|
|
13251
|
+
}
|
|
13252
|
+
this.emit("workers.started", { concurrency, workerId: this.workerId });
|
|
13253
|
+
}
|
|
13254
|
+
async stopProcessing() {
|
|
13255
|
+
if (!this.isRunning) return;
|
|
13256
|
+
this.isRunning = false;
|
|
13257
|
+
if (this.cacheCleanupInterval) {
|
|
13258
|
+
clearInterval(this.cacheCleanupInterval);
|
|
13259
|
+
this.cacheCleanupInterval = null;
|
|
13260
|
+
}
|
|
13261
|
+
if (this.lockCleanupInterval) {
|
|
13262
|
+
clearInterval(this.lockCleanupInterval);
|
|
13263
|
+
this.lockCleanupInterval = null;
|
|
13264
|
+
}
|
|
13265
|
+
await Promise.all(this.workers);
|
|
13266
|
+
this.workers = [];
|
|
13267
|
+
this.processedCache.clear();
|
|
13268
|
+
if (this.config.verbose) {
|
|
13269
|
+
console.log("[S3QueuePlugin] Stopped all workers");
|
|
13270
|
+
}
|
|
13271
|
+
this.emit("workers.stopped", { workerId: this.workerId });
|
|
13272
|
+
}
|
|
13273
|
+
createWorker(handler, workerIndex) {
|
|
13274
|
+
return (async () => {
|
|
13275
|
+
while (this.isRunning) {
|
|
13276
|
+
try {
|
|
13277
|
+
const message = await this.claimMessage();
|
|
13278
|
+
if (message) {
|
|
13279
|
+
await this.processMessage(message, handler);
|
|
13280
|
+
} else {
|
|
13281
|
+
await new Promise((resolve) => setTimeout(resolve, this.config.pollInterval));
|
|
13282
|
+
}
|
|
13283
|
+
} catch (error) {
|
|
13284
|
+
if (this.config.verbose) {
|
|
13285
|
+
console.error(`[Worker ${workerIndex}] Error:`, error.message);
|
|
13286
|
+
}
|
|
13287
|
+
await new Promise((resolve) => setTimeout(resolve, 1e3));
|
|
13288
|
+
}
|
|
13289
|
+
}
|
|
13290
|
+
})();
|
|
13291
|
+
}
|
|
13292
|
+
async claimMessage() {
|
|
13293
|
+
const now = Date.now();
|
|
13294
|
+
const [ok, err, messages] = await tryFn(
|
|
13295
|
+
() => this.queueResource.query({
|
|
13296
|
+
status: "pending"
|
|
13297
|
+
})
|
|
13298
|
+
);
|
|
13299
|
+
if (!ok || !messages || messages.length === 0) {
|
|
13300
|
+
return null;
|
|
13301
|
+
}
|
|
13302
|
+
const available = messages.filter((m) => m.visibleAt <= now);
|
|
13303
|
+
if (available.length === 0) {
|
|
13304
|
+
return null;
|
|
13305
|
+
}
|
|
13306
|
+
for (const msg of available) {
|
|
13307
|
+
const claimed = await this.attemptClaim(msg);
|
|
13308
|
+
if (claimed) {
|
|
13309
|
+
return claimed;
|
|
13310
|
+
}
|
|
13311
|
+
}
|
|
13312
|
+
return null;
|
|
13313
|
+
}
|
|
13314
|
+
/**
|
|
13315
|
+
* Acquire a distributed lock using ETag-based conditional updates
|
|
13316
|
+
* This ensures only one worker can claim a message at a time
|
|
13317
|
+
*
|
|
13318
|
+
* Uses a two-step process:
|
|
13319
|
+
* 1. Create lock resource (similar to queue resource) if not exists
|
|
13320
|
+
* 2. Try to claim lock using ETag-based conditional update
|
|
13321
|
+
*/
|
|
13322
|
+
async acquireLock(messageId) {
|
|
13323
|
+
if (!this.lockResource) {
|
|
13324
|
+
return true;
|
|
13325
|
+
}
|
|
13326
|
+
const lockId = `lock-${messageId}`;
|
|
13327
|
+
const now = Date.now();
|
|
13328
|
+
try {
|
|
13329
|
+
const [okGet, errGet, existingLock] = await tryFn(
|
|
13330
|
+
() => this.lockResource.get(lockId)
|
|
13331
|
+
);
|
|
13332
|
+
if (existingLock) {
|
|
13333
|
+
const lockAge = now - existingLock.timestamp;
|
|
13334
|
+
if (lockAge < existingLock.ttl) {
|
|
13335
|
+
return false;
|
|
13336
|
+
}
|
|
13337
|
+
const [ok, err, result] = await tryFn(
|
|
13338
|
+
() => this.lockResource.updateConditional(lockId, {
|
|
13339
|
+
workerId: this.workerId,
|
|
13340
|
+
timestamp: now,
|
|
13341
|
+
ttl: 5e3
|
|
13342
|
+
}, {
|
|
13343
|
+
ifMatch: existingLock._etag
|
|
13344
|
+
})
|
|
13345
|
+
);
|
|
13346
|
+
return ok && result.success;
|
|
13347
|
+
}
|
|
13348
|
+
const [okCreate, errCreate] = await tryFn(
|
|
13349
|
+
() => this.lockResource.insert({
|
|
13350
|
+
id: lockId,
|
|
13351
|
+
workerId: this.workerId,
|
|
13352
|
+
timestamp: now,
|
|
13353
|
+
ttl: 5e3
|
|
13354
|
+
})
|
|
13355
|
+
);
|
|
13356
|
+
return okCreate;
|
|
13357
|
+
} catch (error) {
|
|
13358
|
+
if (this.config.verbose) {
|
|
13359
|
+
console.log(`[acquireLock] Error: ${error.message}`);
|
|
13360
|
+
}
|
|
13361
|
+
return false;
|
|
13362
|
+
}
|
|
13363
|
+
}
|
|
13364
|
+
/**
|
|
13365
|
+
* Release a distributed lock by deleting the lock record
|
|
13366
|
+
*/
|
|
13367
|
+
async releaseLock(messageId) {
|
|
13368
|
+
if (!this.lockResource) {
|
|
13369
|
+
return;
|
|
13370
|
+
}
|
|
13371
|
+
const lockId = `lock-${messageId}`;
|
|
13372
|
+
try {
|
|
13373
|
+
await this.lockResource.delete(lockId);
|
|
13374
|
+
} catch (error) {
|
|
13375
|
+
if (this.config.verbose) {
|
|
13376
|
+
console.log(`[releaseLock] Failed to release lock for ${messageId}: ${error.message}`);
|
|
13377
|
+
}
|
|
13378
|
+
}
|
|
13379
|
+
}
|
|
13380
|
+
/**
|
|
13381
|
+
* Clean up stale locks (older than TTL)
|
|
13382
|
+
* This prevents deadlocks if a worker crashes while holding a lock
|
|
13383
|
+
*/
|
|
13384
|
+
async cleanupStaleLocks() {
|
|
13385
|
+
if (!this.lockResource) {
|
|
13386
|
+
return;
|
|
13387
|
+
}
|
|
13388
|
+
const now = Date.now();
|
|
13389
|
+
try {
|
|
13390
|
+
const locks = await this.lockResource.list();
|
|
13391
|
+
for (const lock of locks) {
|
|
13392
|
+
const lockAge = now - lock.timestamp;
|
|
13393
|
+
if (lockAge > lock.ttl) {
|
|
13394
|
+
await this.lockResource.delete(lock.id);
|
|
13395
|
+
if (this.config.verbose) {
|
|
13396
|
+
console.log(`[cleanupStaleLocks] Removed expired lock: ${lock.id}`);
|
|
13397
|
+
}
|
|
13398
|
+
}
|
|
13399
|
+
}
|
|
13400
|
+
} catch (error) {
|
|
13401
|
+
if (this.config.verbose) {
|
|
13402
|
+
console.log(`[cleanupStaleLocks] Error during cleanup: ${error.message}`);
|
|
13403
|
+
}
|
|
13404
|
+
}
|
|
13405
|
+
}
|
|
13406
|
+
async attemptClaim(msg) {
|
|
13407
|
+
const now = Date.now();
|
|
13408
|
+
const lockAcquired = await this.acquireLock(msg.id);
|
|
13409
|
+
if (!lockAcquired) {
|
|
13410
|
+
return null;
|
|
13411
|
+
}
|
|
13412
|
+
if (this.processedCache.has(msg.id)) {
|
|
13413
|
+
await this.releaseLock(msg.id);
|
|
13414
|
+
if (this.config.verbose) {
|
|
13415
|
+
console.log(`[attemptClaim] Message ${msg.id} already processed (in cache)`);
|
|
13416
|
+
}
|
|
13417
|
+
return null;
|
|
13418
|
+
}
|
|
13419
|
+
this.processedCache.set(msg.id, Date.now());
|
|
13420
|
+
await this.releaseLock(msg.id);
|
|
13421
|
+
const [okGet, errGet, msgWithETag] = await tryFn(
|
|
13422
|
+
() => this.queueResource.get(msg.id)
|
|
13423
|
+
);
|
|
13424
|
+
if (!okGet || !msgWithETag) {
|
|
13425
|
+
this.processedCache.delete(msg.id);
|
|
13426
|
+
if (this.config.verbose) {
|
|
13427
|
+
console.log(`[attemptClaim] Message ${msg.id} not found or error: ${errGet?.message}`);
|
|
13428
|
+
}
|
|
13429
|
+
return null;
|
|
13430
|
+
}
|
|
13431
|
+
if (msgWithETag.status !== "pending" || msgWithETag.visibleAt > now) {
|
|
13432
|
+
this.processedCache.delete(msg.id);
|
|
13433
|
+
if (this.config.verbose) {
|
|
13434
|
+
console.log(`[attemptClaim] Message ${msg.id} not claimable: status=${msgWithETag.status}, visibleAt=${msgWithETag.visibleAt}, now=${now}`);
|
|
13435
|
+
}
|
|
13436
|
+
return null;
|
|
13437
|
+
}
|
|
13438
|
+
if (this.config.verbose) {
|
|
13439
|
+
console.log(`[attemptClaim] Attempting to claim ${msg.id} with ETag: ${msgWithETag._etag}`);
|
|
13440
|
+
}
|
|
13441
|
+
const [ok, err, result] = await tryFn(
|
|
13442
|
+
() => this.queueResource.updateConditional(msgWithETag.id, {
|
|
13443
|
+
status: "processing",
|
|
13444
|
+
claimedBy: this.workerId,
|
|
13445
|
+
claimedAt: now,
|
|
13446
|
+
visibleAt: now + this.config.visibilityTimeout,
|
|
13447
|
+
attempts: msgWithETag.attempts + 1
|
|
13448
|
+
}, {
|
|
13449
|
+
ifMatch: msgWithETag._etag
|
|
13450
|
+
// ← ATOMIC CLAIM using ETag!
|
|
13451
|
+
})
|
|
13452
|
+
);
|
|
13453
|
+
if (!ok || !result.success) {
|
|
13454
|
+
this.processedCache.delete(msg.id);
|
|
13455
|
+
if (this.config.verbose) {
|
|
13456
|
+
console.log(`[attemptClaim] Failed to claim ${msg.id}: ${err?.message || result.error}`);
|
|
13457
|
+
}
|
|
13458
|
+
return null;
|
|
11788
13459
|
}
|
|
11789
|
-
if (
|
|
11790
|
-
|
|
13460
|
+
if (this.config.verbose) {
|
|
13461
|
+
console.log(`[attemptClaim] Successfully claimed ${msg.id}`);
|
|
11791
13462
|
}
|
|
11792
|
-
|
|
11793
|
-
|
|
13463
|
+
const [okRecord, errRecord, record] = await tryFn(
|
|
13464
|
+
() => this.targetResource.get(msgWithETag.originalId)
|
|
13465
|
+
);
|
|
13466
|
+
if (!okRecord) {
|
|
13467
|
+
await this.failMessage(msgWithETag.id, "Original record not found");
|
|
13468
|
+
return null;
|
|
11794
13469
|
}
|
|
11795
|
-
|
|
11796
|
-
|
|
13470
|
+
return {
|
|
13471
|
+
queueId: msgWithETag.id,
|
|
13472
|
+
record,
|
|
13473
|
+
attempts: msgWithETag.attempts + 1,
|
|
13474
|
+
maxAttempts: msgWithETag.maxAttempts
|
|
13475
|
+
};
|
|
11797
13476
|
}
|
|
11798
|
-
async
|
|
11799
|
-
|
|
11800
|
-
|
|
13477
|
+
async processMessage(message, handler) {
|
|
13478
|
+
const startTime = Date.now();
|
|
13479
|
+
try {
|
|
13480
|
+
const result = await handler(message.record, {
|
|
13481
|
+
queueId: message.queueId,
|
|
13482
|
+
attempts: message.attempts,
|
|
13483
|
+
workerId: this.workerId
|
|
13484
|
+
});
|
|
13485
|
+
await this.completeMessage(message.queueId, result);
|
|
13486
|
+
const duration = Date.now() - startTime;
|
|
13487
|
+
this.emit("message.completed", {
|
|
13488
|
+
queueId: message.queueId,
|
|
13489
|
+
originalId: message.record.id,
|
|
13490
|
+
duration,
|
|
13491
|
+
attempts: message.attempts
|
|
13492
|
+
});
|
|
13493
|
+
if (this.config.onComplete) {
|
|
13494
|
+
await this.config.onComplete(message.record, result);
|
|
13495
|
+
}
|
|
13496
|
+
} catch (error) {
|
|
13497
|
+
const shouldRetry = message.attempts < message.maxAttempts;
|
|
13498
|
+
if (shouldRetry) {
|
|
13499
|
+
await this.retryMessage(message.queueId, message.attempts, error.message);
|
|
13500
|
+
this.emit("message.retry", {
|
|
13501
|
+
queueId: message.queueId,
|
|
13502
|
+
originalId: message.record.id,
|
|
13503
|
+
attempts: message.attempts,
|
|
13504
|
+
error: error.message
|
|
13505
|
+
});
|
|
13506
|
+
} else {
|
|
13507
|
+
await this.moveToDeadLetter(message.queueId, message.record, error.message);
|
|
13508
|
+
this.emit("message.dead", {
|
|
13509
|
+
queueId: message.queueId,
|
|
13510
|
+
originalId: message.record.id,
|
|
13511
|
+
error: error.message
|
|
13512
|
+
});
|
|
13513
|
+
}
|
|
13514
|
+
if (this.config.onError) {
|
|
13515
|
+
await this.config.onError(error, message.record);
|
|
13516
|
+
}
|
|
11801
13517
|
}
|
|
11802
|
-
|
|
11803
|
-
|
|
13518
|
+
}
|
|
13519
|
+
async completeMessage(queueId, result) {
|
|
13520
|
+
await this.queueResource.update(queueId, {
|
|
13521
|
+
status: "completed",
|
|
13522
|
+
completedAt: Date.now(),
|
|
13523
|
+
result
|
|
11804
13524
|
});
|
|
11805
|
-
|
|
11806
|
-
|
|
11807
|
-
|
|
11808
|
-
|
|
11809
|
-
|
|
11810
|
-
|
|
11811
|
-
|
|
11812
|
-
|
|
11813
|
-
|
|
13525
|
+
}
|
|
13526
|
+
async failMessage(queueId, error) {
|
|
13527
|
+
await this.queueResource.update(queueId, {
|
|
13528
|
+
status: "failed",
|
|
13529
|
+
error
|
|
13530
|
+
});
|
|
13531
|
+
}
|
|
13532
|
+
async retryMessage(queueId, attempts, error) {
|
|
13533
|
+
const backoff = Math.min(Math.pow(2, attempts) * 1e3, 3e4);
|
|
13534
|
+
await this.queueResource.update(queueId, {
|
|
13535
|
+
status: "pending",
|
|
13536
|
+
visibleAt: Date.now() + backoff,
|
|
13537
|
+
error
|
|
13538
|
+
});
|
|
13539
|
+
this.processedCache.delete(queueId);
|
|
13540
|
+
}
|
|
13541
|
+
async moveToDeadLetter(queueId, record, error) {
|
|
13542
|
+
if (this.config.deadLetterResource && this.deadLetterResourceObj) {
|
|
13543
|
+
const msg = await this.queueResource.get(queueId);
|
|
13544
|
+
await this.deadLetterResourceObj.insert({
|
|
13545
|
+
id: idGenerator(),
|
|
13546
|
+
originalId: record.id,
|
|
13547
|
+
queueId,
|
|
13548
|
+
data: record,
|
|
13549
|
+
error,
|
|
13550
|
+
attempts: msg.attempts,
|
|
13551
|
+
createdAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
11814
13552
|
});
|
|
11815
|
-
if (ok) {
|
|
11816
|
-
retried++;
|
|
11817
|
-
}
|
|
11818
13553
|
}
|
|
11819
|
-
|
|
13554
|
+
await this.queueResource.update(queueId, {
|
|
13555
|
+
status: "dead",
|
|
13556
|
+
error
|
|
13557
|
+
});
|
|
11820
13558
|
}
|
|
11821
|
-
async
|
|
11822
|
-
const
|
|
11823
|
-
|
|
11824
|
-
|
|
13559
|
+
async getStats() {
|
|
13560
|
+
const [ok, err, allMessages] = await tryFn(
|
|
13561
|
+
() => this.queueResource.list()
|
|
13562
|
+
);
|
|
13563
|
+
if (!ok) {
|
|
13564
|
+
if (this.config.verbose) {
|
|
13565
|
+
console.warn("[S3QueuePlugin] Failed to get stats:", err.message);
|
|
13566
|
+
}
|
|
13567
|
+
return null;
|
|
11825
13568
|
}
|
|
11826
|
-
|
|
11827
|
-
|
|
11828
|
-
|
|
11829
|
-
|
|
11830
|
-
|
|
11831
|
-
|
|
11832
|
-
|
|
11833
|
-
|
|
11834
|
-
|
|
11835
|
-
|
|
13569
|
+
const stats = {
|
|
13570
|
+
total: allMessages.length,
|
|
13571
|
+
pending: 0,
|
|
13572
|
+
processing: 0,
|
|
13573
|
+
completed: 0,
|
|
13574
|
+
failed: 0,
|
|
13575
|
+
dead: 0
|
|
13576
|
+
};
|
|
13577
|
+
for (const msg of allMessages) {
|
|
13578
|
+
if (stats[msg.status] !== void 0) {
|
|
13579
|
+
stats[msg.status]++;
|
|
11836
13580
|
}
|
|
11837
13581
|
}
|
|
11838
|
-
|
|
13582
|
+
return stats;
|
|
11839
13583
|
}
|
|
11840
|
-
async
|
|
11841
|
-
const [ok,
|
|
11842
|
-
|
|
11843
|
-
|
|
11844
|
-
|
|
11845
|
-
|
|
11846
|
-
|
|
11847
|
-
|
|
11848
|
-
|
|
11849
|
-
|
|
11850
|
-
|
|
11851
|
-
|
|
11852
|
-
|
|
11853
|
-
|
|
11854
|
-
|
|
11855
|
-
|
|
11856
|
-
|
|
11857
|
-
|
|
11858
|
-
|
|
11859
|
-
});
|
|
11860
|
-
await Promise.allSettled(cleanupPromises);
|
|
11861
|
-
}
|
|
11862
|
-
this.replicators = [];
|
|
11863
|
-
this.database = null;
|
|
11864
|
-
this.eventListenersInstalled.clear();
|
|
11865
|
-
this.removeAllListeners();
|
|
11866
|
-
});
|
|
11867
|
-
if (!ok) {
|
|
13584
|
+
async createDeadLetterResource() {
|
|
13585
|
+
const [ok, err] = await tryFn(
|
|
13586
|
+
() => this.database.createResource({
|
|
13587
|
+
name: this.config.deadLetterResource,
|
|
13588
|
+
attributes: {
|
|
13589
|
+
id: "string|required",
|
|
13590
|
+
originalId: "string|required",
|
|
13591
|
+
queueId: "string|required",
|
|
13592
|
+
data: "json|required",
|
|
13593
|
+
error: "string|required",
|
|
13594
|
+
attempts: "number|required",
|
|
13595
|
+
createdAt: "string|required"
|
|
13596
|
+
},
|
|
13597
|
+
behavior: "body-overflow",
|
|
13598
|
+
timestamps: true
|
|
13599
|
+
})
|
|
13600
|
+
);
|
|
13601
|
+
if (ok || this.database.resources[this.config.deadLetterResource]) {
|
|
13602
|
+
this.deadLetterResourceObj = this.database.resources[this.config.deadLetterResource];
|
|
11868
13603
|
if (this.config.verbose) {
|
|
11869
|
-
console.
|
|
13604
|
+
console.log(`[S3QueuePlugin] Dead letter queue created: ${this.config.deadLetterResource}`);
|
|
11870
13605
|
}
|
|
11871
|
-
this.emit("replicator_plugin_cleanup_error", {
|
|
11872
|
-
error: error.message
|
|
11873
|
-
});
|
|
11874
13606
|
}
|
|
11875
13607
|
}
|
|
11876
13608
|
}
|
|
@@ -11884,7 +13616,7 @@ class SchedulerPlugin extends Plugin {
|
|
|
11884
13616
|
defaultTimeout: options.defaultTimeout || 3e5,
|
|
11885
13617
|
// 5 minutes
|
|
11886
13618
|
defaultRetries: options.defaultRetries || 1,
|
|
11887
|
-
jobHistoryResource: options.jobHistoryResource || "
|
|
13619
|
+
jobHistoryResource: options.jobHistoryResource || "plg_job_executions",
|
|
11888
13620
|
persistJobs: options.persistJobs !== false,
|
|
11889
13621
|
verbose: options.verbose || false,
|
|
11890
13622
|
onJobStart: options.onJobStart || null,
|
|
@@ -11893,12 +13625,20 @@ class SchedulerPlugin extends Plugin {
|
|
|
11893
13625
|
...options
|
|
11894
13626
|
};
|
|
11895
13627
|
this.database = null;
|
|
13628
|
+
this.lockResource = null;
|
|
11896
13629
|
this.jobs = /* @__PURE__ */ new Map();
|
|
11897
13630
|
this.activeJobs = /* @__PURE__ */ new Map();
|
|
11898
13631
|
this.timers = /* @__PURE__ */ new Map();
|
|
11899
13632
|
this.statistics = /* @__PURE__ */ new Map();
|
|
11900
13633
|
this._validateConfiguration();
|
|
11901
13634
|
}
|
|
13635
|
+
/**
|
|
13636
|
+
* Helper to detect test environment
|
|
13637
|
+
* @private
|
|
13638
|
+
*/
|
|
13639
|
+
_isTestEnvironment() {
|
|
13640
|
+
return process.env.NODE_ENV === "test" || process.env.JEST_WORKER_ID !== void 0 || global.expect !== void 0;
|
|
13641
|
+
}
|
|
11902
13642
|
_validateConfiguration() {
|
|
11903
13643
|
if (Object.keys(this.config.jobs).length === 0) {
|
|
11904
13644
|
throw new Error("SchedulerPlugin: At least one job must be defined");
|
|
@@ -11925,6 +13665,7 @@ class SchedulerPlugin extends Plugin {
|
|
|
11925
13665
|
}
|
|
11926
13666
|
async setup(database) {
|
|
11927
13667
|
this.database = database;
|
|
13668
|
+
await this._createLockResource();
|
|
11928
13669
|
if (this.config.persistJobs) {
|
|
11929
13670
|
await this._createJobHistoryResource();
|
|
11930
13671
|
}
|
|
@@ -11953,6 +13694,25 @@ class SchedulerPlugin extends Plugin {
|
|
|
11953
13694
|
await this._startScheduling();
|
|
11954
13695
|
this.emit("initialized", { jobs: this.jobs.size });
|
|
11955
13696
|
}
|
|
13697
|
+
async _createLockResource() {
|
|
13698
|
+
const [ok, err, lockResource] = await tryFn(
|
|
13699
|
+
() => this.database.createResource({
|
|
13700
|
+
name: "plg_scheduler_job_locks",
|
|
13701
|
+
attributes: {
|
|
13702
|
+
id: "string|required",
|
|
13703
|
+
jobName: "string|required",
|
|
13704
|
+
lockedAt: "number|required",
|
|
13705
|
+
instanceId: "string|optional"
|
|
13706
|
+
},
|
|
13707
|
+
behavior: "body-only",
|
|
13708
|
+
timestamps: false
|
|
13709
|
+
})
|
|
13710
|
+
);
|
|
13711
|
+
if (!ok && !this.database.resources.plg_scheduler_job_locks) {
|
|
13712
|
+
throw new Error(`Failed to create lock resource: ${err?.message}`);
|
|
13713
|
+
}
|
|
13714
|
+
this.lockResource = ok ? lockResource : this.database.resources.plg_scheduler_job_locks;
|
|
13715
|
+
}
|
|
11956
13716
|
async _createJobHistoryResource() {
|
|
11957
13717
|
const [ok] = await tryFn(() => this.database.createResource({
|
|
11958
13718
|
name: this.config.jobHistoryResource,
|
|
@@ -12046,18 +13806,37 @@ class SchedulerPlugin extends Plugin {
|
|
|
12046
13806
|
next.setHours(next.getHours() + 1);
|
|
12047
13807
|
}
|
|
12048
13808
|
}
|
|
12049
|
-
|
|
12050
|
-
if (isTestEnvironment) {
|
|
13809
|
+
if (this._isTestEnvironment()) {
|
|
12051
13810
|
next.setTime(next.getTime() + 1e3);
|
|
12052
13811
|
}
|
|
12053
13812
|
return next;
|
|
12054
13813
|
}
|
|
12055
13814
|
async _executeJob(jobName) {
|
|
12056
13815
|
const job = this.jobs.get(jobName);
|
|
12057
|
-
if (!job
|
|
13816
|
+
if (!job) {
|
|
13817
|
+
return;
|
|
13818
|
+
}
|
|
13819
|
+
if (this.activeJobs.has(jobName)) {
|
|
13820
|
+
return;
|
|
13821
|
+
}
|
|
13822
|
+
this.activeJobs.set(jobName, "acquiring-lock");
|
|
13823
|
+
const lockId = `lock-${jobName}`;
|
|
13824
|
+
const [lockAcquired, lockErr] = await tryFn(
|
|
13825
|
+
() => this.lockResource.insert({
|
|
13826
|
+
id: lockId,
|
|
13827
|
+
jobName,
|
|
13828
|
+
lockedAt: Date.now(),
|
|
13829
|
+
instanceId: process.pid ? String(process.pid) : "unknown"
|
|
13830
|
+
})
|
|
13831
|
+
);
|
|
13832
|
+
if (!lockAcquired) {
|
|
13833
|
+
if (this.config.verbose) {
|
|
13834
|
+
console.log(`[SchedulerPlugin] Job '${jobName}' already running on another instance`);
|
|
13835
|
+
}
|
|
13836
|
+
this.activeJobs.delete(jobName);
|
|
12058
13837
|
return;
|
|
12059
13838
|
}
|
|
12060
|
-
const executionId = `${jobName}_${
|
|
13839
|
+
const executionId = `${jobName}_${idGenerator()}`;
|
|
12061
13840
|
const startTime = Date.now();
|
|
12062
13841
|
const context = {
|
|
12063
13842
|
jobName,
|
|
@@ -12066,91 +13845,95 @@ class SchedulerPlugin extends Plugin {
|
|
|
12066
13845
|
database: this.database
|
|
12067
13846
|
};
|
|
12068
13847
|
this.activeJobs.set(jobName, executionId);
|
|
12069
|
-
|
|
12070
|
-
|
|
12071
|
-
|
|
12072
|
-
|
|
12073
|
-
|
|
12074
|
-
|
|
12075
|
-
|
|
12076
|
-
|
|
12077
|
-
|
|
12078
|
-
|
|
12079
|
-
|
|
12080
|
-
const actualTimeout = isTestEnvironment ? Math.min(job.timeout, 1e3) : job.timeout;
|
|
12081
|
-
let timeoutId;
|
|
12082
|
-
const timeoutPromise = new Promise((_, reject) => {
|
|
12083
|
-
timeoutId = setTimeout(() => reject(new Error("Job execution timeout")), actualTimeout);
|
|
12084
|
-
});
|
|
12085
|
-
const jobPromise = job.action(this.database, context, this);
|
|
13848
|
+
try {
|
|
13849
|
+
if (this.config.onJobStart) {
|
|
13850
|
+
await this._executeHook(this.config.onJobStart, jobName, context);
|
|
13851
|
+
}
|
|
13852
|
+
this.emit("job_start", { jobName, executionId, startTime });
|
|
13853
|
+
let attempt = 0;
|
|
13854
|
+
let lastError = null;
|
|
13855
|
+
let result = null;
|
|
13856
|
+
let status = "success";
|
|
13857
|
+
const isTestEnvironment = this._isTestEnvironment();
|
|
13858
|
+
while (attempt <= job.retries) {
|
|
12086
13859
|
try {
|
|
12087
|
-
|
|
12088
|
-
|
|
12089
|
-
|
|
12090
|
-
|
|
12091
|
-
|
|
12092
|
-
|
|
12093
|
-
|
|
12094
|
-
|
|
12095
|
-
|
|
12096
|
-
|
|
12097
|
-
|
|
12098
|
-
|
|
12099
|
-
|
|
12100
|
-
|
|
13860
|
+
const actualTimeout = isTestEnvironment ? Math.min(job.timeout, 1e3) : job.timeout;
|
|
13861
|
+
let timeoutId;
|
|
13862
|
+
const timeoutPromise = new Promise((_, reject) => {
|
|
13863
|
+
timeoutId = setTimeout(() => reject(new Error("Job execution timeout")), actualTimeout);
|
|
13864
|
+
});
|
|
13865
|
+
const jobPromise = job.action(this.database, context, this);
|
|
13866
|
+
try {
|
|
13867
|
+
result = await Promise.race([jobPromise, timeoutPromise]);
|
|
13868
|
+
clearTimeout(timeoutId);
|
|
13869
|
+
} catch (raceError) {
|
|
13870
|
+
clearTimeout(timeoutId);
|
|
13871
|
+
throw raceError;
|
|
13872
|
+
}
|
|
13873
|
+
status = "success";
|
|
13874
|
+
break;
|
|
13875
|
+
} catch (error) {
|
|
13876
|
+
lastError = error;
|
|
13877
|
+
attempt++;
|
|
13878
|
+
if (attempt <= job.retries) {
|
|
13879
|
+
if (this.config.verbose) {
|
|
13880
|
+
console.warn(`[SchedulerPlugin] Job '${jobName}' failed (attempt ${attempt + 1}):`, error.message);
|
|
13881
|
+
}
|
|
13882
|
+
const baseDelay = Math.min(Math.pow(2, attempt) * 1e3, 5e3);
|
|
13883
|
+
const delay = isTestEnvironment ? 1 : baseDelay;
|
|
13884
|
+
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
12101
13885
|
}
|
|
12102
|
-
const baseDelay = Math.min(Math.pow(2, attempt) * 1e3, 5e3);
|
|
12103
|
-
const delay = isTestEnvironment ? 1 : baseDelay;
|
|
12104
|
-
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
12105
13886
|
}
|
|
12106
13887
|
}
|
|
12107
|
-
|
|
12108
|
-
|
|
12109
|
-
|
|
12110
|
-
|
|
12111
|
-
|
|
12112
|
-
|
|
12113
|
-
|
|
12114
|
-
|
|
12115
|
-
|
|
12116
|
-
|
|
12117
|
-
|
|
12118
|
-
|
|
12119
|
-
|
|
12120
|
-
|
|
12121
|
-
|
|
12122
|
-
|
|
12123
|
-
|
|
12124
|
-
|
|
12125
|
-
|
|
12126
|
-
|
|
12127
|
-
|
|
12128
|
-
|
|
12129
|
-
|
|
12130
|
-
|
|
12131
|
-
|
|
12132
|
-
|
|
12133
|
-
|
|
12134
|
-
|
|
12135
|
-
|
|
12136
|
-
|
|
12137
|
-
|
|
12138
|
-
|
|
12139
|
-
|
|
12140
|
-
|
|
12141
|
-
|
|
12142
|
-
|
|
12143
|
-
|
|
12144
|
-
|
|
12145
|
-
|
|
12146
|
-
|
|
12147
|
-
|
|
12148
|
-
|
|
12149
|
-
|
|
12150
|
-
|
|
12151
|
-
|
|
12152
|
-
|
|
12153
|
-
|
|
13888
|
+
const endTime = Date.now();
|
|
13889
|
+
const duration = Math.max(1, endTime - startTime);
|
|
13890
|
+
if (lastError && attempt > job.retries) {
|
|
13891
|
+
status = lastError.message.includes("timeout") ? "timeout" : "error";
|
|
13892
|
+
}
|
|
13893
|
+
job.lastRun = new Date(endTime);
|
|
13894
|
+
job.runCount++;
|
|
13895
|
+
if (status === "success") {
|
|
13896
|
+
job.successCount++;
|
|
13897
|
+
} else {
|
|
13898
|
+
job.errorCount++;
|
|
13899
|
+
}
|
|
13900
|
+
const stats = this.statistics.get(jobName);
|
|
13901
|
+
stats.totalRuns++;
|
|
13902
|
+
stats.lastRun = new Date(endTime);
|
|
13903
|
+
if (status === "success") {
|
|
13904
|
+
stats.totalSuccesses++;
|
|
13905
|
+
stats.lastSuccess = new Date(endTime);
|
|
13906
|
+
} else {
|
|
13907
|
+
stats.totalErrors++;
|
|
13908
|
+
stats.lastError = { time: new Date(endTime), message: lastError?.message };
|
|
13909
|
+
}
|
|
13910
|
+
stats.avgDuration = (stats.avgDuration * (stats.totalRuns - 1) + duration) / stats.totalRuns;
|
|
13911
|
+
if (this.config.persistJobs) {
|
|
13912
|
+
await this._persistJobExecution(jobName, executionId, startTime, endTime, duration, status, result, lastError, attempt);
|
|
13913
|
+
}
|
|
13914
|
+
if (status === "success" && this.config.onJobComplete) {
|
|
13915
|
+
await this._executeHook(this.config.onJobComplete, jobName, result, duration);
|
|
13916
|
+
} else if (status !== "success" && this.config.onJobError) {
|
|
13917
|
+
await this._executeHook(this.config.onJobError, jobName, lastError, attempt);
|
|
13918
|
+
}
|
|
13919
|
+
this.emit("job_complete", {
|
|
13920
|
+
jobName,
|
|
13921
|
+
executionId,
|
|
13922
|
+
status,
|
|
13923
|
+
duration,
|
|
13924
|
+
result,
|
|
13925
|
+
error: lastError?.message,
|
|
13926
|
+
retryCount: attempt
|
|
13927
|
+
});
|
|
13928
|
+
this.activeJobs.delete(jobName);
|
|
13929
|
+
if (job.enabled) {
|
|
13930
|
+
this._scheduleNextExecution(jobName);
|
|
13931
|
+
}
|
|
13932
|
+
if (lastError && status !== "success") {
|
|
13933
|
+
throw lastError;
|
|
13934
|
+
}
|
|
13935
|
+
} finally {
|
|
13936
|
+
await tryFn(() => this.lockResource.delete(lockId));
|
|
12154
13937
|
}
|
|
12155
13938
|
}
|
|
12156
13939
|
async _persistJobExecution(jobName, executionId, startTime, endTime, duration, status, result, error, retryCount) {
|
|
@@ -12182,6 +13965,7 @@ class SchedulerPlugin extends Plugin {
|
|
|
12182
13965
|
}
|
|
12183
13966
|
/**
|
|
12184
13967
|
* Manually trigger a job execution
|
|
13968
|
+
* Note: Race conditions are prevented by distributed locking in _executeJob()
|
|
12185
13969
|
*/
|
|
12186
13970
|
async runJob(jobName, context = {}) {
|
|
12187
13971
|
const job = this.jobs.get(jobName);
|
|
@@ -12267,12 +14051,15 @@ class SchedulerPlugin extends Plugin {
|
|
|
12267
14051
|
return [];
|
|
12268
14052
|
}
|
|
12269
14053
|
const { limit = 50, status = null } = options;
|
|
12270
|
-
const
|
|
12271
|
-
|
|
12272
|
-
|
|
12273
|
-
|
|
12274
|
-
|
|
12275
|
-
|
|
14054
|
+
const queryParams = {
|
|
14055
|
+
jobName
|
|
14056
|
+
// Uses byJob partition for efficient lookup
|
|
14057
|
+
};
|
|
14058
|
+
if (status) {
|
|
14059
|
+
queryParams.status = status;
|
|
14060
|
+
}
|
|
14061
|
+
const [ok, err, history] = await tryFn(
|
|
14062
|
+
() => this.database.resource(this.config.jobHistoryResource).query(queryParams)
|
|
12276
14063
|
);
|
|
12277
14064
|
if (!ok) {
|
|
12278
14065
|
if (this.config.verbose) {
|
|
@@ -12280,11 +14067,7 @@ class SchedulerPlugin extends Plugin {
|
|
|
12280
14067
|
}
|
|
12281
14068
|
return [];
|
|
12282
14069
|
}
|
|
12283
|
-
let filtered =
|
|
12284
|
-
if (status) {
|
|
12285
|
-
filtered = filtered.filter((h) => h.status === status);
|
|
12286
|
-
}
|
|
12287
|
-
filtered = filtered.sort((a, b) => b.startTime - a.startTime).slice(0, limit);
|
|
14070
|
+
let filtered = history.sort((a, b) => b.startTime - a.startTime).slice(0, limit);
|
|
12288
14071
|
return filtered.map((h) => {
|
|
12289
14072
|
let result = null;
|
|
12290
14073
|
if (h.result) {
|
|
@@ -12379,8 +14162,7 @@ class SchedulerPlugin extends Plugin {
|
|
|
12379
14162
|
clearTimeout(timer);
|
|
12380
14163
|
}
|
|
12381
14164
|
this.timers.clear();
|
|
12382
|
-
|
|
12383
|
-
if (!isTestEnvironment && this.activeJobs.size > 0) {
|
|
14165
|
+
if (!this._isTestEnvironment() && this.activeJobs.size > 0) {
|
|
12384
14166
|
if (this.config.verbose) {
|
|
12385
14167
|
console.log(`[SchedulerPlugin] Waiting for ${this.activeJobs.size} active jobs to complete...`);
|
|
12386
14168
|
}
|
|
@@ -12393,7 +14175,7 @@ class SchedulerPlugin extends Plugin {
|
|
|
12393
14175
|
console.warn(`[SchedulerPlugin] ${this.activeJobs.size} jobs still running after timeout`);
|
|
12394
14176
|
}
|
|
12395
14177
|
}
|
|
12396
|
-
if (
|
|
14178
|
+
if (this._isTestEnvironment()) {
|
|
12397
14179
|
this.activeJobs.clear();
|
|
12398
14180
|
}
|
|
12399
14181
|
}
|
|
@@ -12414,14 +14196,14 @@ class StateMachinePlugin extends Plugin {
|
|
|
12414
14196
|
actions: options.actions || {},
|
|
12415
14197
|
guards: options.guards || {},
|
|
12416
14198
|
persistTransitions: options.persistTransitions !== false,
|
|
12417
|
-
transitionLogResource: options.transitionLogResource || "
|
|
12418
|
-
stateResource: options.stateResource || "
|
|
12419
|
-
|
|
12420
|
-
|
|
14199
|
+
transitionLogResource: options.transitionLogResource || "plg_state_transitions",
|
|
14200
|
+
stateResource: options.stateResource || "plg_entity_states",
|
|
14201
|
+
retryAttempts: options.retryAttempts || 3,
|
|
14202
|
+
retryDelay: options.retryDelay || 100,
|
|
14203
|
+
verbose: options.verbose || false
|
|
12421
14204
|
};
|
|
12422
14205
|
this.database = null;
|
|
12423
14206
|
this.machines = /* @__PURE__ */ new Map();
|
|
12424
|
-
this.stateStorage = /* @__PURE__ */ new Map();
|
|
12425
14207
|
this._validateConfiguration();
|
|
12426
14208
|
}
|
|
12427
14209
|
_validateConfiguration() {
|
|
@@ -12562,43 +14344,55 @@ class StateMachinePlugin extends Plugin {
|
|
|
12562
14344
|
machine.currentStates.set(entityId, toState);
|
|
12563
14345
|
if (this.config.persistTransitions) {
|
|
12564
14346
|
const transitionId = `${machineId}_${entityId}_${timestamp}`;
|
|
12565
|
-
|
|
12566
|
-
|
|
12567
|
-
|
|
12568
|
-
|
|
12569
|
-
|
|
12570
|
-
|
|
12571
|
-
|
|
12572
|
-
|
|
12573
|
-
|
|
12574
|
-
|
|
12575
|
-
|
|
12576
|
-
|
|
12577
|
-
|
|
12578
|
-
|
|
14347
|
+
let logOk = false;
|
|
14348
|
+
let lastLogErr;
|
|
14349
|
+
for (let attempt = 0; attempt < this.config.retryAttempts; attempt++) {
|
|
14350
|
+
const [ok, err] = await tryFn(
|
|
14351
|
+
() => this.database.resource(this.config.transitionLogResource).insert({
|
|
14352
|
+
id: transitionId,
|
|
14353
|
+
machineId,
|
|
14354
|
+
entityId,
|
|
14355
|
+
fromState,
|
|
14356
|
+
toState,
|
|
14357
|
+
event,
|
|
14358
|
+
context,
|
|
14359
|
+
timestamp,
|
|
14360
|
+
createdAt: now.slice(0, 10)
|
|
14361
|
+
// YYYY-MM-DD for partitioning
|
|
14362
|
+
})
|
|
14363
|
+
);
|
|
14364
|
+
if (ok) {
|
|
14365
|
+
logOk = true;
|
|
14366
|
+
break;
|
|
14367
|
+
}
|
|
14368
|
+
lastLogErr = err;
|
|
14369
|
+
if (attempt < this.config.retryAttempts - 1) {
|
|
14370
|
+
const delay = this.config.retryDelay * Math.pow(2, attempt);
|
|
14371
|
+
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
14372
|
+
}
|
|
14373
|
+
}
|
|
12579
14374
|
if (!logOk && this.config.verbose) {
|
|
12580
|
-
console.warn(`[StateMachinePlugin] Failed to log transition:`,
|
|
14375
|
+
console.warn(`[StateMachinePlugin] Failed to log transition after ${this.config.retryAttempts} attempts:`, lastLogErr.message);
|
|
12581
14376
|
}
|
|
12582
14377
|
const stateId = `${machineId}_${entityId}`;
|
|
12583
|
-
const
|
|
12584
|
-
|
|
12585
|
-
|
|
12586
|
-
|
|
12587
|
-
|
|
12588
|
-
|
|
12589
|
-
|
|
12590
|
-
|
|
12591
|
-
|
|
12592
|
-
|
|
12593
|
-
|
|
12594
|
-
|
|
12595
|
-
|
|
12596
|
-
|
|
12597
|
-
|
|
14378
|
+
const stateData = {
|
|
14379
|
+
machineId,
|
|
14380
|
+
entityId,
|
|
14381
|
+
currentState: toState,
|
|
14382
|
+
context,
|
|
14383
|
+
lastTransition: transitionId,
|
|
14384
|
+
updatedAt: now
|
|
14385
|
+
};
|
|
14386
|
+
const [updateOk] = await tryFn(
|
|
14387
|
+
() => this.database.resource(this.config.stateResource).update(stateId, stateData)
|
|
14388
|
+
);
|
|
14389
|
+
if (!updateOk) {
|
|
14390
|
+
const [insertOk, insertErr] = await tryFn(
|
|
14391
|
+
() => this.database.resource(this.config.stateResource).insert({ id: stateId, ...stateData })
|
|
14392
|
+
);
|
|
14393
|
+
if (!insertOk && this.config.verbose) {
|
|
14394
|
+
console.warn(`[StateMachinePlugin] Failed to upsert state:`, insertErr.message);
|
|
12598
14395
|
}
|
|
12599
|
-
});
|
|
12600
|
-
if (!stateOk && this.config.verbose) {
|
|
12601
|
-
console.warn(`[StateMachinePlugin] Failed to update state:`, stateErr.message);
|
|
12602
14396
|
}
|
|
12603
14397
|
}
|
|
12604
14398
|
}
|
|
@@ -12629,8 +14423,9 @@ class StateMachinePlugin extends Plugin {
|
|
|
12629
14423
|
}
|
|
12630
14424
|
/**
|
|
12631
14425
|
* Get valid events for current state
|
|
14426
|
+
* Can accept either a state name (sync) or entityId (async to fetch latest state)
|
|
12632
14427
|
*/
|
|
12633
|
-
getValidEvents(machineId, stateOrEntityId) {
|
|
14428
|
+
async getValidEvents(machineId, stateOrEntityId) {
|
|
12634
14429
|
const machine = this.machines.get(machineId);
|
|
12635
14430
|
if (!machine) {
|
|
12636
14431
|
throw new Error(`State machine '${machineId}' not found`);
|
|
@@ -12639,7 +14434,7 @@ class StateMachinePlugin extends Plugin {
|
|
|
12639
14434
|
if (machine.config.states[stateOrEntityId]) {
|
|
12640
14435
|
state = stateOrEntityId;
|
|
12641
14436
|
} else {
|
|
12642
|
-
state =
|
|
14437
|
+
state = await this.getState(machineId, stateOrEntityId);
|
|
12643
14438
|
}
|
|
12644
14439
|
const stateConfig = machine.config.states[state];
|
|
12645
14440
|
return stateConfig && stateConfig.on ? Object.keys(stateConfig.on) : [];
|
|
@@ -12653,9 +14448,10 @@ class StateMachinePlugin extends Plugin {
|
|
|
12653
14448
|
}
|
|
12654
14449
|
const { limit = 50, offset = 0 } = options;
|
|
12655
14450
|
const [ok, err, transitions] = await tryFn(
|
|
12656
|
-
() => this.database.resource(this.config.transitionLogResource).
|
|
12657
|
-
|
|
12658
|
-
|
|
14451
|
+
() => this.database.resource(this.config.transitionLogResource).query({
|
|
14452
|
+
machineId,
|
|
14453
|
+
entityId
|
|
14454
|
+
}, {
|
|
12659
14455
|
limit,
|
|
12660
14456
|
offset
|
|
12661
14457
|
})
|
|
@@ -12666,8 +14462,8 @@ class StateMachinePlugin extends Plugin {
|
|
|
12666
14462
|
}
|
|
12667
14463
|
return [];
|
|
12668
14464
|
}
|
|
12669
|
-
const
|
|
12670
|
-
return
|
|
14465
|
+
const sorted = (transitions || []).sort((a, b) => b.timestamp - a.timestamp);
|
|
14466
|
+
return sorted.map((t) => ({
|
|
12671
14467
|
from: t.fromState,
|
|
12672
14468
|
to: t.toState,
|
|
12673
14469
|
event: t.event,
|
|
@@ -12688,15 +14484,20 @@ class StateMachinePlugin extends Plugin {
|
|
|
12688
14484
|
if (this.config.persistTransitions) {
|
|
12689
14485
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
12690
14486
|
const stateId = `${machineId}_${entityId}`;
|
|
12691
|
-
await
|
|
12692
|
-
|
|
12693
|
-
|
|
12694
|
-
|
|
12695
|
-
|
|
12696
|
-
|
|
12697
|
-
|
|
12698
|
-
|
|
12699
|
-
|
|
14487
|
+
const [ok, err] = await tryFn(
|
|
14488
|
+
() => this.database.resource(this.config.stateResource).insert({
|
|
14489
|
+
id: stateId,
|
|
14490
|
+
machineId,
|
|
14491
|
+
entityId,
|
|
14492
|
+
currentState: initialState,
|
|
14493
|
+
context,
|
|
14494
|
+
lastTransition: null,
|
|
14495
|
+
updatedAt: now
|
|
14496
|
+
})
|
|
14497
|
+
);
|
|
14498
|
+
if (!ok && err && !err.message?.includes("already exists")) {
|
|
14499
|
+
throw new Error(`Failed to initialize entity state: ${err.message}`);
|
|
14500
|
+
}
|
|
12700
14501
|
}
|
|
12701
14502
|
const initialStateConfig = machine.config.states[initialState];
|
|
12702
14503
|
if (initialStateConfig && initialStateConfig.entry) {
|
|
@@ -12761,7 +14562,6 @@ class StateMachinePlugin extends Plugin {
|
|
|
12761
14562
|
}
|
|
12762
14563
|
async stop() {
|
|
12763
14564
|
this.machines.clear();
|
|
12764
|
-
this.stateStorage.clear();
|
|
12765
14565
|
}
|
|
12766
14566
|
async cleanup() {
|
|
12767
14567
|
await this.stop();
|
|
@@ -12785,6 +14585,7 @@ exports.Database = Database;
|
|
|
12785
14585
|
exports.DatabaseError = DatabaseError;
|
|
12786
14586
|
exports.EncryptionError = EncryptionError;
|
|
12787
14587
|
exports.ErrorMap = ErrorMap;
|
|
14588
|
+
exports.EventualConsistencyPlugin = EventualConsistencyPlugin;
|
|
12788
14589
|
exports.FullTextPlugin = FullTextPlugin;
|
|
12789
14590
|
exports.InvalidResourceItem = InvalidResourceItem;
|
|
12790
14591
|
exports.MetricsPlugin = MetricsPlugin;
|
|
@@ -12804,6 +14605,7 @@ exports.ResourceIdsReader = ResourceIdsReader;
|
|
|
12804
14605
|
exports.ResourceNotFound = ResourceNotFound;
|
|
12805
14606
|
exports.ResourceReader = ResourceReader;
|
|
12806
14607
|
exports.ResourceWriter = ResourceWriter;
|
|
14608
|
+
exports.S3QueuePlugin = S3QueuePlugin;
|
|
12807
14609
|
exports.S3db = Database;
|
|
12808
14610
|
exports.S3dbError = S3dbError;
|
|
12809
14611
|
exports.SchedulerPlugin = SchedulerPlugin;
|