s3db.js 10.0.0 → 10.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -1
- package/dist/s3db.cjs.js +2204 -612
- package/dist/s3db.cjs.js.map +1 -1
- package/dist/s3db.d.ts +70 -3
- package/dist/s3db.es.js +2203 -613
- package/dist/s3db.es.js.map +1 -1
- package/package.json +1 -1
- package/src/client.class.js +6 -5
- package/src/plugins/audit.plugin.js +8 -6
- package/src/plugins/backup.plugin.js +383 -106
- package/src/plugins/cache.plugin.js +203 -150
- package/src/plugins/eventual-consistency.plugin.js +609 -206
- package/src/plugins/fulltext.plugin.js +6 -6
- package/src/plugins/index.js +1 -0
- package/src/plugins/metrics.plugin.js +13 -13
- package/src/plugins/queue-consumer.plugin.js +4 -2
- package/src/plugins/replicator.plugin.js +108 -70
- package/src/plugins/replicators/s3db-replicator.class.js +7 -3
- package/src/plugins/replicators/sqs-replicator.class.js +11 -3
- package/src/plugins/s3-queue.plugin.js +776 -0
- package/src/plugins/scheduler.plugin.js +226 -164
- package/src/plugins/state-machine.plugin.js +109 -81
- package/src/resource.class.js +205 -0
- package/src/s3db.d.ts +70 -3
package/dist/s3db.es.js
CHANGED
|
@@ -6,11 +6,12 @@ import { pipeline } from 'stream/promises';
|
|
|
6
6
|
import path, { join } from 'path';
|
|
7
7
|
import crypto, { createHash } from 'crypto';
|
|
8
8
|
import zlib from 'node:zlib';
|
|
9
|
+
import os from 'os';
|
|
10
|
+
import jsonStableStringify from 'json-stable-stringify';
|
|
9
11
|
import { Transform, Writable } from 'stream';
|
|
10
12
|
import { PromisePool } from '@supercharge/promise-pool';
|
|
11
13
|
import { ReadableStream } from 'node:stream/web';
|
|
12
14
|
import { chunk, merge, isString, isEmpty, invert, uniq, cloneDeep, get, set, isObject, isFunction } from 'lodash-es';
|
|
13
|
-
import jsonStableStringify from 'json-stable-stringify';
|
|
14
15
|
import { Agent } from 'http';
|
|
15
16
|
import { Agent as Agent$1 } from 'https';
|
|
16
17
|
import { NodeHttpHandler } from '@smithy/node-http-handler';
|
|
@@ -838,7 +839,7 @@ class AuditPlugin extends Plugin {
|
|
|
838
839
|
}
|
|
839
840
|
async onSetup() {
|
|
840
841
|
const [ok, err, auditResource] = await tryFn(() => this.database.createResource({
|
|
841
|
-
name: "
|
|
842
|
+
name: "plg_audits",
|
|
842
843
|
attributes: {
|
|
843
844
|
id: "string|required",
|
|
844
845
|
resourceName: "string|required",
|
|
@@ -854,15 +855,15 @@ class AuditPlugin extends Plugin {
|
|
|
854
855
|
},
|
|
855
856
|
behavior: "body-overflow"
|
|
856
857
|
}));
|
|
857
|
-
this.auditResource = ok ? auditResource : this.database.resources.
|
|
858
|
+
this.auditResource = ok ? auditResource : this.database.resources.plg_audits || null;
|
|
858
859
|
if (!ok && !this.auditResource) return;
|
|
859
860
|
this.database.addHook("afterCreateResource", (context) => {
|
|
860
|
-
if (context.resource.name !== "
|
|
861
|
+
if (context.resource.name !== "plg_audits") {
|
|
861
862
|
this.setupResourceAuditing(context.resource);
|
|
862
863
|
}
|
|
863
864
|
});
|
|
864
865
|
for (const resource of Object.values(this.database.resources)) {
|
|
865
|
-
if (resource.name !== "
|
|
866
|
+
if (resource.name !== "plg_audits") {
|
|
866
867
|
this.setupResourceAuditing(resource);
|
|
867
868
|
}
|
|
868
869
|
}
|
|
@@ -1882,11 +1883,10 @@ function validateBackupConfig(driver, config = {}) {
|
|
|
1882
1883
|
class BackupPlugin extends Plugin {
|
|
1883
1884
|
constructor(options = {}) {
|
|
1884
1885
|
super();
|
|
1885
|
-
this.driverName = options.driver || "filesystem";
|
|
1886
|
-
this.driverConfig = options.config || {};
|
|
1887
1886
|
this.config = {
|
|
1888
|
-
//
|
|
1889
|
-
|
|
1887
|
+
// Driver configuration
|
|
1888
|
+
driver: options.driver || "filesystem",
|
|
1889
|
+
driverConfig: options.config || {},
|
|
1890
1890
|
// Scheduling configuration
|
|
1891
1891
|
schedule: options.schedule || {},
|
|
1892
1892
|
// Retention policy (Grandfather-Father-Son)
|
|
@@ -1904,8 +1904,8 @@ class BackupPlugin extends Plugin {
|
|
|
1904
1904
|
parallelism: options.parallelism || 4,
|
|
1905
1905
|
include: options.include || null,
|
|
1906
1906
|
exclude: options.exclude || [],
|
|
1907
|
-
backupMetadataResource: options.backupMetadataResource || "
|
|
1908
|
-
tempDir: options.tempDir || "
|
|
1907
|
+
backupMetadataResource: options.backupMetadataResource || "plg_backup_metadata",
|
|
1908
|
+
tempDir: options.tempDir || path.join(os.tmpdir(), "s3db", "backups"),
|
|
1909
1909
|
verbose: options.verbose || false,
|
|
1910
1910
|
// Hooks
|
|
1911
1911
|
onBackupStart: options.onBackupStart || null,
|
|
@@ -1917,32 +1917,9 @@ class BackupPlugin extends Plugin {
|
|
|
1917
1917
|
};
|
|
1918
1918
|
this.driver = null;
|
|
1919
1919
|
this.activeBackups = /* @__PURE__ */ new Set();
|
|
1920
|
-
this.
|
|
1921
|
-
validateBackupConfig(this.driverName, this.driverConfig);
|
|
1920
|
+
validateBackupConfig(this.config.driver, this.config.driverConfig);
|
|
1922
1921
|
this._validateConfiguration();
|
|
1923
1922
|
}
|
|
1924
|
-
/**
|
|
1925
|
-
* Convert legacy destinations format to multi driver format
|
|
1926
|
-
*/
|
|
1927
|
-
_handleLegacyDestinations() {
|
|
1928
|
-
if (this.config.destinations && Array.isArray(this.config.destinations)) {
|
|
1929
|
-
this.driverName = "multi";
|
|
1930
|
-
this.driverConfig = {
|
|
1931
|
-
strategy: "all",
|
|
1932
|
-
destinations: this.config.destinations.map((dest) => {
|
|
1933
|
-
const { type, ...config } = dest;
|
|
1934
|
-
return {
|
|
1935
|
-
driver: type,
|
|
1936
|
-
config
|
|
1937
|
-
};
|
|
1938
|
-
})
|
|
1939
|
-
};
|
|
1940
|
-
this.config.destinations = null;
|
|
1941
|
-
if (this.config.verbose) {
|
|
1942
|
-
console.log("[BackupPlugin] Converted legacy destinations format to multi driver");
|
|
1943
|
-
}
|
|
1944
|
-
}
|
|
1945
|
-
}
|
|
1946
1923
|
_validateConfiguration() {
|
|
1947
1924
|
if (this.config.encryption && (!this.config.encryption.key || !this.config.encryption.algorithm)) {
|
|
1948
1925
|
throw new Error("BackupPlugin: Encryption requires both key and algorithm");
|
|
@@ -1952,7 +1929,7 @@ class BackupPlugin extends Plugin {
|
|
|
1952
1929
|
}
|
|
1953
1930
|
}
|
|
1954
1931
|
async onSetup() {
|
|
1955
|
-
this.driver = createBackupDriver(this.
|
|
1932
|
+
this.driver = createBackupDriver(this.config.driver, this.config.driverConfig);
|
|
1956
1933
|
await this.driver.setup(this.database);
|
|
1957
1934
|
await mkdir(this.config.tempDir, { recursive: true });
|
|
1958
1935
|
await this._createBackupMetadataResource();
|
|
@@ -2000,6 +1977,9 @@ class BackupPlugin extends Plugin {
|
|
|
2000
1977
|
async backup(type = "full", options = {}) {
|
|
2001
1978
|
const backupId = this._generateBackupId(type);
|
|
2002
1979
|
const startTime = Date.now();
|
|
1980
|
+
if (this.activeBackups.has(backupId)) {
|
|
1981
|
+
throw new Error(`Backup '${backupId}' is already in progress`);
|
|
1982
|
+
}
|
|
2003
1983
|
try {
|
|
2004
1984
|
this.activeBackups.add(backupId);
|
|
2005
1985
|
if (this.config.onBackupStart) {
|
|
@@ -2015,16 +1995,9 @@ class BackupPlugin extends Plugin {
|
|
|
2015
1995
|
if (exportedFiles.length === 0) {
|
|
2016
1996
|
throw new Error("No resources were exported for backup");
|
|
2017
1997
|
}
|
|
2018
|
-
|
|
2019
|
-
|
|
2020
|
-
|
|
2021
|
-
finalPath = path.join(tempBackupDir, `${backupId}.tar.gz`);
|
|
2022
|
-
totalSize = await this._createCompressedArchive(exportedFiles, finalPath);
|
|
2023
|
-
} else {
|
|
2024
|
-
finalPath = exportedFiles[0];
|
|
2025
|
-
const [statOk, , stats] = await tryFn(() => stat(finalPath));
|
|
2026
|
-
totalSize = statOk ? stats.size : 0;
|
|
2027
|
-
}
|
|
1998
|
+
const archiveExtension = this.config.compression !== "none" ? ".tar.gz" : ".json";
|
|
1999
|
+
const finalPath = path.join(tempBackupDir, `${backupId}${archiveExtension}`);
|
|
2000
|
+
const totalSize = await this._createArchive(exportedFiles, finalPath, this.config.compression);
|
|
2028
2001
|
const checksum = await this._generateChecksum(finalPath);
|
|
2029
2002
|
const uploadResult = await this.driver.upload(finalPath, backupId, manifest);
|
|
2030
2003
|
if (this.config.verification) {
|
|
@@ -2133,15 +2106,35 @@ class BackupPlugin extends Plugin {
|
|
|
2133
2106
|
for (const resourceName of resourceNames) {
|
|
2134
2107
|
const resource = this.database.resources[resourceName];
|
|
2135
2108
|
if (!resource) {
|
|
2136
|
-
|
|
2109
|
+
if (this.config.verbose) {
|
|
2110
|
+
console.warn(`[BackupPlugin] Resource '${resourceName}' not found, skipping`);
|
|
2111
|
+
}
|
|
2137
2112
|
continue;
|
|
2138
2113
|
}
|
|
2139
2114
|
const exportPath = path.join(tempDir, `${resourceName}.json`);
|
|
2140
2115
|
let records;
|
|
2141
2116
|
if (type === "incremental") {
|
|
2142
|
-
const
|
|
2117
|
+
const [lastBackupOk, , lastBackups] = await tryFn(
|
|
2118
|
+
() => this.database.resource(this.config.backupMetadataResource).list({
|
|
2119
|
+
filter: {
|
|
2120
|
+
status: "completed",
|
|
2121
|
+
type: { $in: ["full", "incremental"] }
|
|
2122
|
+
},
|
|
2123
|
+
sort: { timestamp: -1 },
|
|
2124
|
+
limit: 1
|
|
2125
|
+
})
|
|
2126
|
+
);
|
|
2127
|
+
let sinceTimestamp;
|
|
2128
|
+
if (lastBackupOk && lastBackups && lastBackups.length > 0) {
|
|
2129
|
+
sinceTimestamp = new Date(lastBackups[0].timestamp);
|
|
2130
|
+
} else {
|
|
2131
|
+
sinceTimestamp = new Date(Date.now() - 24 * 60 * 60 * 1e3);
|
|
2132
|
+
}
|
|
2133
|
+
if (this.config.verbose) {
|
|
2134
|
+
console.log(`[BackupPlugin] Incremental backup for '${resourceName}' since ${sinceTimestamp.toISOString()}`);
|
|
2135
|
+
}
|
|
2143
2136
|
records = await resource.list({
|
|
2144
|
-
filter: { updatedAt: { ">":
|
|
2137
|
+
filter: { updatedAt: { ">": sinceTimestamp.toISOString() } }
|
|
2145
2138
|
});
|
|
2146
2139
|
} else {
|
|
2147
2140
|
records = await resource.list();
|
|
@@ -2161,29 +2154,57 @@ class BackupPlugin extends Plugin {
|
|
|
2161
2154
|
}
|
|
2162
2155
|
return exportedFiles;
|
|
2163
2156
|
}
|
|
2164
|
-
async
|
|
2165
|
-
const
|
|
2166
|
-
|
|
2157
|
+
async _createArchive(files, targetPath, compressionType) {
|
|
2158
|
+
const archive = {
|
|
2159
|
+
version: "1.0",
|
|
2160
|
+
created: (/* @__PURE__ */ new Date()).toISOString(),
|
|
2161
|
+
files: []
|
|
2162
|
+
};
|
|
2167
2163
|
let totalSize = 0;
|
|
2168
|
-
|
|
2169
|
-
|
|
2170
|
-
|
|
2171
|
-
|
|
2172
|
-
|
|
2173
|
-
yield content;
|
|
2164
|
+
for (const filePath of files) {
|
|
2165
|
+
const [readOk, readErr, content] = await tryFn(() => readFile(filePath, "utf8"));
|
|
2166
|
+
if (!readOk) {
|
|
2167
|
+
if (this.config.verbose) {
|
|
2168
|
+
console.warn(`[BackupPlugin] Failed to read ${filePath}: ${readErr?.message}`);
|
|
2174
2169
|
}
|
|
2175
|
-
|
|
2176
|
-
|
|
2177
|
-
|
|
2178
|
-
|
|
2170
|
+
continue;
|
|
2171
|
+
}
|
|
2172
|
+
const fileName = path.basename(filePath);
|
|
2173
|
+
totalSize += content.length;
|
|
2174
|
+
archive.files.push({
|
|
2175
|
+
name: fileName,
|
|
2176
|
+
size: content.length,
|
|
2177
|
+
content
|
|
2178
|
+
});
|
|
2179
|
+
}
|
|
2180
|
+
const archiveJson = JSON.stringify(archive);
|
|
2181
|
+
if (compressionType === "none") {
|
|
2182
|
+
await writeFile(targetPath, archiveJson, "utf8");
|
|
2183
|
+
} else {
|
|
2184
|
+
const output = createWriteStream(targetPath);
|
|
2185
|
+
const gzip = zlib.createGzip({ level: 6 });
|
|
2186
|
+
await pipeline(
|
|
2187
|
+
async function* () {
|
|
2188
|
+
yield Buffer.from(archiveJson, "utf8");
|
|
2189
|
+
},
|
|
2190
|
+
gzip,
|
|
2191
|
+
output
|
|
2192
|
+
);
|
|
2193
|
+
}
|
|
2179
2194
|
const [statOk, , stats] = await tryFn(() => stat(targetPath));
|
|
2180
2195
|
return statOk ? stats.size : totalSize;
|
|
2181
2196
|
}
|
|
2182
2197
|
async _generateChecksum(filePath) {
|
|
2183
|
-
const
|
|
2184
|
-
|
|
2185
|
-
|
|
2186
|
-
|
|
2198
|
+
const [ok, err, result] = await tryFn(async () => {
|
|
2199
|
+
const hash = crypto.createHash("sha256");
|
|
2200
|
+
const stream = createReadStream(filePath);
|
|
2201
|
+
await pipeline(stream, hash);
|
|
2202
|
+
return hash.digest("hex");
|
|
2203
|
+
});
|
|
2204
|
+
if (!ok) {
|
|
2205
|
+
throw new Error(`Failed to generate checksum for ${filePath}: ${err?.message}`);
|
|
2206
|
+
}
|
|
2207
|
+
return result;
|
|
2187
2208
|
}
|
|
2188
2209
|
async _cleanupTempFiles(tempDir) {
|
|
2189
2210
|
const [ok] = await tryFn(
|
|
@@ -2245,7 +2266,109 @@ class BackupPlugin extends Plugin {
|
|
|
2245
2266
|
}
|
|
2246
2267
|
async _restoreFromBackup(backupPath, options) {
|
|
2247
2268
|
const restoredResources = [];
|
|
2248
|
-
|
|
2269
|
+
try {
|
|
2270
|
+
let archiveData = "";
|
|
2271
|
+
if (this.config.compression !== "none") {
|
|
2272
|
+
const input = createReadStream(backupPath);
|
|
2273
|
+
const gunzip = zlib.createGunzip();
|
|
2274
|
+
const chunks = [];
|
|
2275
|
+
await new Promise((resolve, reject) => {
|
|
2276
|
+
input.pipe(gunzip).on("data", (chunk) => chunks.push(chunk)).on("end", resolve).on("error", reject);
|
|
2277
|
+
});
|
|
2278
|
+
archiveData = Buffer.concat(chunks).toString("utf8");
|
|
2279
|
+
} else {
|
|
2280
|
+
archiveData = await readFile(backupPath, "utf8");
|
|
2281
|
+
}
|
|
2282
|
+
let archive;
|
|
2283
|
+
try {
|
|
2284
|
+
archive = JSON.parse(archiveData);
|
|
2285
|
+
} catch (parseError) {
|
|
2286
|
+
throw new Error(`Failed to parse backup archive: ${parseError.message}`);
|
|
2287
|
+
}
|
|
2288
|
+
if (!archive || typeof archive !== "object") {
|
|
2289
|
+
throw new Error("Invalid backup archive: not a valid JSON object");
|
|
2290
|
+
}
|
|
2291
|
+
if (!archive.version || !archive.files) {
|
|
2292
|
+
throw new Error("Invalid backup archive format: missing version or files array");
|
|
2293
|
+
}
|
|
2294
|
+
if (this.config.verbose) {
|
|
2295
|
+
console.log(`[BackupPlugin] Restoring ${archive.files.length} files from backup`);
|
|
2296
|
+
}
|
|
2297
|
+
for (const file of archive.files) {
|
|
2298
|
+
try {
|
|
2299
|
+
const resourceData = JSON.parse(file.content);
|
|
2300
|
+
if (!resourceData.resourceName || !resourceData.definition) {
|
|
2301
|
+
if (this.config.verbose) {
|
|
2302
|
+
console.warn(`[BackupPlugin] Skipping invalid file: ${file.name}`);
|
|
2303
|
+
}
|
|
2304
|
+
continue;
|
|
2305
|
+
}
|
|
2306
|
+
const resourceName = resourceData.resourceName;
|
|
2307
|
+
if (options.resources && !options.resources.includes(resourceName)) {
|
|
2308
|
+
continue;
|
|
2309
|
+
}
|
|
2310
|
+
let resource = this.database.resources[resourceName];
|
|
2311
|
+
if (!resource) {
|
|
2312
|
+
if (this.config.verbose) {
|
|
2313
|
+
console.log(`[BackupPlugin] Creating resource '${resourceName}'`);
|
|
2314
|
+
}
|
|
2315
|
+
const [createOk, createErr] = await tryFn(
|
|
2316
|
+
() => this.database.createResource(resourceData.definition)
|
|
2317
|
+
);
|
|
2318
|
+
if (!createOk) {
|
|
2319
|
+
if (this.config.verbose) {
|
|
2320
|
+
console.warn(`[BackupPlugin] Failed to create resource '${resourceName}': ${createErr?.message}`);
|
|
2321
|
+
}
|
|
2322
|
+
continue;
|
|
2323
|
+
}
|
|
2324
|
+
resource = this.database.resources[resourceName];
|
|
2325
|
+
}
|
|
2326
|
+
if (resourceData.records && Array.isArray(resourceData.records)) {
|
|
2327
|
+
const mode = options.mode || "merge";
|
|
2328
|
+
if (mode === "replace") {
|
|
2329
|
+
const ids = await resource.listIds();
|
|
2330
|
+
for (const id of ids) {
|
|
2331
|
+
await resource.delete(id);
|
|
2332
|
+
}
|
|
2333
|
+
}
|
|
2334
|
+
let insertedCount = 0;
|
|
2335
|
+
for (const record of resourceData.records) {
|
|
2336
|
+
const [insertOk] = await tryFn(async () => {
|
|
2337
|
+
if (mode === "skip") {
|
|
2338
|
+
const existing = await resource.get(record.id);
|
|
2339
|
+
if (existing) {
|
|
2340
|
+
return false;
|
|
2341
|
+
}
|
|
2342
|
+
}
|
|
2343
|
+
await resource.insert(record);
|
|
2344
|
+
return true;
|
|
2345
|
+
});
|
|
2346
|
+
if (insertOk) {
|
|
2347
|
+
insertedCount++;
|
|
2348
|
+
}
|
|
2349
|
+
}
|
|
2350
|
+
restoredResources.push({
|
|
2351
|
+
name: resourceName,
|
|
2352
|
+
recordsRestored: insertedCount,
|
|
2353
|
+
totalRecords: resourceData.records.length
|
|
2354
|
+
});
|
|
2355
|
+
if (this.config.verbose) {
|
|
2356
|
+
console.log(`[BackupPlugin] Restored ${insertedCount}/${resourceData.records.length} records to '${resourceName}'`);
|
|
2357
|
+
}
|
|
2358
|
+
}
|
|
2359
|
+
} catch (fileError) {
|
|
2360
|
+
if (this.config.verbose) {
|
|
2361
|
+
console.warn(`[BackupPlugin] Error processing file ${file.name}: ${fileError.message}`);
|
|
2362
|
+
}
|
|
2363
|
+
}
|
|
2364
|
+
}
|
|
2365
|
+
return restoredResources;
|
|
2366
|
+
} catch (error) {
|
|
2367
|
+
if (this.config.verbose) {
|
|
2368
|
+
console.error(`[BackupPlugin] Error restoring backup: ${error.message}`);
|
|
2369
|
+
}
|
|
2370
|
+
throw new Error(`Failed to restore backup: ${error.message}`);
|
|
2371
|
+
}
|
|
2249
2372
|
}
|
|
2250
2373
|
/**
|
|
2251
2374
|
* List available backups
|
|
@@ -2289,6 +2412,90 @@ class BackupPlugin extends Plugin {
|
|
|
2289
2412
|
return ok ? backup : null;
|
|
2290
2413
|
}
|
|
2291
2414
|
async _cleanupOldBackups() {
|
|
2415
|
+
try {
|
|
2416
|
+
const [listOk, , allBackups] = await tryFn(
|
|
2417
|
+
() => this.database.resource(this.config.backupMetadataResource).list({
|
|
2418
|
+
filter: { status: "completed" },
|
|
2419
|
+
sort: { timestamp: -1 }
|
|
2420
|
+
})
|
|
2421
|
+
);
|
|
2422
|
+
if (!listOk || !allBackups || allBackups.length === 0) {
|
|
2423
|
+
return;
|
|
2424
|
+
}
|
|
2425
|
+
const now = Date.now();
|
|
2426
|
+
const msPerDay = 24 * 60 * 60 * 1e3;
|
|
2427
|
+
const msPerWeek = 7 * msPerDay;
|
|
2428
|
+
const msPerMonth = 30 * msPerDay;
|
|
2429
|
+
const msPerYear = 365 * msPerDay;
|
|
2430
|
+
const categorized = {
|
|
2431
|
+
daily: [],
|
|
2432
|
+
weekly: [],
|
|
2433
|
+
monthly: [],
|
|
2434
|
+
yearly: []
|
|
2435
|
+
};
|
|
2436
|
+
for (const backup of allBackups) {
|
|
2437
|
+
const age = now - backup.timestamp;
|
|
2438
|
+
if (age <= msPerDay * this.config.retention.daily) {
|
|
2439
|
+
categorized.daily.push(backup);
|
|
2440
|
+
} else if (age <= msPerWeek * this.config.retention.weekly) {
|
|
2441
|
+
categorized.weekly.push(backup);
|
|
2442
|
+
} else if (age <= msPerMonth * this.config.retention.monthly) {
|
|
2443
|
+
categorized.monthly.push(backup);
|
|
2444
|
+
} else if (age <= msPerYear * this.config.retention.yearly) {
|
|
2445
|
+
categorized.yearly.push(backup);
|
|
2446
|
+
}
|
|
2447
|
+
}
|
|
2448
|
+
const toKeep = /* @__PURE__ */ new Set();
|
|
2449
|
+
categorized.daily.forEach((b) => toKeep.add(b.id));
|
|
2450
|
+
const weeklyByWeek = /* @__PURE__ */ new Map();
|
|
2451
|
+
for (const backup of categorized.weekly) {
|
|
2452
|
+
const weekNum = Math.floor((now - backup.timestamp) / msPerWeek);
|
|
2453
|
+
if (!weeklyByWeek.has(weekNum)) {
|
|
2454
|
+
weeklyByWeek.set(weekNum, backup);
|
|
2455
|
+
toKeep.add(backup.id);
|
|
2456
|
+
}
|
|
2457
|
+
}
|
|
2458
|
+
const monthlyByMonth = /* @__PURE__ */ new Map();
|
|
2459
|
+
for (const backup of categorized.monthly) {
|
|
2460
|
+
const monthNum = Math.floor((now - backup.timestamp) / msPerMonth);
|
|
2461
|
+
if (!monthlyByMonth.has(monthNum)) {
|
|
2462
|
+
monthlyByMonth.set(monthNum, backup);
|
|
2463
|
+
toKeep.add(backup.id);
|
|
2464
|
+
}
|
|
2465
|
+
}
|
|
2466
|
+
const yearlyByYear = /* @__PURE__ */ new Map();
|
|
2467
|
+
for (const backup of categorized.yearly) {
|
|
2468
|
+
const yearNum = Math.floor((now - backup.timestamp) / msPerYear);
|
|
2469
|
+
if (!yearlyByYear.has(yearNum)) {
|
|
2470
|
+
yearlyByYear.set(yearNum, backup);
|
|
2471
|
+
toKeep.add(backup.id);
|
|
2472
|
+
}
|
|
2473
|
+
}
|
|
2474
|
+
const backupsToDelete = allBackups.filter((b) => !toKeep.has(b.id));
|
|
2475
|
+
if (backupsToDelete.length === 0) {
|
|
2476
|
+
return;
|
|
2477
|
+
}
|
|
2478
|
+
if (this.config.verbose) {
|
|
2479
|
+
console.log(`[BackupPlugin] Cleaning up ${backupsToDelete.length} old backups (keeping ${toKeep.size})`);
|
|
2480
|
+
}
|
|
2481
|
+
for (const backup of backupsToDelete) {
|
|
2482
|
+
try {
|
|
2483
|
+
await this.driver.delete(backup.id, backup.driverInfo);
|
|
2484
|
+
await this.database.resource(this.config.backupMetadataResource).delete(backup.id);
|
|
2485
|
+
if (this.config.verbose) {
|
|
2486
|
+
console.log(`[BackupPlugin] Deleted old backup: ${backup.id}`);
|
|
2487
|
+
}
|
|
2488
|
+
} catch (deleteError) {
|
|
2489
|
+
if (this.config.verbose) {
|
|
2490
|
+
console.warn(`[BackupPlugin] Failed to delete backup ${backup.id}: ${deleteError.message}`);
|
|
2491
|
+
}
|
|
2492
|
+
}
|
|
2493
|
+
}
|
|
2494
|
+
} catch (error) {
|
|
2495
|
+
if (this.config.verbose) {
|
|
2496
|
+
console.warn(`[BackupPlugin] Error during cleanup: ${error.message}`);
|
|
2497
|
+
}
|
|
2498
|
+
}
|
|
2292
2499
|
}
|
|
2293
2500
|
async _executeHook(hook, ...args) {
|
|
2294
2501
|
if (typeof hook === "function") {
|
|
@@ -3578,81 +3785,58 @@ class PartitionAwareFilesystemCache extends FilesystemCache {
|
|
|
3578
3785
|
class CachePlugin extends Plugin {
|
|
3579
3786
|
constructor(options = {}) {
|
|
3580
3787
|
super(options);
|
|
3581
|
-
this.
|
|
3582
|
-
|
|
3583
|
-
|
|
3584
|
-
|
|
3585
|
-
|
|
3586
|
-
|
|
3587
|
-
|
|
3588
|
-
|
|
3589
|
-
|
|
3590
|
-
|
|
3591
|
-
|
|
3592
|
-
|
|
3593
|
-
|
|
3594
|
-
|
|
3788
|
+
this.config = {
|
|
3789
|
+
// Driver configuration
|
|
3790
|
+
driver: options.driver || "s3",
|
|
3791
|
+
config: {
|
|
3792
|
+
ttl: options.ttl,
|
|
3793
|
+
maxSize: options.maxSize,
|
|
3794
|
+
...options.config
|
|
3795
|
+
// Driver-specific config (can override ttl/maxSize)
|
|
3796
|
+
},
|
|
3797
|
+
// Resource filtering
|
|
3798
|
+
include: options.include || null,
|
|
3799
|
+
// Array of resource names to cache (null = all)
|
|
3800
|
+
exclude: options.exclude || [],
|
|
3801
|
+
// Array of resource names to exclude
|
|
3802
|
+
// Partition settings
|
|
3803
|
+
includePartitions: options.includePartitions !== false,
|
|
3804
|
+
partitionStrategy: options.partitionStrategy || "hierarchical",
|
|
3805
|
+
partitionAware: options.partitionAware !== false,
|
|
3806
|
+
trackUsage: options.trackUsage !== false,
|
|
3807
|
+
preloadRelated: options.preloadRelated !== false,
|
|
3808
|
+
// Retry configuration
|
|
3809
|
+
retryAttempts: options.retryAttempts || 3,
|
|
3810
|
+
retryDelay: options.retryDelay || 100,
|
|
3811
|
+
// ms
|
|
3812
|
+
// Logging
|
|
3813
|
+
verbose: options.verbose || false
|
|
3595
3814
|
};
|
|
3596
3815
|
}
|
|
3597
3816
|
async setup(database) {
|
|
3598
3817
|
await super.setup(database);
|
|
3599
3818
|
}
|
|
3600
3819
|
async onSetup() {
|
|
3601
|
-
if (this.
|
|
3602
|
-
this.driver = this.
|
|
3603
|
-
} else if (this.
|
|
3604
|
-
|
|
3605
|
-
|
|
3606
|
-
|
|
3607
|
-
...this.config
|
|
3608
|
-
// New config format (medium priority)
|
|
3609
|
-
};
|
|
3610
|
-
if (this.ttl !== void 0) {
|
|
3611
|
-
driverConfig.ttl = this.ttl;
|
|
3612
|
-
}
|
|
3613
|
-
if (this.maxSize !== void 0) {
|
|
3614
|
-
driverConfig.maxSize = this.maxSize;
|
|
3615
|
-
}
|
|
3616
|
-
this.driver = new MemoryCache(driverConfig);
|
|
3617
|
-
} else if (this.driverName === "filesystem") {
|
|
3618
|
-
const driverConfig = {
|
|
3619
|
-
...this.legacyConfig.filesystemOptions,
|
|
3620
|
-
// Legacy support (lowest priority)
|
|
3621
|
-
...this.config
|
|
3622
|
-
// New config format (medium priority)
|
|
3623
|
-
};
|
|
3624
|
-
if (this.ttl !== void 0) {
|
|
3625
|
-
driverConfig.ttl = this.ttl;
|
|
3626
|
-
}
|
|
3627
|
-
if (this.maxSize !== void 0) {
|
|
3628
|
-
driverConfig.maxSize = this.maxSize;
|
|
3629
|
-
}
|
|
3630
|
-
if (this.partitionAware) {
|
|
3820
|
+
if (this.config.driver && typeof this.config.driver === "object") {
|
|
3821
|
+
this.driver = this.config.driver;
|
|
3822
|
+
} else if (this.config.driver === "memory") {
|
|
3823
|
+
this.driver = new MemoryCache(this.config.config);
|
|
3824
|
+
} else if (this.config.driver === "filesystem") {
|
|
3825
|
+
if (this.config.partitionAware) {
|
|
3631
3826
|
this.driver = new PartitionAwareFilesystemCache({
|
|
3632
|
-
partitionStrategy: this.partitionStrategy,
|
|
3633
|
-
trackUsage: this.trackUsage,
|
|
3634
|
-
preloadRelated: this.preloadRelated,
|
|
3635
|
-
...
|
|
3827
|
+
partitionStrategy: this.config.partitionStrategy,
|
|
3828
|
+
trackUsage: this.config.trackUsage,
|
|
3829
|
+
preloadRelated: this.config.preloadRelated,
|
|
3830
|
+
...this.config.config
|
|
3636
3831
|
});
|
|
3637
3832
|
} else {
|
|
3638
|
-
this.driver = new FilesystemCache(
|
|
3833
|
+
this.driver = new FilesystemCache(this.config.config);
|
|
3639
3834
|
}
|
|
3640
3835
|
} else {
|
|
3641
|
-
|
|
3836
|
+
this.driver = new S3Cache({
|
|
3642
3837
|
client: this.database.client,
|
|
3643
|
-
|
|
3644
|
-
|
|
3645
|
-
// Legacy support (lowest priority)
|
|
3646
|
-
...this.config
|
|
3647
|
-
// New config format (medium priority)
|
|
3648
|
-
};
|
|
3649
|
-
if (this.ttl !== void 0) {
|
|
3650
|
-
driverConfig.ttl = this.ttl;
|
|
3651
|
-
}
|
|
3652
|
-
if (this.maxSize !== void 0) {
|
|
3653
|
-
driverConfig.maxSize = this.maxSize;
|
|
3654
|
-
}
|
|
3655
|
-
this.driver = new S3Cache(driverConfig);
|
|
3838
|
+
...this.config.config
|
|
3839
|
+
});
|
|
3656
3840
|
}
|
|
3657
3841
|
this.installDatabaseHooks();
|
|
3658
3842
|
this.installResourceHooks();
|
|
@@ -3662,7 +3846,9 @@ class CachePlugin extends Plugin {
|
|
|
3662
3846
|
*/
|
|
3663
3847
|
installDatabaseHooks() {
|
|
3664
3848
|
this.database.addHook("afterCreateResource", async ({ resource }) => {
|
|
3665
|
-
this.
|
|
3849
|
+
if (this.shouldCacheResource(resource.name)) {
|
|
3850
|
+
this.installResourceHooksForResource(resource);
|
|
3851
|
+
}
|
|
3666
3852
|
});
|
|
3667
3853
|
}
|
|
3668
3854
|
async onStart() {
|
|
@@ -3672,9 +3858,24 @@ class CachePlugin extends Plugin {
|
|
|
3672
3858
|
// Remove the old installDatabaseProxy method
|
|
3673
3859
|
installResourceHooks() {
|
|
3674
3860
|
for (const resource of Object.values(this.database.resources)) {
|
|
3861
|
+
if (!this.shouldCacheResource(resource.name)) {
|
|
3862
|
+
continue;
|
|
3863
|
+
}
|
|
3675
3864
|
this.installResourceHooksForResource(resource);
|
|
3676
3865
|
}
|
|
3677
3866
|
}
|
|
3867
|
+
shouldCacheResource(resourceName) {
|
|
3868
|
+
if (resourceName.startsWith("plg_") && !this.config.include) {
|
|
3869
|
+
return false;
|
|
3870
|
+
}
|
|
3871
|
+
if (this.config.exclude.includes(resourceName)) {
|
|
3872
|
+
return false;
|
|
3873
|
+
}
|
|
3874
|
+
if (this.config.include && !this.config.include.includes(resourceName)) {
|
|
3875
|
+
return false;
|
|
3876
|
+
}
|
|
3877
|
+
return true;
|
|
3878
|
+
}
|
|
3678
3879
|
installResourceHooksForResource(resource) {
|
|
3679
3880
|
if (!this.driver) return;
|
|
3680
3881
|
Object.defineProperty(resource, "cache", {
|
|
@@ -3823,37 +4024,74 @@ class CachePlugin extends Plugin {
|
|
|
3823
4024
|
if (data && data.id) {
|
|
3824
4025
|
const itemSpecificMethods = ["get", "exists", "content", "hasContent"];
|
|
3825
4026
|
for (const method of itemSpecificMethods) {
|
|
3826
|
-
|
|
3827
|
-
|
|
3828
|
-
|
|
3829
|
-
|
|
4027
|
+
const specificKey = await this.generateCacheKey(resource, method, { id: data.id });
|
|
4028
|
+
const [ok2, err2] = await this.clearCacheWithRetry(resource.cache, specificKey);
|
|
4029
|
+
if (!ok2) {
|
|
4030
|
+
this.emit("cache_clear_error", {
|
|
4031
|
+
resource: resource.name,
|
|
4032
|
+
method,
|
|
4033
|
+
id: data.id,
|
|
4034
|
+
error: err2.message
|
|
4035
|
+
});
|
|
4036
|
+
if (this.config.verbose) {
|
|
4037
|
+
console.warn(`[CachePlugin] Failed to clear ${method} cache for ${resource.name}:${data.id}:`, err2.message);
|
|
4038
|
+
}
|
|
3830
4039
|
}
|
|
3831
4040
|
}
|
|
3832
4041
|
if (this.config.includePartitions === true && resource.config?.partitions && Object.keys(resource.config.partitions).length > 0) {
|
|
3833
4042
|
const partitionValues = this.getPartitionValues(data, resource);
|
|
3834
4043
|
for (const [partitionName, values] of Object.entries(partitionValues)) {
|
|
3835
4044
|
if (values && Object.keys(values).length > 0 && Object.values(values).some((v) => v !== null && v !== void 0)) {
|
|
3836
|
-
|
|
3837
|
-
|
|
3838
|
-
|
|
3839
|
-
|
|
4045
|
+
const partitionKeyPrefix = join(keyPrefix, `partition=${partitionName}`);
|
|
4046
|
+
const [ok2, err2] = await this.clearCacheWithRetry(resource.cache, partitionKeyPrefix);
|
|
4047
|
+
if (!ok2) {
|
|
4048
|
+
this.emit("cache_clear_error", {
|
|
4049
|
+
resource: resource.name,
|
|
4050
|
+
partition: partitionName,
|
|
4051
|
+
error: err2.message
|
|
4052
|
+
});
|
|
4053
|
+
if (this.config.verbose) {
|
|
4054
|
+
console.warn(`[CachePlugin] Failed to clear partition cache for ${resource.name}/${partitionName}:`, err2.message);
|
|
4055
|
+
}
|
|
3840
4056
|
}
|
|
3841
4057
|
}
|
|
3842
4058
|
}
|
|
3843
4059
|
}
|
|
3844
4060
|
}
|
|
3845
|
-
|
|
3846
|
-
|
|
3847
|
-
|
|
4061
|
+
const [ok, err] = await this.clearCacheWithRetry(resource.cache, keyPrefix);
|
|
4062
|
+
if (!ok) {
|
|
4063
|
+
this.emit("cache_clear_error", {
|
|
4064
|
+
resource: resource.name,
|
|
4065
|
+
type: "broad",
|
|
4066
|
+
error: err.message
|
|
4067
|
+
});
|
|
4068
|
+
if (this.config.verbose) {
|
|
4069
|
+
console.warn(`[CachePlugin] Failed to clear broad cache for ${resource.name}, trying specific methods:`, err.message);
|
|
4070
|
+
}
|
|
3848
4071
|
const aggregateMethods = ["count", "list", "listIds", "getAll", "page", "query"];
|
|
3849
4072
|
for (const method of aggregateMethods) {
|
|
3850
|
-
|
|
3851
|
-
|
|
3852
|
-
|
|
3853
|
-
|
|
3854
|
-
|
|
4073
|
+
await this.clearCacheWithRetry(resource.cache, `${keyPrefix}/action=${method}`);
|
|
4074
|
+
await this.clearCacheWithRetry(resource.cache, `resource=${resource.name}/action=${method}`);
|
|
4075
|
+
}
|
|
4076
|
+
}
|
|
4077
|
+
}
|
|
4078
|
+
async clearCacheWithRetry(cache, key) {
|
|
4079
|
+
let lastError;
|
|
4080
|
+
for (let attempt = 0; attempt < this.config.retryAttempts; attempt++) {
|
|
4081
|
+
const [ok, err] = await tryFn(() => cache.clear(key));
|
|
4082
|
+
if (ok) {
|
|
4083
|
+
return [true, null];
|
|
4084
|
+
}
|
|
4085
|
+
lastError = err;
|
|
4086
|
+
if (err.name === "NoSuchKey" || err.code === "NoSuchKey") {
|
|
4087
|
+
return [true, null];
|
|
4088
|
+
}
|
|
4089
|
+
if (attempt < this.config.retryAttempts - 1) {
|
|
4090
|
+
const delay = this.config.retryDelay * Math.pow(2, attempt);
|
|
4091
|
+
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
3855
4092
|
}
|
|
3856
4093
|
}
|
|
4094
|
+
return [false, lastError];
|
|
3857
4095
|
}
|
|
3858
4096
|
async generateCacheKey(resource, action, params = {}, partition = null, partitionValues = null) {
|
|
3859
4097
|
const keyParts = [
|
|
@@ -3869,14 +4107,14 @@ class CachePlugin extends Plugin {
|
|
|
3869
4107
|
}
|
|
3870
4108
|
}
|
|
3871
4109
|
if (Object.keys(params).length > 0) {
|
|
3872
|
-
const paramsHash =
|
|
4110
|
+
const paramsHash = this.hashParams(params);
|
|
3873
4111
|
keyParts.push(paramsHash);
|
|
3874
4112
|
}
|
|
3875
4113
|
return join(...keyParts) + ".json.gz";
|
|
3876
4114
|
}
|
|
3877
|
-
|
|
3878
|
-
const
|
|
3879
|
-
return
|
|
4115
|
+
hashParams(params) {
|
|
4116
|
+
const serialized = jsonStableStringify(params) || "empty";
|
|
4117
|
+
return crypto.createHash("md5").update(serialized).digest("hex").substring(0, 16);
|
|
3880
4118
|
}
|
|
3881
4119
|
// Utility methods
|
|
3882
4120
|
async getCacheStats() {
|
|
@@ -3901,50 +4139,48 @@ class CachePlugin extends Plugin {
|
|
|
3901
4139
|
if (!resource) {
|
|
3902
4140
|
throw new Error(`Resource '${resourceName}' not found`);
|
|
3903
4141
|
}
|
|
3904
|
-
const { includePartitions = true } = options;
|
|
4142
|
+
const { includePartitions = true, sampleSize = 100 } = options;
|
|
3905
4143
|
if (this.driver instanceof PartitionAwareFilesystemCache && resource.warmPartitionCache) {
|
|
3906
4144
|
const partitionNames = resource.config.partitions ? Object.keys(resource.config.partitions) : [];
|
|
3907
4145
|
return await resource.warmPartitionCache(partitionNames, options);
|
|
3908
4146
|
}
|
|
3909
|
-
|
|
3910
|
-
|
|
4147
|
+
let offset = 0;
|
|
4148
|
+
const pageSize = 100;
|
|
4149
|
+
const sampledRecords = [];
|
|
4150
|
+
while (sampledRecords.length < sampleSize) {
|
|
4151
|
+
const [ok, err, pageResult] = await tryFn(() => resource.page({ offset, size: pageSize }));
|
|
4152
|
+
if (!ok || !pageResult) {
|
|
4153
|
+
break;
|
|
4154
|
+
}
|
|
4155
|
+
const pageItems = Array.isArray(pageResult) ? pageResult : pageResult.items || [];
|
|
4156
|
+
if (pageItems.length === 0) {
|
|
4157
|
+
break;
|
|
4158
|
+
}
|
|
4159
|
+
sampledRecords.push(...pageItems);
|
|
4160
|
+
offset += pageSize;
|
|
4161
|
+
}
|
|
4162
|
+
if (includePartitions && resource.config.partitions && sampledRecords.length > 0) {
|
|
3911
4163
|
for (const [partitionName, partitionDef] of Object.entries(resource.config.partitions)) {
|
|
3912
4164
|
if (partitionDef.fields) {
|
|
3913
|
-
const
|
|
3914
|
-
const
|
|
3915
|
-
const partitionValues = /* @__PURE__ */ new Set();
|
|
3916
|
-
for (const record of recordsArray.slice(0, 10)) {
|
|
4165
|
+
const partitionValuesSet = /* @__PURE__ */ new Set();
|
|
4166
|
+
for (const record of sampledRecords) {
|
|
3917
4167
|
const values = this.getPartitionValues(record, resource);
|
|
3918
4168
|
if (values[partitionName]) {
|
|
3919
|
-
|
|
4169
|
+
partitionValuesSet.add(JSON.stringify(values[partitionName]));
|
|
3920
4170
|
}
|
|
3921
4171
|
}
|
|
3922
|
-
for (const partitionValueStr of
|
|
3923
|
-
const
|
|
3924
|
-
await resource.list({ partition: partitionName, partitionValues
|
|
4172
|
+
for (const partitionValueStr of partitionValuesSet) {
|
|
4173
|
+
const partitionValues = JSON.parse(partitionValueStr);
|
|
4174
|
+
await tryFn(() => resource.list({ partition: partitionName, partitionValues }));
|
|
3925
4175
|
}
|
|
3926
4176
|
}
|
|
3927
4177
|
}
|
|
3928
4178
|
}
|
|
3929
|
-
|
|
3930
|
-
|
|
3931
|
-
|
|
3932
|
-
|
|
3933
|
-
|
|
3934
|
-
}
|
|
3935
|
-
return await this.driver.getPartitionStats(resourceName, partition);
|
|
3936
|
-
}
|
|
3937
|
-
async getCacheRecommendations(resourceName) {
|
|
3938
|
-
if (!(this.driver instanceof PartitionAwareFilesystemCache)) {
|
|
3939
|
-
throw new Error("Cache recommendations are only available with PartitionAwareFilesystemCache");
|
|
3940
|
-
}
|
|
3941
|
-
return await this.driver.getCacheRecommendations(resourceName);
|
|
3942
|
-
}
|
|
3943
|
-
async clearPartitionCache(resourceName, partition, partitionValues = {}) {
|
|
3944
|
-
if (!(this.driver instanceof PartitionAwareFilesystemCache)) {
|
|
3945
|
-
throw new Error("Partition cache clearing is only available with PartitionAwareFilesystemCache");
|
|
3946
|
-
}
|
|
3947
|
-
return await this.driver.clearPartition(resourceName, partition, partitionValues);
|
|
4179
|
+
return {
|
|
4180
|
+
resourceName,
|
|
4181
|
+
recordsSampled: sampledRecords.length,
|
|
4182
|
+
partitionsWarmed: includePartitions && resource.config.partitions ? Object.keys(resource.config.partitions).length : 0
|
|
4183
|
+
};
|
|
3948
4184
|
}
|
|
3949
4185
|
async analyzeCacheUsage() {
|
|
3950
4186
|
if (!(this.driver instanceof PartitionAwareFilesystemCache)) {
|
|
@@ -3961,6 +4197,9 @@ class CachePlugin extends Plugin {
|
|
|
3961
4197
|
}
|
|
3962
4198
|
};
|
|
3963
4199
|
for (const [resourceName, resource] of Object.entries(this.database.resources)) {
|
|
4200
|
+
if (!this.shouldCacheResource(resourceName)) {
|
|
4201
|
+
continue;
|
|
4202
|
+
}
|
|
3964
4203
|
try {
|
|
3965
4204
|
analysis.resourceStats[resourceName] = await this.driver.getPartitionStats(resourceName);
|
|
3966
4205
|
analysis.recommendations[resourceName] = await this.driver.getCacheRecommendations(resourceName);
|
|
@@ -4061,13 +4300,12 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
4061
4300
|
if (!options.field) {
|
|
4062
4301
|
throw new Error("EventualConsistencyPlugin requires 'field' option");
|
|
4063
4302
|
}
|
|
4303
|
+
const detectedTimezone = this._detectTimezone();
|
|
4064
4304
|
this.config = {
|
|
4065
4305
|
resource: options.resource,
|
|
4066
4306
|
field: options.field,
|
|
4067
4307
|
cohort: {
|
|
4068
|
-
|
|
4069
|
-
timezone: options.cohort?.timezone || "UTC",
|
|
4070
|
-
...options.cohort
|
|
4308
|
+
timezone: options.cohort?.timezone || detectedTimezone
|
|
4071
4309
|
},
|
|
4072
4310
|
reducer: options.reducer || ((transactions) => {
|
|
4073
4311
|
let baseValue = 0;
|
|
@@ -4082,19 +4320,42 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
4082
4320
|
}
|
|
4083
4321
|
return baseValue;
|
|
4084
4322
|
}),
|
|
4085
|
-
consolidationInterval: options.consolidationInterval
|
|
4086
|
-
//
|
|
4323
|
+
consolidationInterval: options.consolidationInterval ?? 300,
|
|
4324
|
+
// 5 minutes (in seconds)
|
|
4325
|
+
consolidationConcurrency: options.consolidationConcurrency || 5,
|
|
4326
|
+
consolidationWindow: options.consolidationWindow || 24,
|
|
4327
|
+
// Hours to look back for pending transactions (watermark)
|
|
4087
4328
|
autoConsolidate: options.autoConsolidate !== false,
|
|
4329
|
+
lateArrivalStrategy: options.lateArrivalStrategy || "warn",
|
|
4330
|
+
// 'ignore', 'warn', 'process'
|
|
4088
4331
|
batchTransactions: options.batchTransactions || false,
|
|
4332
|
+
// CAUTION: Not safe in distributed environments! Loses data on container crash
|
|
4089
4333
|
batchSize: options.batchSize || 100,
|
|
4090
4334
|
mode: options.mode || "async",
|
|
4091
4335
|
// 'async' or 'sync'
|
|
4092
|
-
|
|
4336
|
+
lockTimeout: options.lockTimeout || 300,
|
|
4337
|
+
// 5 minutes (in seconds, configurable)
|
|
4338
|
+
transactionRetention: options.transactionRetention || 30,
|
|
4339
|
+
// Days to keep applied transactions
|
|
4340
|
+
gcInterval: options.gcInterval || 86400,
|
|
4341
|
+
// 24 hours (in seconds)
|
|
4342
|
+
verbose: options.verbose || false
|
|
4093
4343
|
};
|
|
4094
4344
|
this.transactionResource = null;
|
|
4095
4345
|
this.targetResource = null;
|
|
4096
4346
|
this.consolidationTimer = null;
|
|
4347
|
+
this.gcTimer = null;
|
|
4097
4348
|
this.pendingTransactions = /* @__PURE__ */ new Map();
|
|
4349
|
+
if (this.config.batchTransactions && !this.config.verbose) {
|
|
4350
|
+
console.warn(
|
|
4351
|
+
`[EventualConsistency] WARNING: batchTransactions is enabled. This stores transactions in memory and will lose data if container crashes. Not recommended for distributed/production environments. Set verbose: true to suppress this warning.`
|
|
4352
|
+
);
|
|
4353
|
+
}
|
|
4354
|
+
if (this.config.verbose && !options.cohort?.timezone) {
|
|
4355
|
+
console.log(
|
|
4356
|
+
`[EventualConsistency] Auto-detected timezone: ${this.config.cohort.timezone} (from ${process.env.TZ ? "TZ env var" : "system Intl API"})`
|
|
4357
|
+
);
|
|
4358
|
+
}
|
|
4098
4359
|
}
|
|
4099
4360
|
async onSetup() {
|
|
4100
4361
|
this.targetResource = this.database.resources[this.config.resource];
|
|
@@ -4131,7 +4392,9 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
4131
4392
|
// 'set', 'add', or 'sub'
|
|
4132
4393
|
timestamp: "string|required",
|
|
4133
4394
|
cohortDate: "string|required",
|
|
4134
|
-
// For partitioning
|
|
4395
|
+
// For daily partitioning
|
|
4396
|
+
cohortHour: "string|required",
|
|
4397
|
+
// For hourly partitioning
|
|
4135
4398
|
cohortMonth: "string|optional",
|
|
4136
4399
|
// For monthly partitioning
|
|
4137
4400
|
source: "string|optional",
|
|
@@ -4149,10 +4412,28 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
4149
4412
|
throw new Error(`Failed to create transaction resource: ${err?.message}`);
|
|
4150
4413
|
}
|
|
4151
4414
|
this.transactionResource = ok ? transactionResource : this.database.resources[transactionResourceName];
|
|
4415
|
+
const lockResourceName = `${this.config.resource}_consolidation_locks_${this.config.field}`;
|
|
4416
|
+
const [lockOk, lockErr, lockResource] = await tryFn(
|
|
4417
|
+
() => this.database.createResource({
|
|
4418
|
+
name: lockResourceName,
|
|
4419
|
+
attributes: {
|
|
4420
|
+
id: "string|required",
|
|
4421
|
+
lockedAt: "number|required",
|
|
4422
|
+
workerId: "string|optional"
|
|
4423
|
+
},
|
|
4424
|
+
behavior: "body-only",
|
|
4425
|
+
timestamps: false
|
|
4426
|
+
})
|
|
4427
|
+
);
|
|
4428
|
+
if (!lockOk && !this.database.resources[lockResourceName]) {
|
|
4429
|
+
throw new Error(`Failed to create lock resource: ${lockErr?.message}`);
|
|
4430
|
+
}
|
|
4431
|
+
this.lockResource = lockOk ? lockResource : this.database.resources[lockResourceName];
|
|
4152
4432
|
this.addHelperMethods();
|
|
4153
4433
|
if (this.config.autoConsolidate) {
|
|
4154
4434
|
this.startConsolidationTimer();
|
|
4155
4435
|
}
|
|
4436
|
+
this.startGarbageCollectionTimer();
|
|
4156
4437
|
}
|
|
4157
4438
|
async onStart() {
|
|
4158
4439
|
if (this.deferredSetup) {
|
|
@@ -4169,6 +4450,10 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
4169
4450
|
clearInterval(this.consolidationTimer);
|
|
4170
4451
|
this.consolidationTimer = null;
|
|
4171
4452
|
}
|
|
4453
|
+
if (this.gcTimer) {
|
|
4454
|
+
clearInterval(this.gcTimer);
|
|
4455
|
+
this.gcTimer = null;
|
|
4456
|
+
}
|
|
4172
4457
|
await this.flushPendingTransactions();
|
|
4173
4458
|
this.emit("eventual-consistency.stopped", {
|
|
4174
4459
|
resource: this.config.resource,
|
|
@@ -4177,6 +4462,11 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
4177
4462
|
}
|
|
4178
4463
|
createPartitionConfig() {
|
|
4179
4464
|
const partitions = {
|
|
4465
|
+
byHour: {
|
|
4466
|
+
fields: {
|
|
4467
|
+
cohortHour: "string"
|
|
4468
|
+
}
|
|
4469
|
+
},
|
|
4180
4470
|
byDay: {
|
|
4181
4471
|
fields: {
|
|
4182
4472
|
cohortDate: "string"
|
|
@@ -4190,6 +4480,65 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
4190
4480
|
};
|
|
4191
4481
|
return partitions;
|
|
4192
4482
|
}
|
|
4483
|
+
/**
|
|
4484
|
+
* Auto-detect timezone from environment or system
|
|
4485
|
+
* @private
|
|
4486
|
+
*/
|
|
4487
|
+
_detectTimezone() {
|
|
4488
|
+
if (process.env.TZ) {
|
|
4489
|
+
return process.env.TZ;
|
|
4490
|
+
}
|
|
4491
|
+
try {
|
|
4492
|
+
const systemTimezone = Intl.DateTimeFormat().resolvedOptions().timeZone;
|
|
4493
|
+
if (systemTimezone) {
|
|
4494
|
+
return systemTimezone;
|
|
4495
|
+
}
|
|
4496
|
+
} catch (err) {
|
|
4497
|
+
}
|
|
4498
|
+
return "UTC";
|
|
4499
|
+
}
|
|
4500
|
+
/**
|
|
4501
|
+
* Helper method to resolve field and plugin from arguments
|
|
4502
|
+
* Supports both single-field (field, value) and multi-field (field, value) signatures
|
|
4503
|
+
* @private
|
|
4504
|
+
*/
|
|
4505
|
+
_resolveFieldAndPlugin(resource, fieldOrValue, value) {
|
|
4506
|
+
const hasMultipleFields = Object.keys(resource._eventualConsistencyPlugins).length > 1;
|
|
4507
|
+
if (hasMultipleFields && value === void 0) {
|
|
4508
|
+
throw new Error(`Multiple fields have eventual consistency. Please specify the field explicitly.`);
|
|
4509
|
+
}
|
|
4510
|
+
const field = value !== void 0 ? fieldOrValue : this.config.field;
|
|
4511
|
+
const actualValue = value !== void 0 ? value : fieldOrValue;
|
|
4512
|
+
const fieldPlugin = resource._eventualConsistencyPlugins[field];
|
|
4513
|
+
if (!fieldPlugin) {
|
|
4514
|
+
throw new Error(`No eventual consistency plugin found for field "${field}"`);
|
|
4515
|
+
}
|
|
4516
|
+
return { field, value: actualValue, plugin: fieldPlugin };
|
|
4517
|
+
}
|
|
4518
|
+
/**
|
|
4519
|
+
* Helper method to perform atomic consolidation in sync mode
|
|
4520
|
+
* @private
|
|
4521
|
+
*/
|
|
4522
|
+
async _syncModeConsolidate(id, field) {
|
|
4523
|
+
const consolidatedValue = await this.consolidateRecord(id);
|
|
4524
|
+
await this.targetResource.update(id, {
|
|
4525
|
+
[field]: consolidatedValue
|
|
4526
|
+
});
|
|
4527
|
+
return consolidatedValue;
|
|
4528
|
+
}
|
|
4529
|
+
/**
|
|
4530
|
+
* Create synthetic 'set' transaction from current value
|
|
4531
|
+
* @private
|
|
4532
|
+
*/
|
|
4533
|
+
_createSyntheticSetTransaction(currentValue) {
|
|
4534
|
+
return {
|
|
4535
|
+
id: "__synthetic__",
|
|
4536
|
+
operation: "set",
|
|
4537
|
+
value: currentValue,
|
|
4538
|
+
timestamp: (/* @__PURE__ */ new Date(0)).toISOString(),
|
|
4539
|
+
synthetic: true
|
|
4540
|
+
};
|
|
4541
|
+
}
|
|
4193
4542
|
addHelperMethods() {
|
|
4194
4543
|
const resource = this.targetResource;
|
|
4195
4544
|
const defaultField = this.config.field;
|
|
@@ -4199,16 +4548,7 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
4199
4548
|
}
|
|
4200
4549
|
resource._eventualConsistencyPlugins[defaultField] = plugin;
|
|
4201
4550
|
resource.set = async (id, fieldOrValue, value) => {
|
|
4202
|
-
const
|
|
4203
|
-
if (hasMultipleFields && value === void 0) {
|
|
4204
|
-
throw new Error(`Multiple fields have eventual consistency. Please specify the field: set(id, field, value)`);
|
|
4205
|
-
}
|
|
4206
|
-
const field = value !== void 0 ? fieldOrValue : defaultField;
|
|
4207
|
-
const actualValue = value !== void 0 ? value : fieldOrValue;
|
|
4208
|
-
const fieldPlugin = resource._eventualConsistencyPlugins[field];
|
|
4209
|
-
if (!fieldPlugin) {
|
|
4210
|
-
throw new Error(`No eventual consistency plugin found for field "${field}"`);
|
|
4211
|
-
}
|
|
4551
|
+
const { field, value: actualValue, plugin: fieldPlugin } = plugin._resolveFieldAndPlugin(resource, fieldOrValue, value);
|
|
4212
4552
|
await fieldPlugin.createTransaction({
|
|
4213
4553
|
originalId: id,
|
|
4214
4554
|
operation: "set",
|
|
@@ -4216,25 +4556,12 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
4216
4556
|
source: "set"
|
|
4217
4557
|
});
|
|
4218
4558
|
if (fieldPlugin.config.mode === "sync") {
|
|
4219
|
-
|
|
4220
|
-
await resource.update(id, {
|
|
4221
|
-
[field]: consolidatedValue
|
|
4222
|
-
});
|
|
4223
|
-
return consolidatedValue;
|
|
4559
|
+
return await fieldPlugin._syncModeConsolidate(id, field);
|
|
4224
4560
|
}
|
|
4225
4561
|
return actualValue;
|
|
4226
4562
|
};
|
|
4227
4563
|
resource.add = async (id, fieldOrAmount, amount) => {
|
|
4228
|
-
const
|
|
4229
|
-
if (hasMultipleFields && amount === void 0) {
|
|
4230
|
-
throw new Error(`Multiple fields have eventual consistency. Please specify the field: add(id, field, amount)`);
|
|
4231
|
-
}
|
|
4232
|
-
const field = amount !== void 0 ? fieldOrAmount : defaultField;
|
|
4233
|
-
const actualAmount = amount !== void 0 ? amount : fieldOrAmount;
|
|
4234
|
-
const fieldPlugin = resource._eventualConsistencyPlugins[field];
|
|
4235
|
-
if (!fieldPlugin) {
|
|
4236
|
-
throw new Error(`No eventual consistency plugin found for field "${field}"`);
|
|
4237
|
-
}
|
|
4564
|
+
const { field, value: actualAmount, plugin: fieldPlugin } = plugin._resolveFieldAndPlugin(resource, fieldOrAmount, amount);
|
|
4238
4565
|
await fieldPlugin.createTransaction({
|
|
4239
4566
|
originalId: id,
|
|
4240
4567
|
operation: "add",
|
|
@@ -4242,26 +4569,13 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
4242
4569
|
source: "add"
|
|
4243
4570
|
});
|
|
4244
4571
|
if (fieldPlugin.config.mode === "sync") {
|
|
4245
|
-
|
|
4246
|
-
await resource.update(id, {
|
|
4247
|
-
[field]: consolidatedValue
|
|
4248
|
-
});
|
|
4249
|
-
return consolidatedValue;
|
|
4572
|
+
return await fieldPlugin._syncModeConsolidate(id, field);
|
|
4250
4573
|
}
|
|
4251
4574
|
const currentValue = await fieldPlugin.getConsolidatedValue(id);
|
|
4252
4575
|
return currentValue + actualAmount;
|
|
4253
4576
|
};
|
|
4254
4577
|
resource.sub = async (id, fieldOrAmount, amount) => {
|
|
4255
|
-
const
|
|
4256
|
-
if (hasMultipleFields && amount === void 0) {
|
|
4257
|
-
throw new Error(`Multiple fields have eventual consistency. Please specify the field: sub(id, field, amount)`);
|
|
4258
|
-
}
|
|
4259
|
-
const field = amount !== void 0 ? fieldOrAmount : defaultField;
|
|
4260
|
-
const actualAmount = amount !== void 0 ? amount : fieldOrAmount;
|
|
4261
|
-
const fieldPlugin = resource._eventualConsistencyPlugins[field];
|
|
4262
|
-
if (!fieldPlugin) {
|
|
4263
|
-
throw new Error(`No eventual consistency plugin found for field "${field}"`);
|
|
4264
|
-
}
|
|
4578
|
+
const { field, value: actualAmount, plugin: fieldPlugin } = plugin._resolveFieldAndPlugin(resource, fieldOrAmount, amount);
|
|
4265
4579
|
await fieldPlugin.createTransaction({
|
|
4266
4580
|
originalId: id,
|
|
4267
4581
|
operation: "sub",
|
|
@@ -4269,11 +4583,7 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
4269
4583
|
source: "sub"
|
|
4270
4584
|
});
|
|
4271
4585
|
if (fieldPlugin.config.mode === "sync") {
|
|
4272
|
-
|
|
4273
|
-
await resource.update(id, {
|
|
4274
|
-
[field]: consolidatedValue
|
|
4275
|
-
});
|
|
4276
|
-
return consolidatedValue;
|
|
4586
|
+
return await fieldPlugin._syncModeConsolidate(id, field);
|
|
4277
4587
|
}
|
|
4278
4588
|
const currentValue = await fieldPlugin.getConsolidatedValue(id);
|
|
4279
4589
|
return currentValue - actualAmount;
|
|
@@ -4303,14 +4613,34 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
4303
4613
|
async createTransaction(data) {
|
|
4304
4614
|
const now = /* @__PURE__ */ new Date();
|
|
4305
4615
|
const cohortInfo = this.getCohortInfo(now);
|
|
4616
|
+
const watermarkMs = this.config.consolidationWindow * 60 * 60 * 1e3;
|
|
4617
|
+
const watermarkTime = now.getTime() - watermarkMs;
|
|
4618
|
+
const cohortHourDate = /* @__PURE__ */ new Date(cohortInfo.hour + ":00:00Z");
|
|
4619
|
+
if (cohortHourDate.getTime() < watermarkTime) {
|
|
4620
|
+
const hoursLate = Math.floor((now.getTime() - cohortHourDate.getTime()) / (60 * 60 * 1e3));
|
|
4621
|
+
if (this.config.lateArrivalStrategy === "ignore") {
|
|
4622
|
+
if (this.config.verbose) {
|
|
4623
|
+
console.warn(
|
|
4624
|
+
`[EventualConsistency] Late arrival ignored: transaction for ${cohortInfo.hour} is ${hoursLate}h late (watermark: ${this.config.consolidationWindow}h)`
|
|
4625
|
+
);
|
|
4626
|
+
}
|
|
4627
|
+
return null;
|
|
4628
|
+
} else if (this.config.lateArrivalStrategy === "warn") {
|
|
4629
|
+
console.warn(
|
|
4630
|
+
`[EventualConsistency] Late arrival detected: transaction for ${cohortInfo.hour} is ${hoursLate}h late (watermark: ${this.config.consolidationWindow}h). Processing anyway, but consolidation may not pick it up.`
|
|
4631
|
+
);
|
|
4632
|
+
}
|
|
4633
|
+
}
|
|
4306
4634
|
const transaction = {
|
|
4307
|
-
id:
|
|
4635
|
+
id: idGenerator(),
|
|
4636
|
+
// Use nanoid for guaranteed uniqueness
|
|
4308
4637
|
originalId: data.originalId,
|
|
4309
4638
|
field: this.config.field,
|
|
4310
4639
|
value: data.value || 0,
|
|
4311
4640
|
operation: data.operation || "set",
|
|
4312
4641
|
timestamp: now.toISOString(),
|
|
4313
4642
|
cohortDate: cohortInfo.date,
|
|
4643
|
+
cohortHour: cohortInfo.hour,
|
|
4314
4644
|
cohortMonth: cohortInfo.month,
|
|
4315
4645
|
source: data.source || "unknown",
|
|
4316
4646
|
applied: false
|
|
@@ -4328,9 +4658,16 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
4328
4658
|
async flushPendingTransactions() {
|
|
4329
4659
|
if (this.pendingTransactions.size === 0) return;
|
|
4330
4660
|
const transactions = Array.from(this.pendingTransactions.values());
|
|
4331
|
-
|
|
4332
|
-
|
|
4333
|
-
|
|
4661
|
+
try {
|
|
4662
|
+
await Promise.all(
|
|
4663
|
+
transactions.map(
|
|
4664
|
+
(transaction) => this.transactionResource.insert(transaction)
|
|
4665
|
+
)
|
|
4666
|
+
);
|
|
4667
|
+
this.pendingTransactions.clear();
|
|
4668
|
+
} catch (error) {
|
|
4669
|
+
console.error("Failed to flush pending transactions:", error);
|
|
4670
|
+
throw error;
|
|
4334
4671
|
}
|
|
4335
4672
|
}
|
|
4336
4673
|
getCohortInfo(date) {
|
|
@@ -4340,53 +4677,90 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
4340
4677
|
const year = localDate.getFullYear();
|
|
4341
4678
|
const month = String(localDate.getMonth() + 1).padStart(2, "0");
|
|
4342
4679
|
const day = String(localDate.getDate()).padStart(2, "0");
|
|
4680
|
+
const hour = String(localDate.getHours()).padStart(2, "0");
|
|
4343
4681
|
return {
|
|
4344
4682
|
date: `${year}-${month}-${day}`,
|
|
4683
|
+
hour: `${year}-${month}-${day}T${hour}`,
|
|
4684
|
+
// ISO-like format for hour partition
|
|
4345
4685
|
month: `${year}-${month}`
|
|
4346
4686
|
};
|
|
4347
4687
|
}
|
|
4348
4688
|
getTimezoneOffset(timezone) {
|
|
4349
|
-
|
|
4350
|
-
|
|
4351
|
-
"
|
|
4352
|
-
"
|
|
4353
|
-
|
|
4354
|
-
|
|
4355
|
-
|
|
4356
|
-
|
|
4357
|
-
|
|
4358
|
-
|
|
4359
|
-
|
|
4360
|
-
|
|
4361
|
-
|
|
4362
|
-
|
|
4363
|
-
|
|
4364
|
-
|
|
4365
|
-
|
|
4366
|
-
|
|
4367
|
-
|
|
4368
|
-
|
|
4369
|
-
|
|
4689
|
+
try {
|
|
4690
|
+
const now = /* @__PURE__ */ new Date();
|
|
4691
|
+
const utcDate = new Date(now.toLocaleString("en-US", { timeZone: "UTC" }));
|
|
4692
|
+
const tzDate = new Date(now.toLocaleString("en-US", { timeZone: timezone }));
|
|
4693
|
+
return tzDate.getTime() - utcDate.getTime();
|
|
4694
|
+
} catch (err) {
|
|
4695
|
+
const offsets = {
|
|
4696
|
+
"UTC": 0,
|
|
4697
|
+
"America/New_York": -5 * 36e5,
|
|
4698
|
+
"America/Chicago": -6 * 36e5,
|
|
4699
|
+
"America/Denver": -7 * 36e5,
|
|
4700
|
+
"America/Los_Angeles": -8 * 36e5,
|
|
4701
|
+
"America/Sao_Paulo": -3 * 36e5,
|
|
4702
|
+
"Europe/London": 0,
|
|
4703
|
+
"Europe/Paris": 1 * 36e5,
|
|
4704
|
+
"Europe/Berlin": 1 * 36e5,
|
|
4705
|
+
"Asia/Tokyo": 9 * 36e5,
|
|
4706
|
+
"Asia/Shanghai": 8 * 36e5,
|
|
4707
|
+
"Australia/Sydney": 10 * 36e5
|
|
4708
|
+
};
|
|
4709
|
+
if (this.config.verbose && !offsets[timezone]) {
|
|
4710
|
+
console.warn(
|
|
4711
|
+
`[EventualConsistency] Unknown timezone '${timezone}', using UTC. Consider using a valid IANA timezone (e.g., 'America/New_York')`
|
|
4712
|
+
);
|
|
4713
|
+
}
|
|
4714
|
+
return offsets[timezone] || 0;
|
|
4715
|
+
}
|
|
4716
|
+
}
|
|
4717
|
+
startConsolidationTimer() {
|
|
4718
|
+
const intervalMs = this.config.consolidationInterval * 1e3;
|
|
4719
|
+
this.consolidationTimer = setInterval(async () => {
|
|
4720
|
+
await this.runConsolidation();
|
|
4721
|
+
}, intervalMs);
|
|
4370
4722
|
}
|
|
4371
4723
|
async runConsolidation() {
|
|
4372
4724
|
try {
|
|
4373
|
-
const
|
|
4374
|
-
|
|
4375
|
-
|
|
4725
|
+
const now = /* @__PURE__ */ new Date();
|
|
4726
|
+
const hoursToCheck = this.config.consolidationWindow || 24;
|
|
4727
|
+
const cohortHours = [];
|
|
4728
|
+
for (let i = 0; i < hoursToCheck; i++) {
|
|
4729
|
+
const date = new Date(now.getTime() - i * 60 * 60 * 1e3);
|
|
4730
|
+
const cohortInfo = this.getCohortInfo(date);
|
|
4731
|
+
cohortHours.push(cohortInfo.hour);
|
|
4732
|
+
}
|
|
4733
|
+
const transactionsByHour = await Promise.all(
|
|
4734
|
+
cohortHours.map(async (cohortHour) => {
|
|
4735
|
+
const [ok, err, txns] = await tryFn(
|
|
4736
|
+
() => this.transactionResource.query({
|
|
4737
|
+
cohortHour,
|
|
4738
|
+
applied: false
|
|
4739
|
+
})
|
|
4740
|
+
);
|
|
4741
|
+
return ok ? txns : [];
|
|
4376
4742
|
})
|
|
4377
4743
|
);
|
|
4378
|
-
|
|
4379
|
-
|
|
4744
|
+
const transactions = transactionsByHour.flat();
|
|
4745
|
+
if (transactions.length === 0) {
|
|
4746
|
+
if (this.config.verbose) {
|
|
4747
|
+
console.log(`[EventualConsistency] No pending transactions to consolidate`);
|
|
4748
|
+
}
|
|
4380
4749
|
return;
|
|
4381
4750
|
}
|
|
4382
4751
|
const uniqueIds = [...new Set(transactions.map((t) => t.originalId))];
|
|
4383
|
-
|
|
4384
|
-
await this.consolidateRecord(id);
|
|
4752
|
+
const { results, errors } = await PromisePool.for(uniqueIds).withConcurrency(this.config.consolidationConcurrency).process(async (id) => {
|
|
4753
|
+
return await this.consolidateRecord(id);
|
|
4754
|
+
});
|
|
4755
|
+
if (errors && errors.length > 0) {
|
|
4756
|
+
console.error(`Consolidation completed with ${errors.length} errors:`, errors);
|
|
4385
4757
|
}
|
|
4386
4758
|
this.emit("eventual-consistency.consolidated", {
|
|
4387
4759
|
resource: this.config.resource,
|
|
4388
4760
|
field: this.config.field,
|
|
4389
|
-
recordCount: uniqueIds.length
|
|
4761
|
+
recordCount: uniqueIds.length,
|
|
4762
|
+
successCount: results.length,
|
|
4763
|
+
errorCount: errors.length
|
|
4390
4764
|
});
|
|
4391
4765
|
} catch (error) {
|
|
4392
4766
|
console.error("Consolidation error:", error);
|
|
@@ -4394,49 +4768,73 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
4394
4768
|
}
|
|
4395
4769
|
}
|
|
4396
4770
|
async consolidateRecord(originalId) {
|
|
4397
|
-
|
|
4398
|
-
|
|
4399
|
-
|
|
4400
|
-
|
|
4401
|
-
|
|
4402
|
-
|
|
4403
|
-
|
|
4404
|
-
applied: false
|
|
4771
|
+
await this.cleanupStaleLocks();
|
|
4772
|
+
const lockId = `lock-${originalId}`;
|
|
4773
|
+
const [lockAcquired, lockErr, lock] = await tryFn(
|
|
4774
|
+
() => this.lockResource.insert({
|
|
4775
|
+
id: lockId,
|
|
4776
|
+
lockedAt: Date.now(),
|
|
4777
|
+
workerId: process.pid ? String(process.pid) : "unknown"
|
|
4405
4778
|
})
|
|
4406
4779
|
);
|
|
4407
|
-
if (!
|
|
4408
|
-
|
|
4409
|
-
|
|
4410
|
-
|
|
4411
|
-
|
|
4412
|
-
|
|
4413
|
-
|
|
4414
|
-
|
|
4415
|
-
transactions.unshift({
|
|
4416
|
-
id: "__synthetic__",
|
|
4417
|
-
// Synthetic ID that we'll skip when marking as applied
|
|
4418
|
-
operation: "set",
|
|
4419
|
-
value: currentValue,
|
|
4420
|
-
timestamp: (/* @__PURE__ */ new Date(0)).toISOString()
|
|
4421
|
-
// Very old timestamp to ensure it's first
|
|
4422
|
-
});
|
|
4780
|
+
if (!lockAcquired) {
|
|
4781
|
+
if (this.config.verbose) {
|
|
4782
|
+
console.log(`[EventualConsistency] Lock for ${originalId} already held, skipping`);
|
|
4783
|
+
}
|
|
4784
|
+
const [recordOk, recordErr, record] = await tryFn(
|
|
4785
|
+
() => this.targetResource.get(originalId)
|
|
4786
|
+
);
|
|
4787
|
+
return recordOk && record ? record[this.config.field] || 0 : 0;
|
|
4423
4788
|
}
|
|
4424
|
-
|
|
4425
|
-
|
|
4426
|
-
|
|
4427
|
-
|
|
4428
|
-
|
|
4429
|
-
|
|
4430
|
-
|
|
4431
|
-
|
|
4432
|
-
|
|
4433
|
-
|
|
4434
|
-
|
|
4435
|
-
|
|
4789
|
+
try {
|
|
4790
|
+
const [recordOk, recordErr, record] = await tryFn(
|
|
4791
|
+
() => this.targetResource.get(originalId)
|
|
4792
|
+
);
|
|
4793
|
+
const currentValue = recordOk && record ? record[this.config.field] || 0 : 0;
|
|
4794
|
+
const [ok, err, transactions] = await tryFn(
|
|
4795
|
+
() => this.transactionResource.query({
|
|
4796
|
+
originalId,
|
|
4797
|
+
applied: false
|
|
4798
|
+
})
|
|
4799
|
+
);
|
|
4800
|
+
if (!ok || !transactions || transactions.length === 0) {
|
|
4801
|
+
return currentValue;
|
|
4802
|
+
}
|
|
4803
|
+
transactions.sort(
|
|
4804
|
+
(a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()
|
|
4805
|
+
);
|
|
4806
|
+
const hasSetOperation = transactions.some((t) => t.operation === "set");
|
|
4807
|
+
if (currentValue !== 0 && !hasSetOperation) {
|
|
4808
|
+
transactions.unshift(this._createSyntheticSetTransaction(currentValue));
|
|
4809
|
+
}
|
|
4810
|
+
const consolidatedValue = this.config.reducer(transactions);
|
|
4811
|
+
const [updateOk, updateErr] = await tryFn(
|
|
4812
|
+
() => this.targetResource.update(originalId, {
|
|
4813
|
+
[this.config.field]: consolidatedValue
|
|
4814
|
+
})
|
|
4815
|
+
);
|
|
4816
|
+
if (updateOk) {
|
|
4817
|
+
const transactionsToUpdate = transactions.filter((txn) => txn.id !== "__synthetic__");
|
|
4818
|
+
const { results, errors } = await PromisePool.for(transactionsToUpdate).withConcurrency(10).process(async (txn) => {
|
|
4819
|
+
const [ok2, err2] = await tryFn(
|
|
4820
|
+
() => this.transactionResource.update(txn.id, { applied: true })
|
|
4821
|
+
);
|
|
4822
|
+
if (!ok2 && this.config.verbose) {
|
|
4823
|
+
console.warn(`[EventualConsistency] Failed to mark transaction ${txn.id} as applied:`, err2?.message);
|
|
4824
|
+
}
|
|
4825
|
+
return ok2;
|
|
4826
|
+
});
|
|
4827
|
+
if (errors && errors.length > 0 && this.config.verbose) {
|
|
4828
|
+
console.warn(`[EventualConsistency] ${errors.length} transactions failed to mark as applied`);
|
|
4436
4829
|
}
|
|
4437
4830
|
}
|
|
4831
|
+
return consolidatedValue;
|
|
4832
|
+
} finally {
|
|
4833
|
+
const [lockReleased, lockReleaseErr] = await tryFn(() => this.lockResource.delete(lockId));
|
|
4834
|
+
if (!lockReleased && this.config.verbose) {
|
|
4835
|
+
console.warn(`[EventualConsistency] Failed to release lock ${lockId}:`, lockReleaseErr?.message);
|
|
4836
|
+
}
|
|
4438
4837
|
}
|
|
4439
|
-
return consolidatedValue;
|
|
4440
4838
|
}
|
|
4441
4839
|
async getConsolidatedValue(originalId, options = {}) {
|
|
4442
4840
|
const includeApplied = options.includeApplied || false;
|
|
@@ -4450,11 +4848,11 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
4450
4848
|
() => this.transactionResource.query(query)
|
|
4451
4849
|
);
|
|
4452
4850
|
if (!ok || !transactions || transactions.length === 0) {
|
|
4453
|
-
const [
|
|
4851
|
+
const [recordOk2, recordErr2, record2] = await tryFn(
|
|
4454
4852
|
() => this.targetResource.get(originalId)
|
|
4455
4853
|
);
|
|
4456
|
-
if (
|
|
4457
|
-
return
|
|
4854
|
+
if (recordOk2 && record2) {
|
|
4855
|
+
return record2[this.config.field] || 0;
|
|
4458
4856
|
}
|
|
4459
4857
|
return 0;
|
|
4460
4858
|
}
|
|
@@ -4467,6 +4865,14 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
4467
4865
|
return true;
|
|
4468
4866
|
});
|
|
4469
4867
|
}
|
|
4868
|
+
const [recordOk, recordErr, record] = await tryFn(
|
|
4869
|
+
() => this.targetResource.get(originalId)
|
|
4870
|
+
);
|
|
4871
|
+
const currentValue = recordOk && record ? record[this.config.field] || 0 : 0;
|
|
4872
|
+
const hasSetOperation = filtered.some((t) => t.operation === "set");
|
|
4873
|
+
if (currentValue !== 0 && !hasSetOperation) {
|
|
4874
|
+
filtered.unshift(this._createSyntheticSetTransaction(currentValue));
|
|
4875
|
+
}
|
|
4470
4876
|
filtered.sort(
|
|
4471
4877
|
(a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()
|
|
4472
4878
|
);
|
|
@@ -4501,6 +4907,133 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
4501
4907
|
}
|
|
4502
4908
|
return stats;
|
|
4503
4909
|
}
|
|
4910
|
+
/**
|
|
4911
|
+
* Clean up stale locks that exceed the configured timeout
|
|
4912
|
+
* Uses distributed locking to prevent multiple containers from cleaning simultaneously
|
|
4913
|
+
*/
|
|
4914
|
+
async cleanupStaleLocks() {
|
|
4915
|
+
const now = Date.now();
|
|
4916
|
+
const lockTimeoutMs = this.config.lockTimeout * 1e3;
|
|
4917
|
+
const cutoffTime = now - lockTimeoutMs;
|
|
4918
|
+
const cleanupLockId = `lock-cleanup-${this.config.resource}-${this.config.field}`;
|
|
4919
|
+
const [lockAcquired] = await tryFn(
|
|
4920
|
+
() => this.lockResource.insert({
|
|
4921
|
+
id: cleanupLockId,
|
|
4922
|
+
lockedAt: Date.now(),
|
|
4923
|
+
workerId: process.pid ? String(process.pid) : "unknown"
|
|
4924
|
+
})
|
|
4925
|
+
);
|
|
4926
|
+
if (!lockAcquired) {
|
|
4927
|
+
if (this.config.verbose) {
|
|
4928
|
+
console.log(`[EventualConsistency] Lock cleanup already running in another container`);
|
|
4929
|
+
}
|
|
4930
|
+
return;
|
|
4931
|
+
}
|
|
4932
|
+
try {
|
|
4933
|
+
const [ok, err, locks] = await tryFn(() => this.lockResource.list());
|
|
4934
|
+
if (!ok || !locks || locks.length === 0) return;
|
|
4935
|
+
const staleLocks = locks.filter(
|
|
4936
|
+
(lock) => lock.id !== cleanupLockId && lock.lockedAt < cutoffTime
|
|
4937
|
+
);
|
|
4938
|
+
if (staleLocks.length === 0) return;
|
|
4939
|
+
if (this.config.verbose) {
|
|
4940
|
+
console.log(`[EventualConsistency] Cleaning up ${staleLocks.length} stale locks`);
|
|
4941
|
+
}
|
|
4942
|
+
const { results, errors } = await PromisePool.for(staleLocks).withConcurrency(5).process(async (lock) => {
|
|
4943
|
+
const [deleted] = await tryFn(() => this.lockResource.delete(lock.id));
|
|
4944
|
+
return deleted;
|
|
4945
|
+
});
|
|
4946
|
+
if (errors && errors.length > 0 && this.config.verbose) {
|
|
4947
|
+
console.warn(`[EventualConsistency] ${errors.length} stale locks failed to delete`);
|
|
4948
|
+
}
|
|
4949
|
+
} catch (error) {
|
|
4950
|
+
if (this.config.verbose) {
|
|
4951
|
+
console.warn(`[EventualConsistency] Error cleaning up stale locks:`, error.message);
|
|
4952
|
+
}
|
|
4953
|
+
} finally {
|
|
4954
|
+
await tryFn(() => this.lockResource.delete(cleanupLockId));
|
|
4955
|
+
}
|
|
4956
|
+
}
|
|
4957
|
+
/**
|
|
4958
|
+
* Start garbage collection timer for old applied transactions
|
|
4959
|
+
*/
|
|
4960
|
+
startGarbageCollectionTimer() {
|
|
4961
|
+
const gcIntervalMs = this.config.gcInterval * 1e3;
|
|
4962
|
+
this.gcTimer = setInterval(async () => {
|
|
4963
|
+
await this.runGarbageCollection();
|
|
4964
|
+
}, gcIntervalMs);
|
|
4965
|
+
}
|
|
4966
|
+
/**
|
|
4967
|
+
* Delete old applied transactions based on retention policy
|
|
4968
|
+
* Uses distributed locking to prevent multiple containers from running GC simultaneously
|
|
4969
|
+
*/
|
|
4970
|
+
async runGarbageCollection() {
|
|
4971
|
+
const gcLockId = `lock-gc-${this.config.resource}-${this.config.field}`;
|
|
4972
|
+
const [lockAcquired] = await tryFn(
|
|
4973
|
+
() => this.lockResource.insert({
|
|
4974
|
+
id: gcLockId,
|
|
4975
|
+
lockedAt: Date.now(),
|
|
4976
|
+
workerId: process.pid ? String(process.pid) : "unknown"
|
|
4977
|
+
})
|
|
4978
|
+
);
|
|
4979
|
+
if (!lockAcquired) {
|
|
4980
|
+
if (this.config.verbose) {
|
|
4981
|
+
console.log(`[EventualConsistency] GC already running in another container`);
|
|
4982
|
+
}
|
|
4983
|
+
return;
|
|
4984
|
+
}
|
|
4985
|
+
try {
|
|
4986
|
+
const now = Date.now();
|
|
4987
|
+
const retentionMs = this.config.transactionRetention * 24 * 60 * 60 * 1e3;
|
|
4988
|
+
const cutoffDate = new Date(now - retentionMs);
|
|
4989
|
+
const cutoffIso = cutoffDate.toISOString();
|
|
4990
|
+
if (this.config.verbose) {
|
|
4991
|
+
console.log(`[EventualConsistency] Running GC for transactions older than ${cutoffIso} (${this.config.transactionRetention} days)`);
|
|
4992
|
+
}
|
|
4993
|
+
const cutoffMonth = cutoffDate.toISOString().substring(0, 7);
|
|
4994
|
+
const [ok, err, oldTransactions] = await tryFn(
|
|
4995
|
+
() => this.transactionResource.query({
|
|
4996
|
+
applied: true,
|
|
4997
|
+
timestamp: { "<": cutoffIso }
|
|
4998
|
+
})
|
|
4999
|
+
);
|
|
5000
|
+
if (!ok) {
|
|
5001
|
+
if (this.config.verbose) {
|
|
5002
|
+
console.warn(`[EventualConsistency] GC failed to query transactions:`, err?.message);
|
|
5003
|
+
}
|
|
5004
|
+
return;
|
|
5005
|
+
}
|
|
5006
|
+
if (!oldTransactions || oldTransactions.length === 0) {
|
|
5007
|
+
if (this.config.verbose) {
|
|
5008
|
+
console.log(`[EventualConsistency] No old transactions to clean up`);
|
|
5009
|
+
}
|
|
5010
|
+
return;
|
|
5011
|
+
}
|
|
5012
|
+
if (this.config.verbose) {
|
|
5013
|
+
console.log(`[EventualConsistency] Deleting ${oldTransactions.length} old transactions`);
|
|
5014
|
+
}
|
|
5015
|
+
const { results, errors } = await PromisePool.for(oldTransactions).withConcurrency(10).process(async (txn) => {
|
|
5016
|
+
const [deleted] = await tryFn(() => this.transactionResource.delete(txn.id));
|
|
5017
|
+
return deleted;
|
|
5018
|
+
});
|
|
5019
|
+
if (this.config.verbose) {
|
|
5020
|
+
console.log(`[EventualConsistency] GC completed: ${results.length} deleted, ${errors.length} errors`);
|
|
5021
|
+
}
|
|
5022
|
+
this.emit("eventual-consistency.gc-completed", {
|
|
5023
|
+
resource: this.config.resource,
|
|
5024
|
+
field: this.config.field,
|
|
5025
|
+
deletedCount: results.length,
|
|
5026
|
+
errorCount: errors.length
|
|
5027
|
+
});
|
|
5028
|
+
} catch (error) {
|
|
5029
|
+
if (this.config.verbose) {
|
|
5030
|
+
console.warn(`[EventualConsistency] GC error:`, error.message);
|
|
5031
|
+
}
|
|
5032
|
+
this.emit("eventual-consistency.gc-error", error);
|
|
5033
|
+
} finally {
|
|
5034
|
+
await tryFn(() => this.lockResource.delete(gcLockId));
|
|
5035
|
+
}
|
|
5036
|
+
}
|
|
4504
5037
|
}
|
|
4505
5038
|
|
|
4506
5039
|
class FullTextPlugin extends Plugin {
|
|
@@ -4517,7 +5050,7 @@ class FullTextPlugin extends Plugin {
|
|
|
4517
5050
|
async setup(database) {
|
|
4518
5051
|
this.database = database;
|
|
4519
5052
|
const [ok, err, indexResource] = await tryFn(() => database.createResource({
|
|
4520
|
-
name: "
|
|
5053
|
+
name: "plg_fulltext_indexes",
|
|
4521
5054
|
attributes: {
|
|
4522
5055
|
id: "string|required",
|
|
4523
5056
|
resourceName: "string|required",
|
|
@@ -4576,7 +5109,7 @@ class FullTextPlugin extends Plugin {
|
|
|
4576
5109
|
}
|
|
4577
5110
|
installDatabaseHooks() {
|
|
4578
5111
|
this.database.addHook("afterCreateResource", (resource) => {
|
|
4579
|
-
if (resource.name !== "
|
|
5112
|
+
if (resource.name !== "plg_fulltext_indexes") {
|
|
4580
5113
|
this.installResourceHooks(resource);
|
|
4581
5114
|
}
|
|
4582
5115
|
});
|
|
@@ -4590,14 +5123,14 @@ class FullTextPlugin extends Plugin {
|
|
|
4590
5123
|
}
|
|
4591
5124
|
this.database.plugins.fulltext = this;
|
|
4592
5125
|
for (const resource of Object.values(this.database.resources)) {
|
|
4593
|
-
if (resource.name === "
|
|
5126
|
+
if (resource.name === "plg_fulltext_indexes") continue;
|
|
4594
5127
|
this.installResourceHooks(resource);
|
|
4595
5128
|
}
|
|
4596
5129
|
if (!this.database._fulltextProxyInstalled) {
|
|
4597
5130
|
this.database._previousCreateResourceForFullText = this.database.createResource;
|
|
4598
5131
|
this.database.createResource = async function(...args) {
|
|
4599
5132
|
const resource = await this._previousCreateResourceForFullText(...args);
|
|
4600
|
-
if (this.plugins?.fulltext && resource.name !== "
|
|
5133
|
+
if (this.plugins?.fulltext && resource.name !== "plg_fulltext_indexes") {
|
|
4601
5134
|
this.plugins.fulltext.installResourceHooks(resource);
|
|
4602
5135
|
}
|
|
4603
5136
|
return resource;
|
|
@@ -4605,7 +5138,7 @@ class FullTextPlugin extends Plugin {
|
|
|
4605
5138
|
this.database._fulltextProxyInstalled = true;
|
|
4606
5139
|
}
|
|
4607
5140
|
for (const resource of Object.values(this.database.resources)) {
|
|
4608
|
-
if (resource.name !== "
|
|
5141
|
+
if (resource.name !== "plg_fulltext_indexes") {
|
|
4609
5142
|
this.installResourceHooks(resource);
|
|
4610
5143
|
}
|
|
4611
5144
|
}
|
|
@@ -4848,7 +5381,7 @@ class FullTextPlugin extends Plugin {
|
|
|
4848
5381
|
return this._rebuildAllIndexesInternal();
|
|
4849
5382
|
}
|
|
4850
5383
|
async _rebuildAllIndexesInternal() {
|
|
4851
|
-
const resourceNames = Object.keys(this.database.resources).filter((name) => name !== "
|
|
5384
|
+
const resourceNames = Object.keys(this.database.resources).filter((name) => name !== "plg_fulltext_indexes");
|
|
4852
5385
|
for (const resourceName of resourceNames) {
|
|
4853
5386
|
const [ok, err] = await tryFn(() => this.rebuildIndex(resourceName));
|
|
4854
5387
|
}
|
|
@@ -4900,7 +5433,7 @@ class MetricsPlugin extends Plugin {
|
|
|
4900
5433
|
if (typeof process !== "undefined" && process.env.NODE_ENV === "test") return;
|
|
4901
5434
|
const [ok, err] = await tryFn(async () => {
|
|
4902
5435
|
const [ok1, err1, metricsResource] = await tryFn(() => database.createResource({
|
|
4903
|
-
name: "
|
|
5436
|
+
name: "plg_metrics",
|
|
4904
5437
|
attributes: {
|
|
4905
5438
|
id: "string|required",
|
|
4906
5439
|
type: "string|required",
|
|
@@ -4915,9 +5448,9 @@ class MetricsPlugin extends Plugin {
|
|
|
4915
5448
|
metadata: "json"
|
|
4916
5449
|
}
|
|
4917
5450
|
}));
|
|
4918
|
-
this.metricsResource = ok1 ? metricsResource : database.resources.
|
|
5451
|
+
this.metricsResource = ok1 ? metricsResource : database.resources.plg_metrics;
|
|
4919
5452
|
const [ok2, err2, errorsResource] = await tryFn(() => database.createResource({
|
|
4920
|
-
name: "
|
|
5453
|
+
name: "plg_error_logs",
|
|
4921
5454
|
attributes: {
|
|
4922
5455
|
id: "string|required",
|
|
4923
5456
|
resourceName: "string|required",
|
|
@@ -4927,9 +5460,9 @@ class MetricsPlugin extends Plugin {
|
|
|
4927
5460
|
metadata: "json"
|
|
4928
5461
|
}
|
|
4929
5462
|
}));
|
|
4930
|
-
this.errorsResource = ok2 ? errorsResource : database.resources.
|
|
5463
|
+
this.errorsResource = ok2 ? errorsResource : database.resources.plg_error_logs;
|
|
4931
5464
|
const [ok3, err3, performanceResource] = await tryFn(() => database.createResource({
|
|
4932
|
-
name: "
|
|
5465
|
+
name: "plg_performance_logs",
|
|
4933
5466
|
attributes: {
|
|
4934
5467
|
id: "string|required",
|
|
4935
5468
|
resourceName: "string|required",
|
|
@@ -4939,12 +5472,12 @@ class MetricsPlugin extends Plugin {
|
|
|
4939
5472
|
metadata: "json"
|
|
4940
5473
|
}
|
|
4941
5474
|
}));
|
|
4942
|
-
this.performanceResource = ok3 ? performanceResource : database.resources.
|
|
5475
|
+
this.performanceResource = ok3 ? performanceResource : database.resources.plg_performance_logs;
|
|
4943
5476
|
});
|
|
4944
5477
|
if (!ok) {
|
|
4945
|
-
this.metricsResource = database.resources.
|
|
4946
|
-
this.errorsResource = database.resources.
|
|
4947
|
-
this.performanceResource = database.resources.
|
|
5478
|
+
this.metricsResource = database.resources.plg_metrics;
|
|
5479
|
+
this.errorsResource = database.resources.plg_error_logs;
|
|
5480
|
+
this.performanceResource = database.resources.plg_performance_logs;
|
|
4948
5481
|
}
|
|
4949
5482
|
this.installDatabaseHooks();
|
|
4950
5483
|
this.installMetricsHooks();
|
|
@@ -4963,7 +5496,7 @@ class MetricsPlugin extends Plugin {
|
|
|
4963
5496
|
}
|
|
4964
5497
|
installDatabaseHooks() {
|
|
4965
5498
|
this.database.addHook("afterCreateResource", (resource) => {
|
|
4966
|
-
if (resource.name !== "
|
|
5499
|
+
if (resource.name !== "plg_metrics" && resource.name !== "plg_error_logs" && resource.name !== "plg_performance_logs") {
|
|
4967
5500
|
this.installResourceHooks(resource);
|
|
4968
5501
|
}
|
|
4969
5502
|
});
|
|
@@ -4973,7 +5506,7 @@ class MetricsPlugin extends Plugin {
|
|
|
4973
5506
|
}
|
|
4974
5507
|
installMetricsHooks() {
|
|
4975
5508
|
for (const resource of Object.values(this.database.resources)) {
|
|
4976
|
-
if (["
|
|
5509
|
+
if (["plg_metrics", "plg_error_logs", "plg_performance_logs"].includes(resource.name)) {
|
|
4977
5510
|
continue;
|
|
4978
5511
|
}
|
|
4979
5512
|
this.installResourceHooks(resource);
|
|
@@ -4981,7 +5514,7 @@ class MetricsPlugin extends Plugin {
|
|
|
4981
5514
|
this.database._createResource = this.database.createResource;
|
|
4982
5515
|
this.database.createResource = async function(...args) {
|
|
4983
5516
|
const resource = await this._createResource(...args);
|
|
4984
|
-
if (this.plugins?.metrics && !["
|
|
5517
|
+
if (this.plugins?.metrics && !["plg_metrics", "plg_error_logs", "plg_performance_logs"].includes(resource.name)) {
|
|
4985
5518
|
this.plugins.metrics.installResourceHooks(resource);
|
|
4986
5519
|
}
|
|
4987
5520
|
return resource;
|
|
@@ -5365,6 +5898,253 @@ class MetricsPlugin extends Plugin {
|
|
|
5365
5898
|
}
|
|
5366
5899
|
}
|
|
5367
5900
|
|
|
5901
|
+
class SqsConsumer {
|
|
5902
|
+
constructor({ queueUrl, onMessage, onError, poolingInterval = 5e3, maxMessages = 10, region = "us-east-1", credentials, endpoint, driver = "sqs" }) {
|
|
5903
|
+
this.driver = driver;
|
|
5904
|
+
this.queueUrl = queueUrl;
|
|
5905
|
+
this.onMessage = onMessage;
|
|
5906
|
+
this.onError = onError;
|
|
5907
|
+
this.poolingInterval = poolingInterval;
|
|
5908
|
+
this.maxMessages = maxMessages;
|
|
5909
|
+
this.region = region;
|
|
5910
|
+
this.credentials = credentials;
|
|
5911
|
+
this.endpoint = endpoint;
|
|
5912
|
+
this.sqs = null;
|
|
5913
|
+
this._stopped = false;
|
|
5914
|
+
this._timer = null;
|
|
5915
|
+
this._pollPromise = null;
|
|
5916
|
+
this._pollResolve = null;
|
|
5917
|
+
this._SQSClient = null;
|
|
5918
|
+
this._ReceiveMessageCommand = null;
|
|
5919
|
+
this._DeleteMessageCommand = null;
|
|
5920
|
+
}
|
|
5921
|
+
async start() {
|
|
5922
|
+
const [ok, err, sdk] = await tryFn(() => import('@aws-sdk/client-sqs'));
|
|
5923
|
+
if (!ok) throw new Error("SqsConsumer: @aws-sdk/client-sqs is not installed. Please install it to use the SQS consumer.");
|
|
5924
|
+
const { SQSClient, ReceiveMessageCommand, DeleteMessageCommand } = sdk;
|
|
5925
|
+
this._SQSClient = SQSClient;
|
|
5926
|
+
this._ReceiveMessageCommand = ReceiveMessageCommand;
|
|
5927
|
+
this._DeleteMessageCommand = DeleteMessageCommand;
|
|
5928
|
+
this.sqs = new SQSClient({ region: this.region, credentials: this.credentials, endpoint: this.endpoint });
|
|
5929
|
+
this._stopped = false;
|
|
5930
|
+
this._pollPromise = new Promise((resolve) => {
|
|
5931
|
+
this._pollResolve = resolve;
|
|
5932
|
+
});
|
|
5933
|
+
this._poll();
|
|
5934
|
+
}
|
|
5935
|
+
async stop() {
|
|
5936
|
+
this._stopped = true;
|
|
5937
|
+
if (this._timer) {
|
|
5938
|
+
clearTimeout(this._timer);
|
|
5939
|
+
this._timer = null;
|
|
5940
|
+
}
|
|
5941
|
+
if (this._pollResolve) {
|
|
5942
|
+
this._pollResolve();
|
|
5943
|
+
}
|
|
5944
|
+
}
|
|
5945
|
+
async _poll() {
|
|
5946
|
+
if (this._stopped) {
|
|
5947
|
+
if (this._pollResolve) this._pollResolve();
|
|
5948
|
+
return;
|
|
5949
|
+
}
|
|
5950
|
+
const [ok, err, result] = await tryFn(async () => {
|
|
5951
|
+
const cmd = new this._ReceiveMessageCommand({
|
|
5952
|
+
QueueUrl: this.queueUrl,
|
|
5953
|
+
MaxNumberOfMessages: this.maxMessages,
|
|
5954
|
+
WaitTimeSeconds: 10,
|
|
5955
|
+
MessageAttributeNames: ["All"]
|
|
5956
|
+
});
|
|
5957
|
+
const { Messages } = await this.sqs.send(cmd);
|
|
5958
|
+
if (Messages && Messages.length > 0) {
|
|
5959
|
+
for (const msg of Messages) {
|
|
5960
|
+
const [okMsg, errMsg] = await tryFn(async () => {
|
|
5961
|
+
const parsedMsg = this._parseMessage(msg);
|
|
5962
|
+
await this.onMessage(parsedMsg, msg);
|
|
5963
|
+
await this.sqs.send(new this._DeleteMessageCommand({
|
|
5964
|
+
QueueUrl: this.queueUrl,
|
|
5965
|
+
ReceiptHandle: msg.ReceiptHandle
|
|
5966
|
+
}));
|
|
5967
|
+
});
|
|
5968
|
+
if (!okMsg && this.onError) {
|
|
5969
|
+
this.onError(errMsg, msg);
|
|
5970
|
+
}
|
|
5971
|
+
}
|
|
5972
|
+
}
|
|
5973
|
+
});
|
|
5974
|
+
if (!ok && this.onError) {
|
|
5975
|
+
this.onError(err);
|
|
5976
|
+
}
|
|
5977
|
+
this._timer = setTimeout(() => this._poll(), this.poolingInterval);
|
|
5978
|
+
}
|
|
5979
|
+
_parseMessage(msg) {
|
|
5980
|
+
let body;
|
|
5981
|
+
const [ok, err, parsed] = tryFn(() => JSON.parse(msg.Body));
|
|
5982
|
+
body = ok ? parsed : msg.Body;
|
|
5983
|
+
const attributes = {};
|
|
5984
|
+
if (msg.MessageAttributes) {
|
|
5985
|
+
for (const [k, v] of Object.entries(msg.MessageAttributes)) {
|
|
5986
|
+
attributes[k] = v.StringValue;
|
|
5987
|
+
}
|
|
5988
|
+
}
|
|
5989
|
+
return { $body: body, $attributes: attributes, $raw: msg };
|
|
5990
|
+
}
|
|
5991
|
+
}
|
|
5992
|
+
|
|
5993
|
+
class RabbitMqConsumer {
|
|
5994
|
+
constructor({ amqpUrl, queue, prefetch = 10, reconnectInterval = 2e3, onMessage, onError, driver = "rabbitmq" }) {
|
|
5995
|
+
this.amqpUrl = amqpUrl;
|
|
5996
|
+
this.queue = queue;
|
|
5997
|
+
this.prefetch = prefetch;
|
|
5998
|
+
this.reconnectInterval = reconnectInterval;
|
|
5999
|
+
this.onMessage = onMessage;
|
|
6000
|
+
this.onError = onError;
|
|
6001
|
+
this.driver = driver;
|
|
6002
|
+
this.connection = null;
|
|
6003
|
+
this.channel = null;
|
|
6004
|
+
this._stopped = false;
|
|
6005
|
+
}
|
|
6006
|
+
async start() {
|
|
6007
|
+
this._stopped = false;
|
|
6008
|
+
await this._connect();
|
|
6009
|
+
}
|
|
6010
|
+
async stop() {
|
|
6011
|
+
this._stopped = true;
|
|
6012
|
+
if (this.channel) await this.channel.close();
|
|
6013
|
+
if (this.connection) await this.connection.close();
|
|
6014
|
+
}
|
|
6015
|
+
async _connect() {
|
|
6016
|
+
const [ok, err] = await tryFn(async () => {
|
|
6017
|
+
const amqp = (await import('amqplib')).default;
|
|
6018
|
+
this.connection = await amqp.connect(this.amqpUrl);
|
|
6019
|
+
this.channel = await this.connection.createChannel();
|
|
6020
|
+
await this.channel.assertQueue(this.queue, { durable: true });
|
|
6021
|
+
this.channel.prefetch(this.prefetch);
|
|
6022
|
+
this.channel.consume(this.queue, async (msg) => {
|
|
6023
|
+
if (msg !== null) {
|
|
6024
|
+
const [okMsg, errMsg] = await tryFn(async () => {
|
|
6025
|
+
const content = JSON.parse(msg.content.toString());
|
|
6026
|
+
await this.onMessage({ $body: content, $raw: msg });
|
|
6027
|
+
this.channel.ack(msg);
|
|
6028
|
+
});
|
|
6029
|
+
if (!okMsg) {
|
|
6030
|
+
if (this.onError) this.onError(errMsg, msg);
|
|
6031
|
+
this.channel.nack(msg, false, false);
|
|
6032
|
+
}
|
|
6033
|
+
}
|
|
6034
|
+
});
|
|
6035
|
+
});
|
|
6036
|
+
if (!ok) {
|
|
6037
|
+
if (this.onError) this.onError(err);
|
|
6038
|
+
if (!this._stopped) {
|
|
6039
|
+
setTimeout(() => this._connect(), this.reconnectInterval);
|
|
6040
|
+
}
|
|
6041
|
+
}
|
|
6042
|
+
}
|
|
6043
|
+
}
|
|
6044
|
+
|
|
6045
|
+
const CONSUMER_DRIVERS = {
|
|
6046
|
+
sqs: SqsConsumer,
|
|
6047
|
+
rabbitmq: RabbitMqConsumer
|
|
6048
|
+
// kafka: KafkaConsumer, // futuro
|
|
6049
|
+
};
|
|
6050
|
+
function createConsumer(driver, config) {
|
|
6051
|
+
const ConsumerClass = CONSUMER_DRIVERS[driver];
|
|
6052
|
+
if (!ConsumerClass) {
|
|
6053
|
+
throw new Error(`Unknown consumer driver: ${driver}. Available: ${Object.keys(CONSUMER_DRIVERS).join(", ")}`);
|
|
6054
|
+
}
|
|
6055
|
+
return new ConsumerClass(config);
|
|
6056
|
+
}
|
|
6057
|
+
|
|
6058
|
+
class QueueConsumerPlugin {
|
|
6059
|
+
constructor(options = {}) {
|
|
6060
|
+
this.options = options;
|
|
6061
|
+
this.driversConfig = Array.isArray(options.consumers) ? options.consumers : [];
|
|
6062
|
+
this.consumers = [];
|
|
6063
|
+
}
|
|
6064
|
+
async setup(database) {
|
|
6065
|
+
this.database = database;
|
|
6066
|
+
for (const driverDef of this.driversConfig) {
|
|
6067
|
+
const { driver, config: driverConfig = {}, consumers: consumerDefs = [] } = driverDef;
|
|
6068
|
+
if (consumerDefs.length === 0 && driverDef.resources) {
|
|
6069
|
+
const { resources, driver: defDriver, config: nestedConfig, ...directConfig } = driverDef;
|
|
6070
|
+
const resourceList = Array.isArray(resources) ? resources : [resources];
|
|
6071
|
+
const flatConfig = nestedConfig ? { ...directConfig, ...nestedConfig } : directConfig;
|
|
6072
|
+
for (const resource of resourceList) {
|
|
6073
|
+
const consumer = createConsumer(driver, {
|
|
6074
|
+
...flatConfig,
|
|
6075
|
+
onMessage: (msg) => this._handleMessage(msg, resource),
|
|
6076
|
+
onError: (err, raw) => this._handleError(err, raw, resource)
|
|
6077
|
+
});
|
|
6078
|
+
await consumer.start();
|
|
6079
|
+
this.consumers.push(consumer);
|
|
6080
|
+
}
|
|
6081
|
+
} else {
|
|
6082
|
+
for (const consumerDef of consumerDefs) {
|
|
6083
|
+
const { resources, ...consumerConfig } = consumerDef;
|
|
6084
|
+
const resourceList = Array.isArray(resources) ? resources : [resources];
|
|
6085
|
+
for (const resource of resourceList) {
|
|
6086
|
+
const mergedConfig = { ...driverConfig, ...consumerConfig };
|
|
6087
|
+
const consumer = createConsumer(driver, {
|
|
6088
|
+
...mergedConfig,
|
|
6089
|
+
onMessage: (msg) => this._handleMessage(msg, resource),
|
|
6090
|
+
onError: (err, raw) => this._handleError(err, raw, resource)
|
|
6091
|
+
});
|
|
6092
|
+
await consumer.start();
|
|
6093
|
+
this.consumers.push(consumer);
|
|
6094
|
+
}
|
|
6095
|
+
}
|
|
6096
|
+
}
|
|
6097
|
+
}
|
|
6098
|
+
}
|
|
6099
|
+
async stop() {
|
|
6100
|
+
if (!Array.isArray(this.consumers)) this.consumers = [];
|
|
6101
|
+
for (const consumer of this.consumers) {
|
|
6102
|
+
if (consumer && typeof consumer.stop === "function") {
|
|
6103
|
+
await consumer.stop();
|
|
6104
|
+
}
|
|
6105
|
+
}
|
|
6106
|
+
this.consumers = [];
|
|
6107
|
+
}
|
|
6108
|
+
async _handleMessage(msg, configuredResource) {
|
|
6109
|
+
this.options;
|
|
6110
|
+
let body = msg.$body || msg;
|
|
6111
|
+
if (body.$body && !body.resource && !body.action && !body.data) {
|
|
6112
|
+
body = body.$body;
|
|
6113
|
+
}
|
|
6114
|
+
let resource = body.resource || msg.resource;
|
|
6115
|
+
let action = body.action || msg.action;
|
|
6116
|
+
let data = body.data || msg.data;
|
|
6117
|
+
if (!resource) {
|
|
6118
|
+
throw new Error("QueueConsumerPlugin: resource not found in message");
|
|
6119
|
+
}
|
|
6120
|
+
if (!action) {
|
|
6121
|
+
throw new Error("QueueConsumerPlugin: action not found in message");
|
|
6122
|
+
}
|
|
6123
|
+
const resourceObj = this.database.resources[resource];
|
|
6124
|
+
if (!resourceObj) throw new Error(`QueueConsumerPlugin: resource '${resource}' not found`);
|
|
6125
|
+
let result;
|
|
6126
|
+
const [ok, err, res] = await tryFn(async () => {
|
|
6127
|
+
if (action === "insert") {
|
|
6128
|
+
result = await resourceObj.insert(data);
|
|
6129
|
+
} else if (action === "update") {
|
|
6130
|
+
const { id: updateId, ...updateAttributes } = data;
|
|
6131
|
+
result = await resourceObj.update(updateId, updateAttributes);
|
|
6132
|
+
} else if (action === "delete") {
|
|
6133
|
+
result = await resourceObj.delete(data.id);
|
|
6134
|
+
} else {
|
|
6135
|
+
throw new Error(`QueueConsumerPlugin: unsupported action '${action}'`);
|
|
6136
|
+
}
|
|
6137
|
+
return result;
|
|
6138
|
+
});
|
|
6139
|
+
if (!ok) {
|
|
6140
|
+
throw err;
|
|
6141
|
+
}
|
|
6142
|
+
return res;
|
|
6143
|
+
}
|
|
6144
|
+
_handleError(err, raw, resourceName) {
|
|
6145
|
+
}
|
|
6146
|
+
}
|
|
6147
|
+
|
|
5368
6148
|
class BaseReplicator extends EventEmitter {
|
|
5369
6149
|
constructor(config = {}) {
|
|
5370
6150
|
super();
|
|
@@ -6352,7 +7132,7 @@ class Client extends EventEmitter {
|
|
|
6352
7132
|
this.emit("command.response", command.constructor.name, response, command.input);
|
|
6353
7133
|
return response;
|
|
6354
7134
|
}
|
|
6355
|
-
async putObject({ key, metadata, contentType, body, contentEncoding, contentLength }) {
|
|
7135
|
+
async putObject({ key, metadata, contentType, body, contentEncoding, contentLength, ifMatch }) {
|
|
6356
7136
|
const keyPrefix = typeof this.config.keyPrefix === "string" ? this.config.keyPrefix : "";
|
|
6357
7137
|
keyPrefix ? path.join(keyPrefix, key) : key;
|
|
6358
7138
|
const stringMetadata = {};
|
|
@@ -6372,6 +7152,7 @@ class Client extends EventEmitter {
|
|
|
6372
7152
|
if (contentType !== void 0) options.ContentType = contentType;
|
|
6373
7153
|
if (contentEncoding !== void 0) options.ContentEncoding = contentEncoding;
|
|
6374
7154
|
if (contentLength !== void 0) options.ContentLength = contentLength;
|
|
7155
|
+
if (ifMatch !== void 0) options.IfMatch = ifMatch;
|
|
6375
7156
|
let response, error;
|
|
6376
7157
|
try {
|
|
6377
7158
|
response = await this.sendCommand(new PutObjectCommand(options));
|
|
@@ -8535,6 +9316,7 @@ ${errorDetails}`,
|
|
|
8535
9316
|
data._lastModified = request.LastModified;
|
|
8536
9317
|
data._hasContent = request.ContentLength > 0;
|
|
8537
9318
|
data._mimeType = request.ContentType || null;
|
|
9319
|
+
data._etag = request.ETag;
|
|
8538
9320
|
data._v = objectVersion;
|
|
8539
9321
|
if (request.VersionId) data._versionId = request.VersionId;
|
|
8540
9322
|
if (request.Expiration) data._expiresAt = request.Expiration;
|
|
@@ -8747,59 +9529,225 @@ ${errorDetails}`,
|
|
|
8747
9529
|
}
|
|
8748
9530
|
}
|
|
8749
9531
|
/**
|
|
8750
|
-
*
|
|
9532
|
+
* Update with conditional check (If-Match ETag)
|
|
8751
9533
|
* @param {string} id - Resource ID
|
|
8752
|
-
* @
|
|
9534
|
+
* @param {Object} attributes - Attributes to update
|
|
9535
|
+
* @param {Object} options - Options including ifMatch (ETag)
|
|
9536
|
+
* @returns {Promise<Object>} { success: boolean, data?: Object, etag?: string, error?: string }
|
|
8753
9537
|
* @example
|
|
8754
|
-
* await resource.
|
|
9538
|
+
* const msg = await resource.get('msg-123');
|
|
9539
|
+
* const result = await resource.updateConditional('msg-123', { status: 'processing' }, { ifMatch: msg._etag });
|
|
9540
|
+
* if (!result.success) {
|
|
9541
|
+
* console.log('Update failed - object was modified by another process');
|
|
9542
|
+
* }
|
|
8755
9543
|
*/
|
|
8756
|
-
async
|
|
9544
|
+
async updateConditional(id, attributes, options = {}) {
|
|
8757
9545
|
if (isEmpty(id)) {
|
|
8758
9546
|
throw new Error("id cannot be empty");
|
|
8759
9547
|
}
|
|
8760
|
-
|
|
8761
|
-
|
|
8762
|
-
|
|
8763
|
-
if (ok) {
|
|
8764
|
-
objectData = data;
|
|
8765
|
-
} else {
|
|
8766
|
-
objectData = { id };
|
|
8767
|
-
deleteError = err;
|
|
9548
|
+
const { ifMatch } = options;
|
|
9549
|
+
if (!ifMatch) {
|
|
9550
|
+
throw new Error("updateConditional requires ifMatch option with ETag value");
|
|
8768
9551
|
}
|
|
8769
|
-
await this.
|
|
8770
|
-
|
|
8771
|
-
|
|
8772
|
-
|
|
8773
|
-
|
|
8774
|
-
|
|
8775
|
-
$after: null
|
|
8776
|
-
});
|
|
8777
|
-
if (deleteError) {
|
|
8778
|
-
throw mapAwsError(deleteError, {
|
|
8779
|
-
bucket: this.client.config.bucket,
|
|
8780
|
-
key,
|
|
8781
|
-
resourceName: this.name,
|
|
8782
|
-
operation: "delete",
|
|
8783
|
-
id
|
|
8784
|
-
});
|
|
9552
|
+
const exists = await this.exists(id);
|
|
9553
|
+
if (!exists) {
|
|
9554
|
+
return {
|
|
9555
|
+
success: false,
|
|
9556
|
+
error: `Resource with id '${id}' does not exist`
|
|
9557
|
+
};
|
|
8785
9558
|
}
|
|
8786
|
-
|
|
8787
|
-
|
|
8788
|
-
|
|
8789
|
-
|
|
8790
|
-
|
|
8791
|
-
|
|
8792
|
-
|
|
8793
|
-
|
|
8794
|
-
|
|
8795
|
-
|
|
8796
|
-
|
|
8797
|
-
|
|
8798
|
-
|
|
8799
|
-
|
|
8800
|
-
|
|
8801
|
-
});
|
|
8802
|
-
}
|
|
9559
|
+
const originalData = await this.get(id);
|
|
9560
|
+
const attributesClone = cloneDeep(attributes);
|
|
9561
|
+
let mergedData = cloneDeep(originalData);
|
|
9562
|
+
for (const [key2, value] of Object.entries(attributesClone)) {
|
|
9563
|
+
if (key2.includes(".")) {
|
|
9564
|
+
let ref = mergedData;
|
|
9565
|
+
const parts = key2.split(".");
|
|
9566
|
+
for (let i = 0; i < parts.length - 1; i++) {
|
|
9567
|
+
if (typeof ref[parts[i]] !== "object" || ref[parts[i]] === null) {
|
|
9568
|
+
ref[parts[i]] = {};
|
|
9569
|
+
}
|
|
9570
|
+
ref = ref[parts[i]];
|
|
9571
|
+
}
|
|
9572
|
+
ref[parts[parts.length - 1]] = cloneDeep(value);
|
|
9573
|
+
} else if (typeof value === "object" && value !== null && !Array.isArray(value)) {
|
|
9574
|
+
mergedData[key2] = merge({}, mergedData[key2], value);
|
|
9575
|
+
} else {
|
|
9576
|
+
mergedData[key2] = cloneDeep(value);
|
|
9577
|
+
}
|
|
9578
|
+
}
|
|
9579
|
+
if (this.config.timestamps) {
|
|
9580
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
9581
|
+
mergedData.updatedAt = now;
|
|
9582
|
+
if (!mergedData.metadata) mergedData.metadata = {};
|
|
9583
|
+
mergedData.metadata.updatedAt = now;
|
|
9584
|
+
}
|
|
9585
|
+
const preProcessedData = await this.executeHooks("beforeUpdate", cloneDeep(mergedData));
|
|
9586
|
+
const completeData = { ...originalData, ...preProcessedData, id };
|
|
9587
|
+
const { isValid, errors, data } = await this.validate(cloneDeep(completeData));
|
|
9588
|
+
if (!isValid) {
|
|
9589
|
+
return {
|
|
9590
|
+
success: false,
|
|
9591
|
+
error: "Validation failed: " + (errors && errors.length ? JSON.stringify(errors) : "unknown"),
|
|
9592
|
+
validationErrors: errors
|
|
9593
|
+
};
|
|
9594
|
+
}
|
|
9595
|
+
const { id: validatedId, ...validatedAttributes } = data;
|
|
9596
|
+
const mappedData = await this.schema.mapper(validatedAttributes);
|
|
9597
|
+
mappedData._v = String(this.version);
|
|
9598
|
+
const behaviorImpl = getBehavior(this.behavior);
|
|
9599
|
+
const { mappedData: processedMetadata, body } = await behaviorImpl.handleUpdate({
|
|
9600
|
+
resource: this,
|
|
9601
|
+
id,
|
|
9602
|
+
data: validatedAttributes,
|
|
9603
|
+
mappedData,
|
|
9604
|
+
originalData: { ...attributesClone, id }
|
|
9605
|
+
});
|
|
9606
|
+
const key = this.getResourceKey(id);
|
|
9607
|
+
let existingContentType = void 0;
|
|
9608
|
+
let finalBody = body;
|
|
9609
|
+
if (body === "" && this.behavior !== "body-overflow") {
|
|
9610
|
+
const [ok2, err2, existingObject] = await tryFn(() => this.client.getObject(key));
|
|
9611
|
+
if (ok2 && existingObject.ContentLength > 0) {
|
|
9612
|
+
const existingBodyBuffer = Buffer.from(await existingObject.Body.transformToByteArray());
|
|
9613
|
+
const existingBodyString = existingBodyBuffer.toString();
|
|
9614
|
+
const [okParse, errParse] = await tryFn(() => Promise.resolve(JSON.parse(existingBodyString)));
|
|
9615
|
+
if (!okParse) {
|
|
9616
|
+
finalBody = existingBodyBuffer;
|
|
9617
|
+
existingContentType = existingObject.ContentType;
|
|
9618
|
+
}
|
|
9619
|
+
}
|
|
9620
|
+
}
|
|
9621
|
+
let finalContentType = existingContentType;
|
|
9622
|
+
if (finalBody && finalBody !== "" && !finalContentType) {
|
|
9623
|
+
const [okParse, errParse] = await tryFn(() => Promise.resolve(JSON.parse(finalBody)));
|
|
9624
|
+
if (okParse) finalContentType = "application/json";
|
|
9625
|
+
}
|
|
9626
|
+
const [ok, err, response] = await tryFn(() => this.client.putObject({
|
|
9627
|
+
key,
|
|
9628
|
+
body: finalBody,
|
|
9629
|
+
contentType: finalContentType,
|
|
9630
|
+
metadata: processedMetadata,
|
|
9631
|
+
ifMatch
|
|
9632
|
+
// ← Conditional write with ETag
|
|
9633
|
+
}));
|
|
9634
|
+
if (!ok) {
|
|
9635
|
+
if (err.name === "PreconditionFailed" || err.$metadata?.httpStatusCode === 412) {
|
|
9636
|
+
return {
|
|
9637
|
+
success: false,
|
|
9638
|
+
error: "ETag mismatch - object was modified by another process"
|
|
9639
|
+
};
|
|
9640
|
+
}
|
|
9641
|
+
return {
|
|
9642
|
+
success: false,
|
|
9643
|
+
error: err.message || "Update failed"
|
|
9644
|
+
};
|
|
9645
|
+
}
|
|
9646
|
+
const updatedData = await this.composeFullObjectFromWrite({
|
|
9647
|
+
id,
|
|
9648
|
+
metadata: processedMetadata,
|
|
9649
|
+
body: finalBody,
|
|
9650
|
+
behavior: this.behavior
|
|
9651
|
+
});
|
|
9652
|
+
const oldData = { ...originalData, id };
|
|
9653
|
+
const newData = { ...validatedAttributes, id };
|
|
9654
|
+
if (this.config.asyncPartitions && this.config.partitions && Object.keys(this.config.partitions).length > 0) {
|
|
9655
|
+
setImmediate(() => {
|
|
9656
|
+
this.handlePartitionReferenceUpdates(oldData, newData).catch((err2) => {
|
|
9657
|
+
this.emit("partitionIndexError", {
|
|
9658
|
+
operation: "updateConditional",
|
|
9659
|
+
id,
|
|
9660
|
+
error: err2,
|
|
9661
|
+
message: err2.message
|
|
9662
|
+
});
|
|
9663
|
+
});
|
|
9664
|
+
});
|
|
9665
|
+
const nonPartitionHooks = this.hooks.afterUpdate.filter(
|
|
9666
|
+
(hook) => !hook.toString().includes("handlePartitionReferenceUpdates")
|
|
9667
|
+
);
|
|
9668
|
+
let finalResult = updatedData;
|
|
9669
|
+
for (const hook of nonPartitionHooks) {
|
|
9670
|
+
finalResult = await hook(finalResult);
|
|
9671
|
+
}
|
|
9672
|
+
this.emit("update", {
|
|
9673
|
+
...updatedData,
|
|
9674
|
+
$before: { ...originalData },
|
|
9675
|
+
$after: { ...finalResult }
|
|
9676
|
+
});
|
|
9677
|
+
return {
|
|
9678
|
+
success: true,
|
|
9679
|
+
data: finalResult,
|
|
9680
|
+
etag: response.ETag
|
|
9681
|
+
};
|
|
9682
|
+
} else {
|
|
9683
|
+
await this.handlePartitionReferenceUpdates(oldData, newData);
|
|
9684
|
+
const finalResult = await this.executeHooks("afterUpdate", updatedData);
|
|
9685
|
+
this.emit("update", {
|
|
9686
|
+
...updatedData,
|
|
9687
|
+
$before: { ...originalData },
|
|
9688
|
+
$after: { ...finalResult }
|
|
9689
|
+
});
|
|
9690
|
+
return {
|
|
9691
|
+
success: true,
|
|
9692
|
+
data: finalResult,
|
|
9693
|
+
etag: response.ETag
|
|
9694
|
+
};
|
|
9695
|
+
}
|
|
9696
|
+
}
|
|
9697
|
+
/**
|
|
9698
|
+
* Delete a resource object by ID
|
|
9699
|
+
* @param {string} id - Resource ID
|
|
9700
|
+
* @returns {Promise<Object>} S3 delete response
|
|
9701
|
+
* @example
|
|
9702
|
+
* await resource.delete('user-123');
|
|
9703
|
+
*/
|
|
9704
|
+
async delete(id) {
|
|
9705
|
+
if (isEmpty(id)) {
|
|
9706
|
+
throw new Error("id cannot be empty");
|
|
9707
|
+
}
|
|
9708
|
+
let objectData;
|
|
9709
|
+
let deleteError = null;
|
|
9710
|
+
const [ok, err, data] = await tryFn(() => this.get(id));
|
|
9711
|
+
if (ok) {
|
|
9712
|
+
objectData = data;
|
|
9713
|
+
} else {
|
|
9714
|
+
objectData = { id };
|
|
9715
|
+
deleteError = err;
|
|
9716
|
+
}
|
|
9717
|
+
await this.executeHooks("beforeDelete", objectData);
|
|
9718
|
+
const key = this.getResourceKey(id);
|
|
9719
|
+
const [ok2, err2, response] = await tryFn(() => this.client.deleteObject(key));
|
|
9720
|
+
this.emit("delete", {
|
|
9721
|
+
...objectData,
|
|
9722
|
+
$before: { ...objectData },
|
|
9723
|
+
$after: null
|
|
9724
|
+
});
|
|
9725
|
+
if (deleteError) {
|
|
9726
|
+
throw mapAwsError(deleteError, {
|
|
9727
|
+
bucket: this.client.config.bucket,
|
|
9728
|
+
key,
|
|
9729
|
+
resourceName: this.name,
|
|
9730
|
+
operation: "delete",
|
|
9731
|
+
id
|
|
9732
|
+
});
|
|
9733
|
+
}
|
|
9734
|
+
if (!ok2) throw mapAwsError(err2, {
|
|
9735
|
+
key,
|
|
9736
|
+
resourceName: this.name,
|
|
9737
|
+
operation: "delete",
|
|
9738
|
+
id
|
|
9739
|
+
});
|
|
9740
|
+
if (this.config.asyncPartitions && this.config.partitions && Object.keys(this.config.partitions).length > 0) {
|
|
9741
|
+
setImmediate(() => {
|
|
9742
|
+
this.deletePartitionReferences(objectData).catch((err3) => {
|
|
9743
|
+
this.emit("partitionIndexError", {
|
|
9744
|
+
operation: "delete",
|
|
9745
|
+
id,
|
|
9746
|
+
error: err3,
|
|
9747
|
+
message: err3.message
|
|
9748
|
+
});
|
|
9749
|
+
});
|
|
9750
|
+
});
|
|
8803
9751
|
const nonPartitionHooks = this.hooks.afterDelete.filter(
|
|
8804
9752
|
(hook) => !hook.toString().includes("deletePartitionReferences")
|
|
8805
9753
|
);
|
|
@@ -10153,7 +11101,7 @@ class Database extends EventEmitter {
|
|
|
10153
11101
|
this.id = idGenerator(7);
|
|
10154
11102
|
this.version = "1";
|
|
10155
11103
|
this.s3dbVersion = (() => {
|
|
10156
|
-
const [ok, err, version] = tryFn(() => true ? "10.0.
|
|
11104
|
+
const [ok, err, version] = tryFn(() => true ? "10.0.3" : "latest");
|
|
10157
11105
|
return ok ? version : "latest";
|
|
10158
11106
|
})();
|
|
10159
11107
|
this.resources = {};
|
|
@@ -11406,16 +12354,20 @@ class S3dbReplicator extends BaseReplicator {
|
|
|
11406
12354
|
return resource;
|
|
11407
12355
|
}
|
|
11408
12356
|
_getDestResourceObj(resource) {
|
|
11409
|
-
const
|
|
12357
|
+
const db = this.targetDatabase || this.client;
|
|
12358
|
+
const available = Object.keys(db.resources || {});
|
|
11410
12359
|
const norm = normalizeResourceName$1(resource);
|
|
11411
12360
|
const found = available.find((r) => normalizeResourceName$1(r) === norm);
|
|
11412
12361
|
if (!found) {
|
|
11413
12362
|
throw new Error(`[S3dbReplicator] Destination resource not found: ${resource}. Available: ${available.join(", ")}`);
|
|
11414
12363
|
}
|
|
11415
|
-
return
|
|
12364
|
+
return db.resources[found];
|
|
11416
12365
|
}
|
|
11417
12366
|
async replicateBatch(resourceName, records) {
|
|
11418
|
-
if (
|
|
12367
|
+
if (this.enabled === false) {
|
|
12368
|
+
return { skipped: true, reason: "replicator_disabled" };
|
|
12369
|
+
}
|
|
12370
|
+
if (!this.shouldReplicateResource(resourceName)) {
|
|
11419
12371
|
return { skipped: true, reason: "resource_not_included" };
|
|
11420
12372
|
}
|
|
11421
12373
|
const results = [];
|
|
@@ -11526,11 +12478,12 @@ class SqsReplicator extends BaseReplicator {
|
|
|
11526
12478
|
this.client = client;
|
|
11527
12479
|
this.queueUrl = config.queueUrl;
|
|
11528
12480
|
this.queues = config.queues || {};
|
|
11529
|
-
this.defaultQueue = config.defaultQueue || config.defaultQueueUrl || config.queueUrlDefault;
|
|
12481
|
+
this.defaultQueue = config.defaultQueue || config.defaultQueueUrl || config.queueUrlDefault || null;
|
|
11530
12482
|
this.region = config.region || "us-east-1";
|
|
11531
12483
|
this.sqsClient = client || null;
|
|
11532
12484
|
this.messageGroupId = config.messageGroupId;
|
|
11533
12485
|
this.deduplicationId = config.deduplicationId;
|
|
12486
|
+
this.resourceQueueMap = config.resourceQueueMap || null;
|
|
11534
12487
|
if (Array.isArray(resources)) {
|
|
11535
12488
|
this.resources = {};
|
|
11536
12489
|
for (const resource of resources) {
|
|
@@ -11661,7 +12614,10 @@ class SqsReplicator extends BaseReplicator {
|
|
|
11661
12614
|
}
|
|
11662
12615
|
}
|
|
11663
12616
|
async replicate(resource, operation, data, id, beforeData = null) {
|
|
11664
|
-
if (
|
|
12617
|
+
if (this.enabled === false) {
|
|
12618
|
+
return { skipped: true, reason: "replicator_disabled" };
|
|
12619
|
+
}
|
|
12620
|
+
if (!this.shouldReplicateResource(resource)) {
|
|
11665
12621
|
return { skipped: true, reason: "resource_not_included" };
|
|
11666
12622
|
}
|
|
11667
12623
|
const [ok, err, result] = await tryFn(async () => {
|
|
@@ -11705,7 +12661,10 @@ class SqsReplicator extends BaseReplicator {
|
|
|
11705
12661
|
return { success: false, error: err.message };
|
|
11706
12662
|
}
|
|
11707
12663
|
async replicateBatch(resource, records) {
|
|
11708
|
-
if (
|
|
12664
|
+
if (this.enabled === false) {
|
|
12665
|
+
return { skipped: true, reason: "replicator_disabled" };
|
|
12666
|
+
}
|
|
12667
|
+
if (!this.shouldReplicateResource(resource)) {
|
|
11709
12668
|
return { skipped: true, reason: "resource_not_included" };
|
|
11710
12669
|
}
|
|
11711
12670
|
const [ok, err, result] = await tryFn(async () => {
|
|
@@ -11859,22 +12818,23 @@ class ReplicatorPlugin extends Plugin {
|
|
|
11859
12818
|
replicators: options.replicators || [],
|
|
11860
12819
|
logErrors: options.logErrors !== false,
|
|
11861
12820
|
replicatorLogResource: options.replicatorLogResource || "replicator_log",
|
|
12821
|
+
persistReplicatorLog: options.persistReplicatorLog || false,
|
|
11862
12822
|
enabled: options.enabled !== false,
|
|
11863
12823
|
batchSize: options.batchSize || 100,
|
|
11864
12824
|
maxRetries: options.maxRetries || 3,
|
|
11865
12825
|
timeout: options.timeout || 3e4,
|
|
11866
|
-
verbose: options.verbose || false
|
|
11867
|
-
...options
|
|
12826
|
+
verbose: options.verbose || false
|
|
11868
12827
|
};
|
|
11869
12828
|
this.replicators = [];
|
|
11870
12829
|
this.database = null;
|
|
11871
12830
|
this.eventListenersInstalled = /* @__PURE__ */ new Set();
|
|
11872
|
-
|
|
11873
|
-
|
|
11874
|
-
|
|
11875
|
-
|
|
11876
|
-
|
|
11877
|
-
|
|
12831
|
+
this.eventHandlers = /* @__PURE__ */ new Map();
|
|
12832
|
+
this.stats = {
|
|
12833
|
+
totalReplications: 0,
|
|
12834
|
+
totalErrors: 0,
|
|
12835
|
+
lastSync: null
|
|
12836
|
+
};
|
|
12837
|
+
this._afterCreateResourceHook = null;
|
|
11878
12838
|
}
|
|
11879
12839
|
// Helper to filter out internal S3DB fields
|
|
11880
12840
|
filterInternalFields(obj) {
|
|
@@ -11895,7 +12855,7 @@ class ReplicatorPlugin extends Plugin {
|
|
|
11895
12855
|
if (!resource || this.eventListenersInstalled.has(resource.name) || resource.name === this.config.replicatorLogResource) {
|
|
11896
12856
|
return;
|
|
11897
12857
|
}
|
|
11898
|
-
|
|
12858
|
+
const insertHandler = async (data) => {
|
|
11899
12859
|
const [ok, error] = await tryFn(async () => {
|
|
11900
12860
|
const completeData = { ...data, createdAt: (/* @__PURE__ */ new Date()).toISOString() };
|
|
11901
12861
|
await plugin.processReplicatorEvent("insert", resource.name, completeData.id, completeData);
|
|
@@ -11906,8 +12866,8 @@ class ReplicatorPlugin extends Plugin {
|
|
|
11906
12866
|
}
|
|
11907
12867
|
this.emit("error", { operation: "insert", error: error.message, resource: resource.name });
|
|
11908
12868
|
}
|
|
11909
|
-
}
|
|
11910
|
-
|
|
12869
|
+
};
|
|
12870
|
+
const updateHandler = async (data, beforeData) => {
|
|
11911
12871
|
const [ok, error] = await tryFn(async () => {
|
|
11912
12872
|
const completeData = await plugin.getCompleteData(resource, data);
|
|
11913
12873
|
const dataWithTimestamp = { ...completeData, updatedAt: (/* @__PURE__ */ new Date()).toISOString() };
|
|
@@ -11919,8 +12879,8 @@ class ReplicatorPlugin extends Plugin {
|
|
|
11919
12879
|
}
|
|
11920
12880
|
this.emit("error", { operation: "update", error: error.message, resource: resource.name });
|
|
11921
12881
|
}
|
|
11922
|
-
}
|
|
11923
|
-
|
|
12882
|
+
};
|
|
12883
|
+
const deleteHandler = async (data) => {
|
|
11924
12884
|
const [ok, error] = await tryFn(async () => {
|
|
11925
12885
|
await plugin.processReplicatorEvent("delete", resource.name, data.id, data);
|
|
11926
12886
|
});
|
|
@@ -11930,14 +12890,22 @@ class ReplicatorPlugin extends Plugin {
|
|
|
11930
12890
|
}
|
|
11931
12891
|
this.emit("error", { operation: "delete", error: error.message, resource: resource.name });
|
|
11932
12892
|
}
|
|
11933
|
-
}
|
|
12893
|
+
};
|
|
12894
|
+
this.eventHandlers.set(resource.name, {
|
|
12895
|
+
insert: insertHandler,
|
|
12896
|
+
update: updateHandler,
|
|
12897
|
+
delete: deleteHandler
|
|
12898
|
+
});
|
|
12899
|
+
resource.on("insert", insertHandler);
|
|
12900
|
+
resource.on("update", updateHandler);
|
|
12901
|
+
resource.on("delete", deleteHandler);
|
|
11934
12902
|
this.eventListenersInstalled.add(resource.name);
|
|
11935
12903
|
}
|
|
11936
12904
|
async setup(database) {
|
|
11937
12905
|
this.database = database;
|
|
11938
12906
|
if (this.config.persistReplicatorLog) {
|
|
11939
12907
|
const [ok, err, logResource] = await tryFn(() => database.createResource({
|
|
11940
|
-
name: this.config.replicatorLogResource || "
|
|
12908
|
+
name: this.config.replicatorLogResource || "plg_replicator_logs",
|
|
11941
12909
|
attributes: {
|
|
11942
12910
|
id: "string|required",
|
|
11943
12911
|
resource: "string|required",
|
|
@@ -11951,13 +12919,13 @@ class ReplicatorPlugin extends Plugin {
|
|
|
11951
12919
|
if (ok) {
|
|
11952
12920
|
this.replicatorLogResource = logResource;
|
|
11953
12921
|
} else {
|
|
11954
|
-
this.replicatorLogResource = database.resources[this.config.replicatorLogResource || "
|
|
12922
|
+
this.replicatorLogResource = database.resources[this.config.replicatorLogResource || "plg_replicator_logs"];
|
|
11955
12923
|
}
|
|
11956
12924
|
}
|
|
11957
12925
|
await this.initializeReplicators(database);
|
|
11958
12926
|
this.installDatabaseHooks();
|
|
11959
12927
|
for (const resource of Object.values(database.resources)) {
|
|
11960
|
-
if (resource.name !== (this.config.replicatorLogResource || "
|
|
12928
|
+
if (resource.name !== (this.config.replicatorLogResource || "plg_replicator_logs")) {
|
|
11961
12929
|
this.installEventListeners(resource, database, this);
|
|
11962
12930
|
}
|
|
11963
12931
|
}
|
|
@@ -11973,14 +12941,18 @@ class ReplicatorPlugin extends Plugin {
|
|
|
11973
12941
|
this.removeDatabaseHooks();
|
|
11974
12942
|
}
|
|
11975
12943
|
installDatabaseHooks() {
|
|
11976
|
-
this.
|
|
11977
|
-
if (resource.name !== (this.config.replicatorLogResource || "
|
|
12944
|
+
this._afterCreateResourceHook = (resource) => {
|
|
12945
|
+
if (resource.name !== (this.config.replicatorLogResource || "plg_replicator_logs")) {
|
|
11978
12946
|
this.installEventListeners(resource, this.database, this);
|
|
11979
12947
|
}
|
|
11980
|
-
}
|
|
12948
|
+
};
|
|
12949
|
+
this.database.addHook("afterCreateResource", this._afterCreateResourceHook);
|
|
11981
12950
|
}
|
|
11982
12951
|
removeDatabaseHooks() {
|
|
11983
|
-
|
|
12952
|
+
if (this._afterCreateResourceHook) {
|
|
12953
|
+
this.database.removeHook("afterCreateResource", this._afterCreateResourceHook);
|
|
12954
|
+
this._afterCreateResourceHook = null;
|
|
12955
|
+
}
|
|
11984
12956
|
}
|
|
11985
12957
|
createReplicator(driver, config, resources, client) {
|
|
11986
12958
|
return createReplicator(driver, config, resources, client);
|
|
@@ -12005,9 +12977,9 @@ class ReplicatorPlugin extends Plugin {
|
|
|
12005
12977
|
async retryWithBackoff(operation, maxRetries = 3) {
|
|
12006
12978
|
let lastError;
|
|
12007
12979
|
for (let attempt = 1; attempt <= maxRetries; attempt++) {
|
|
12008
|
-
const [ok, error] = await tryFn(operation);
|
|
12980
|
+
const [ok, error, result] = await tryFn(operation);
|
|
12009
12981
|
if (ok) {
|
|
12010
|
-
return
|
|
12982
|
+
return result;
|
|
12011
12983
|
} else {
|
|
12012
12984
|
lastError = error;
|
|
12013
12985
|
if (this.config.verbose) {
|
|
@@ -12102,7 +13074,7 @@ class ReplicatorPlugin extends Plugin {
|
|
|
12102
13074
|
});
|
|
12103
13075
|
return Promise.allSettled(promises);
|
|
12104
13076
|
}
|
|
12105
|
-
async
|
|
13077
|
+
async processReplicatorItem(item) {
|
|
12106
13078
|
const applicableReplicators = this.replicators.filter((replicator) => {
|
|
12107
13079
|
const should = replicator.shouldReplicateResource && replicator.shouldReplicateResource(item.resourceName, item.operation);
|
|
12108
13080
|
return should;
|
|
@@ -12162,12 +13134,9 @@ class ReplicatorPlugin extends Plugin {
|
|
|
12162
13134
|
});
|
|
12163
13135
|
return Promise.allSettled(promises);
|
|
12164
13136
|
}
|
|
12165
|
-
async
|
|
13137
|
+
async logReplicator(item) {
|
|
12166
13138
|
const logRes = this.replicatorLog || this.database.resources[normalizeResourceName(this.config.replicatorLogResource)];
|
|
12167
13139
|
if (!logRes) {
|
|
12168
|
-
if (this.database) {
|
|
12169
|
-
if (this.database.options && this.database.options.connectionString) ;
|
|
12170
|
-
}
|
|
12171
13140
|
this.emit("replicator.log.failed", { error: "replicator log resource not found", item });
|
|
12172
13141
|
return;
|
|
12173
13142
|
}
|
|
@@ -12189,7 +13158,7 @@ class ReplicatorPlugin extends Plugin {
|
|
|
12189
13158
|
this.emit("replicator.log.failed", { error: err, item });
|
|
12190
13159
|
}
|
|
12191
13160
|
}
|
|
12192
|
-
async
|
|
13161
|
+
async updateReplicatorLog(logId, updates) {
|
|
12193
13162
|
if (!this.replicatorLog) return;
|
|
12194
13163
|
const [ok, err] = await tryFn(async () => {
|
|
12195
13164
|
await this.replicatorLog.update(logId, {
|
|
@@ -12202,7 +13171,7 @@ class ReplicatorPlugin extends Plugin {
|
|
|
12202
13171
|
}
|
|
12203
13172
|
}
|
|
12204
13173
|
// Utility methods
|
|
12205
|
-
async
|
|
13174
|
+
async getReplicatorStats() {
|
|
12206
13175
|
const replicatorStats = await Promise.all(
|
|
12207
13176
|
this.replicators.map(async (replicator) => {
|
|
12208
13177
|
const status = await replicator.getStatus();
|
|
@@ -12216,15 +13185,11 @@ class ReplicatorPlugin extends Plugin {
|
|
|
12216
13185
|
);
|
|
12217
13186
|
return {
|
|
12218
13187
|
replicators: replicatorStats,
|
|
12219
|
-
queue: {
|
|
12220
|
-
length: this.queue.length,
|
|
12221
|
-
isProcessing: this.isProcessing
|
|
12222
|
-
},
|
|
12223
13188
|
stats: this.stats,
|
|
12224
13189
|
lastSync: this.stats.lastSync
|
|
12225
13190
|
};
|
|
12226
13191
|
}
|
|
12227
|
-
async
|
|
13192
|
+
async getReplicatorLogs(options = {}) {
|
|
12228
13193
|
if (!this.replicatorLog) {
|
|
12229
13194
|
return [];
|
|
12230
13195
|
}
|
|
@@ -12235,32 +13200,32 @@ class ReplicatorPlugin extends Plugin {
|
|
|
12235
13200
|
limit = 100,
|
|
12236
13201
|
offset = 0
|
|
12237
13202
|
} = options;
|
|
12238
|
-
|
|
13203
|
+
const filter = {};
|
|
12239
13204
|
if (resourceName) {
|
|
12240
|
-
|
|
13205
|
+
filter.resourceName = resourceName;
|
|
12241
13206
|
}
|
|
12242
13207
|
if (operation) {
|
|
12243
|
-
|
|
13208
|
+
filter.operation = operation;
|
|
12244
13209
|
}
|
|
12245
13210
|
if (status) {
|
|
12246
|
-
|
|
13211
|
+
filter.status = status;
|
|
12247
13212
|
}
|
|
12248
|
-
const logs = await this.replicatorLog.
|
|
12249
|
-
return logs
|
|
13213
|
+
const logs = await this.replicatorLog.query(filter, { limit, offset });
|
|
13214
|
+
return logs || [];
|
|
12250
13215
|
}
|
|
12251
|
-
async
|
|
13216
|
+
async retryFailedReplicators() {
|
|
12252
13217
|
if (!this.replicatorLog) {
|
|
12253
13218
|
return { retried: 0 };
|
|
12254
13219
|
}
|
|
12255
|
-
const failedLogs = await this.replicatorLog.
|
|
13220
|
+
const failedLogs = await this.replicatorLog.query({
|
|
12256
13221
|
status: "failed"
|
|
12257
13222
|
});
|
|
12258
13223
|
let retried = 0;
|
|
12259
|
-
for (const log of failedLogs) {
|
|
13224
|
+
for (const log of failedLogs || []) {
|
|
12260
13225
|
const [ok, err] = await tryFn(async () => {
|
|
12261
13226
|
await this.processReplicatorEvent(
|
|
12262
|
-
log.resourceName,
|
|
12263
13227
|
log.operation,
|
|
13228
|
+
log.resourceName,
|
|
12264
13229
|
log.recordId,
|
|
12265
13230
|
log.data
|
|
12266
13231
|
);
|
|
@@ -12278,13 +13243,21 @@ class ReplicatorPlugin extends Plugin {
|
|
|
12278
13243
|
}
|
|
12279
13244
|
this.stats.lastSync = (/* @__PURE__ */ new Date()).toISOString();
|
|
12280
13245
|
for (const resourceName in this.database.resources) {
|
|
12281
|
-
if (normalizeResourceName(resourceName) === normalizeResourceName("
|
|
13246
|
+
if (normalizeResourceName(resourceName) === normalizeResourceName("plg_replicator_logs")) continue;
|
|
12282
13247
|
if (replicator.shouldReplicateResource(resourceName)) {
|
|
12283
13248
|
this.emit("replicator.sync.resource", { resourceName, replicatorId });
|
|
12284
13249
|
const resource = this.database.resources[resourceName];
|
|
12285
|
-
|
|
12286
|
-
|
|
12287
|
-
|
|
13250
|
+
let offset = 0;
|
|
13251
|
+
const pageSize = this.config.batchSize || 100;
|
|
13252
|
+
while (true) {
|
|
13253
|
+
const [ok, err, page] = await tryFn(() => resource.page({ offset, size: pageSize }));
|
|
13254
|
+
if (!ok || !page) break;
|
|
13255
|
+
const records = Array.isArray(page) ? page : page.items || [];
|
|
13256
|
+
if (records.length === 0) break;
|
|
13257
|
+
for (const record of records) {
|
|
13258
|
+
await replicator.replicate(resourceName, "insert", record, record.id);
|
|
13259
|
+
}
|
|
13260
|
+
offset += pageSize;
|
|
12288
13261
|
}
|
|
12289
13262
|
}
|
|
12290
13263
|
}
|
|
@@ -12312,9 +13285,21 @@ class ReplicatorPlugin extends Plugin {
|
|
|
12312
13285
|
});
|
|
12313
13286
|
await Promise.allSettled(cleanupPromises);
|
|
12314
13287
|
}
|
|
13288
|
+
if (this.database && this.database.resources) {
|
|
13289
|
+
for (const resourceName of this.eventListenersInstalled) {
|
|
13290
|
+
const resource = this.database.resources[resourceName];
|
|
13291
|
+
const handlers = this.eventHandlers.get(resourceName);
|
|
13292
|
+
if (resource && handlers) {
|
|
13293
|
+
resource.off("insert", handlers.insert);
|
|
13294
|
+
resource.off("update", handlers.update);
|
|
13295
|
+
resource.off("delete", handlers.delete);
|
|
13296
|
+
}
|
|
13297
|
+
}
|
|
13298
|
+
}
|
|
12315
13299
|
this.replicators = [];
|
|
12316
13300
|
this.database = null;
|
|
12317
13301
|
this.eventListenersInstalled.clear();
|
|
13302
|
+
this.eventHandlers.clear();
|
|
12318
13303
|
this.removeAllListeners();
|
|
12319
13304
|
});
|
|
12320
13305
|
if (!ok) {
|
|
@@ -12328,46 +13313,591 @@ class ReplicatorPlugin extends Plugin {
|
|
|
12328
13313
|
}
|
|
12329
13314
|
}
|
|
12330
13315
|
|
|
12331
|
-
class
|
|
13316
|
+
class S3QueuePlugin extends Plugin {
|
|
12332
13317
|
constructor(options = {}) {
|
|
12333
|
-
super();
|
|
13318
|
+
super(options);
|
|
13319
|
+
if (!options.resource) {
|
|
13320
|
+
throw new Error('S3QueuePlugin requires "resource" option');
|
|
13321
|
+
}
|
|
12334
13322
|
this.config = {
|
|
12335
|
-
|
|
12336
|
-
|
|
12337
|
-
|
|
12338
|
-
|
|
12339
|
-
|
|
12340
|
-
|
|
12341
|
-
|
|
13323
|
+
resource: options.resource,
|
|
13324
|
+
visibilityTimeout: options.visibilityTimeout || 3e4,
|
|
13325
|
+
// 30 seconds
|
|
13326
|
+
pollInterval: options.pollInterval || 1e3,
|
|
13327
|
+
// 1 second
|
|
13328
|
+
maxAttempts: options.maxAttempts || 3,
|
|
13329
|
+
concurrency: options.concurrency || 1,
|
|
13330
|
+
deadLetterResource: options.deadLetterResource || null,
|
|
13331
|
+
autoStart: options.autoStart !== false,
|
|
13332
|
+
onMessage: options.onMessage,
|
|
13333
|
+
onError: options.onError,
|
|
13334
|
+
onComplete: options.onComplete,
|
|
12342
13335
|
verbose: options.verbose || false,
|
|
12343
|
-
onJobStart: options.onJobStart || null,
|
|
12344
|
-
onJobComplete: options.onJobComplete || null,
|
|
12345
|
-
onJobError: options.onJobError || null,
|
|
12346
13336
|
...options
|
|
12347
13337
|
};
|
|
12348
|
-
this.
|
|
12349
|
-
this.
|
|
12350
|
-
this.
|
|
12351
|
-
this.
|
|
12352
|
-
this.
|
|
12353
|
-
this.
|
|
13338
|
+
this.queueResource = null;
|
|
13339
|
+
this.targetResource = null;
|
|
13340
|
+
this.deadLetterResourceObj = null;
|
|
13341
|
+
this.workers = [];
|
|
13342
|
+
this.isRunning = false;
|
|
13343
|
+
this.workerId = `worker-${Date.now()}-${Math.random().toString(36).slice(2, 9)}`;
|
|
13344
|
+
this.processedCache = /* @__PURE__ */ new Map();
|
|
13345
|
+
this.cacheCleanupInterval = null;
|
|
13346
|
+
this.lockCleanupInterval = null;
|
|
12354
13347
|
}
|
|
12355
|
-
|
|
12356
|
-
|
|
12357
|
-
|
|
13348
|
+
async onSetup() {
|
|
13349
|
+
this.targetResource = this.database.resources[this.config.resource];
|
|
13350
|
+
if (!this.targetResource) {
|
|
13351
|
+
throw new Error(`S3QueuePlugin: resource '${this.config.resource}' not found`);
|
|
12358
13352
|
}
|
|
12359
|
-
|
|
12360
|
-
|
|
12361
|
-
|
|
12362
|
-
|
|
12363
|
-
|
|
12364
|
-
|
|
12365
|
-
|
|
12366
|
-
|
|
12367
|
-
|
|
12368
|
-
|
|
13353
|
+
const queueName = `${this.config.resource}_queue`;
|
|
13354
|
+
const [ok, err] = await tryFn(
|
|
13355
|
+
() => this.database.createResource({
|
|
13356
|
+
name: queueName,
|
|
13357
|
+
attributes: {
|
|
13358
|
+
id: "string|required",
|
|
13359
|
+
originalId: "string|required",
|
|
13360
|
+
// ID do registro original
|
|
13361
|
+
status: "string|required",
|
|
13362
|
+
// pending/processing/completed/failed/dead
|
|
13363
|
+
visibleAt: "number|required",
|
|
13364
|
+
// Timestamp de visibilidade
|
|
13365
|
+
claimedBy: "string|optional",
|
|
13366
|
+
// Worker que claimed
|
|
13367
|
+
claimedAt: "number|optional",
|
|
13368
|
+
// Timestamp do claim
|
|
13369
|
+
attempts: "number|default:0",
|
|
13370
|
+
maxAttempts: "number|default:3",
|
|
13371
|
+
error: "string|optional",
|
|
13372
|
+
result: "json|optional",
|
|
13373
|
+
createdAt: "string|required",
|
|
13374
|
+
completedAt: "number|optional"
|
|
13375
|
+
},
|
|
13376
|
+
behavior: "body-overflow",
|
|
13377
|
+
timestamps: true,
|
|
13378
|
+
asyncPartitions: true,
|
|
13379
|
+
partitions: {
|
|
13380
|
+
byStatus: { fields: { status: "string" } },
|
|
13381
|
+
byDate: { fields: { createdAt: "string|maxlength:10" } }
|
|
13382
|
+
}
|
|
13383
|
+
})
|
|
13384
|
+
);
|
|
13385
|
+
if (!ok && !this.database.resources[queueName]) {
|
|
13386
|
+
throw new Error(`Failed to create queue resource: ${err?.message}`);
|
|
12369
13387
|
}
|
|
12370
|
-
|
|
13388
|
+
this.queueResource = this.database.resources[queueName];
|
|
13389
|
+
const lockName = `${this.config.resource}_locks`;
|
|
13390
|
+
const [okLock, errLock] = await tryFn(
|
|
13391
|
+
() => this.database.createResource({
|
|
13392
|
+
name: lockName,
|
|
13393
|
+
attributes: {
|
|
13394
|
+
id: "string|required",
|
|
13395
|
+
workerId: "string|required",
|
|
13396
|
+
timestamp: "number|required",
|
|
13397
|
+
ttl: "number|default:5000"
|
|
13398
|
+
},
|
|
13399
|
+
behavior: "body-overflow",
|
|
13400
|
+
timestamps: false
|
|
13401
|
+
})
|
|
13402
|
+
);
|
|
13403
|
+
if (okLock || this.database.resources[lockName]) {
|
|
13404
|
+
this.lockResource = this.database.resources[lockName];
|
|
13405
|
+
} else {
|
|
13406
|
+
this.lockResource = null;
|
|
13407
|
+
if (this.config.verbose) {
|
|
13408
|
+
console.log(`[S3QueuePlugin] Lock resource creation failed, locking disabled: ${errLock?.message}`);
|
|
13409
|
+
}
|
|
13410
|
+
}
|
|
13411
|
+
this.addHelperMethods();
|
|
13412
|
+
if (this.config.deadLetterResource) {
|
|
13413
|
+
await this.createDeadLetterResource();
|
|
13414
|
+
}
|
|
13415
|
+
if (this.config.verbose) {
|
|
13416
|
+
console.log(`[S3QueuePlugin] Setup completed for resource '${this.config.resource}'`);
|
|
13417
|
+
}
|
|
13418
|
+
}
|
|
13419
|
+
async onStart() {
|
|
13420
|
+
if (this.config.autoStart && this.config.onMessage) {
|
|
13421
|
+
await this.startProcessing();
|
|
13422
|
+
}
|
|
13423
|
+
}
|
|
13424
|
+
async onStop() {
|
|
13425
|
+
await this.stopProcessing();
|
|
13426
|
+
}
|
|
13427
|
+
addHelperMethods() {
|
|
13428
|
+
const plugin = this;
|
|
13429
|
+
const resource = this.targetResource;
|
|
13430
|
+
resource.enqueue = async function(data, options = {}) {
|
|
13431
|
+
const recordData = {
|
|
13432
|
+
id: data.id || idGenerator(),
|
|
13433
|
+
...data
|
|
13434
|
+
};
|
|
13435
|
+
const record = await resource.insert(recordData);
|
|
13436
|
+
const queueEntry = {
|
|
13437
|
+
id: idGenerator(),
|
|
13438
|
+
originalId: record.id,
|
|
13439
|
+
status: "pending",
|
|
13440
|
+
visibleAt: Date.now(),
|
|
13441
|
+
attempts: 0,
|
|
13442
|
+
maxAttempts: options.maxAttempts || plugin.config.maxAttempts,
|
|
13443
|
+
createdAt: (/* @__PURE__ */ new Date()).toISOString().slice(0, 10)
|
|
13444
|
+
};
|
|
13445
|
+
await plugin.queueResource.insert(queueEntry);
|
|
13446
|
+
plugin.emit("message.enqueued", { id: record.id, queueId: queueEntry.id });
|
|
13447
|
+
return record;
|
|
13448
|
+
};
|
|
13449
|
+
resource.queueStats = async function() {
|
|
13450
|
+
return await plugin.getStats();
|
|
13451
|
+
};
|
|
13452
|
+
resource.startProcessing = async function(handler, options = {}) {
|
|
13453
|
+
return await plugin.startProcessing(handler, options);
|
|
13454
|
+
};
|
|
13455
|
+
resource.stopProcessing = async function() {
|
|
13456
|
+
return await plugin.stopProcessing();
|
|
13457
|
+
};
|
|
13458
|
+
}
|
|
13459
|
+
async startProcessing(handler = null, options = {}) {
|
|
13460
|
+
if (this.isRunning) {
|
|
13461
|
+
if (this.config.verbose) {
|
|
13462
|
+
console.log("[S3QueuePlugin] Already running");
|
|
13463
|
+
}
|
|
13464
|
+
return;
|
|
13465
|
+
}
|
|
13466
|
+
const messageHandler = handler || this.config.onMessage;
|
|
13467
|
+
if (!messageHandler) {
|
|
13468
|
+
throw new Error("S3QueuePlugin: onMessage handler required");
|
|
13469
|
+
}
|
|
13470
|
+
this.isRunning = true;
|
|
13471
|
+
const concurrency = options.concurrency || this.config.concurrency;
|
|
13472
|
+
this.cacheCleanupInterval = setInterval(() => {
|
|
13473
|
+
const now = Date.now();
|
|
13474
|
+
const maxAge = 3e4;
|
|
13475
|
+
for (const [queueId, timestamp] of this.processedCache.entries()) {
|
|
13476
|
+
if (now - timestamp > maxAge) {
|
|
13477
|
+
this.processedCache.delete(queueId);
|
|
13478
|
+
}
|
|
13479
|
+
}
|
|
13480
|
+
}, 5e3);
|
|
13481
|
+
this.lockCleanupInterval = setInterval(() => {
|
|
13482
|
+
this.cleanupStaleLocks().catch((err) => {
|
|
13483
|
+
if (this.config.verbose) {
|
|
13484
|
+
console.log(`[lockCleanup] Error: ${err.message}`);
|
|
13485
|
+
}
|
|
13486
|
+
});
|
|
13487
|
+
}, 1e4);
|
|
13488
|
+
for (let i = 0; i < concurrency; i++) {
|
|
13489
|
+
const worker = this.createWorker(messageHandler, i);
|
|
13490
|
+
this.workers.push(worker);
|
|
13491
|
+
}
|
|
13492
|
+
if (this.config.verbose) {
|
|
13493
|
+
console.log(`[S3QueuePlugin] Started ${concurrency} workers`);
|
|
13494
|
+
}
|
|
13495
|
+
this.emit("workers.started", { concurrency, workerId: this.workerId });
|
|
13496
|
+
}
|
|
13497
|
+
async stopProcessing() {
|
|
13498
|
+
if (!this.isRunning) return;
|
|
13499
|
+
this.isRunning = false;
|
|
13500
|
+
if (this.cacheCleanupInterval) {
|
|
13501
|
+
clearInterval(this.cacheCleanupInterval);
|
|
13502
|
+
this.cacheCleanupInterval = null;
|
|
13503
|
+
}
|
|
13504
|
+
if (this.lockCleanupInterval) {
|
|
13505
|
+
clearInterval(this.lockCleanupInterval);
|
|
13506
|
+
this.lockCleanupInterval = null;
|
|
13507
|
+
}
|
|
13508
|
+
await Promise.all(this.workers);
|
|
13509
|
+
this.workers = [];
|
|
13510
|
+
this.processedCache.clear();
|
|
13511
|
+
if (this.config.verbose) {
|
|
13512
|
+
console.log("[S3QueuePlugin] Stopped all workers");
|
|
13513
|
+
}
|
|
13514
|
+
this.emit("workers.stopped", { workerId: this.workerId });
|
|
13515
|
+
}
|
|
13516
|
+
createWorker(handler, workerIndex) {
|
|
13517
|
+
return (async () => {
|
|
13518
|
+
while (this.isRunning) {
|
|
13519
|
+
try {
|
|
13520
|
+
const message = await this.claimMessage();
|
|
13521
|
+
if (message) {
|
|
13522
|
+
await this.processMessage(message, handler);
|
|
13523
|
+
} else {
|
|
13524
|
+
await new Promise((resolve) => setTimeout(resolve, this.config.pollInterval));
|
|
13525
|
+
}
|
|
13526
|
+
} catch (error) {
|
|
13527
|
+
if (this.config.verbose) {
|
|
13528
|
+
console.error(`[Worker ${workerIndex}] Error:`, error.message);
|
|
13529
|
+
}
|
|
13530
|
+
await new Promise((resolve) => setTimeout(resolve, 1e3));
|
|
13531
|
+
}
|
|
13532
|
+
}
|
|
13533
|
+
})();
|
|
13534
|
+
}
|
|
13535
|
+
async claimMessage() {
|
|
13536
|
+
const now = Date.now();
|
|
13537
|
+
const [ok, err, messages] = await tryFn(
|
|
13538
|
+
() => this.queueResource.query({
|
|
13539
|
+
status: "pending"
|
|
13540
|
+
})
|
|
13541
|
+
);
|
|
13542
|
+
if (!ok || !messages || messages.length === 0) {
|
|
13543
|
+
return null;
|
|
13544
|
+
}
|
|
13545
|
+
const available = messages.filter((m) => m.visibleAt <= now);
|
|
13546
|
+
if (available.length === 0) {
|
|
13547
|
+
return null;
|
|
13548
|
+
}
|
|
13549
|
+
for (const msg of available) {
|
|
13550
|
+
const claimed = await this.attemptClaim(msg);
|
|
13551
|
+
if (claimed) {
|
|
13552
|
+
return claimed;
|
|
13553
|
+
}
|
|
13554
|
+
}
|
|
13555
|
+
return null;
|
|
13556
|
+
}
|
|
13557
|
+
/**
|
|
13558
|
+
* Acquire a distributed lock using ETag-based conditional updates
|
|
13559
|
+
* This ensures only one worker can claim a message at a time
|
|
13560
|
+
*
|
|
13561
|
+
* Uses a two-step process:
|
|
13562
|
+
* 1. Create lock resource (similar to queue resource) if not exists
|
|
13563
|
+
* 2. Try to claim lock using ETag-based conditional update
|
|
13564
|
+
*/
|
|
13565
|
+
async acquireLock(messageId) {
|
|
13566
|
+
if (!this.lockResource) {
|
|
13567
|
+
return true;
|
|
13568
|
+
}
|
|
13569
|
+
const lockId = `lock-${messageId}`;
|
|
13570
|
+
const now = Date.now();
|
|
13571
|
+
try {
|
|
13572
|
+
const [okGet, errGet, existingLock] = await tryFn(
|
|
13573
|
+
() => this.lockResource.get(lockId)
|
|
13574
|
+
);
|
|
13575
|
+
if (existingLock) {
|
|
13576
|
+
const lockAge = now - existingLock.timestamp;
|
|
13577
|
+
if (lockAge < existingLock.ttl) {
|
|
13578
|
+
return false;
|
|
13579
|
+
}
|
|
13580
|
+
const [ok, err, result] = await tryFn(
|
|
13581
|
+
() => this.lockResource.updateConditional(lockId, {
|
|
13582
|
+
workerId: this.workerId,
|
|
13583
|
+
timestamp: now,
|
|
13584
|
+
ttl: 5e3
|
|
13585
|
+
}, {
|
|
13586
|
+
ifMatch: existingLock._etag
|
|
13587
|
+
})
|
|
13588
|
+
);
|
|
13589
|
+
return ok && result.success;
|
|
13590
|
+
}
|
|
13591
|
+
const [okCreate, errCreate] = await tryFn(
|
|
13592
|
+
() => this.lockResource.insert({
|
|
13593
|
+
id: lockId,
|
|
13594
|
+
workerId: this.workerId,
|
|
13595
|
+
timestamp: now,
|
|
13596
|
+
ttl: 5e3
|
|
13597
|
+
})
|
|
13598
|
+
);
|
|
13599
|
+
return okCreate;
|
|
13600
|
+
} catch (error) {
|
|
13601
|
+
if (this.config.verbose) {
|
|
13602
|
+
console.log(`[acquireLock] Error: ${error.message}`);
|
|
13603
|
+
}
|
|
13604
|
+
return false;
|
|
13605
|
+
}
|
|
13606
|
+
}
|
|
13607
|
+
/**
|
|
13608
|
+
* Release a distributed lock by deleting the lock record
|
|
13609
|
+
*/
|
|
13610
|
+
async releaseLock(messageId) {
|
|
13611
|
+
if (!this.lockResource) {
|
|
13612
|
+
return;
|
|
13613
|
+
}
|
|
13614
|
+
const lockId = `lock-${messageId}`;
|
|
13615
|
+
try {
|
|
13616
|
+
await this.lockResource.delete(lockId);
|
|
13617
|
+
} catch (error) {
|
|
13618
|
+
if (this.config.verbose) {
|
|
13619
|
+
console.log(`[releaseLock] Failed to release lock for ${messageId}: ${error.message}`);
|
|
13620
|
+
}
|
|
13621
|
+
}
|
|
13622
|
+
}
|
|
13623
|
+
/**
|
|
13624
|
+
* Clean up stale locks (older than TTL)
|
|
13625
|
+
* This prevents deadlocks if a worker crashes while holding a lock
|
|
13626
|
+
*/
|
|
13627
|
+
async cleanupStaleLocks() {
|
|
13628
|
+
if (!this.lockResource) {
|
|
13629
|
+
return;
|
|
13630
|
+
}
|
|
13631
|
+
const now = Date.now();
|
|
13632
|
+
try {
|
|
13633
|
+
const locks = await this.lockResource.list();
|
|
13634
|
+
for (const lock of locks) {
|
|
13635
|
+
const lockAge = now - lock.timestamp;
|
|
13636
|
+
if (lockAge > lock.ttl) {
|
|
13637
|
+
await this.lockResource.delete(lock.id);
|
|
13638
|
+
if (this.config.verbose) {
|
|
13639
|
+
console.log(`[cleanupStaleLocks] Removed expired lock: ${lock.id}`);
|
|
13640
|
+
}
|
|
13641
|
+
}
|
|
13642
|
+
}
|
|
13643
|
+
} catch (error) {
|
|
13644
|
+
if (this.config.verbose) {
|
|
13645
|
+
console.log(`[cleanupStaleLocks] Error during cleanup: ${error.message}`);
|
|
13646
|
+
}
|
|
13647
|
+
}
|
|
13648
|
+
}
|
|
13649
|
+
async attemptClaim(msg) {
|
|
13650
|
+
const now = Date.now();
|
|
13651
|
+
const lockAcquired = await this.acquireLock(msg.id);
|
|
13652
|
+
if (!lockAcquired) {
|
|
13653
|
+
return null;
|
|
13654
|
+
}
|
|
13655
|
+
if (this.processedCache.has(msg.id)) {
|
|
13656
|
+
await this.releaseLock(msg.id);
|
|
13657
|
+
if (this.config.verbose) {
|
|
13658
|
+
console.log(`[attemptClaim] Message ${msg.id} already processed (in cache)`);
|
|
13659
|
+
}
|
|
13660
|
+
return null;
|
|
13661
|
+
}
|
|
13662
|
+
this.processedCache.set(msg.id, Date.now());
|
|
13663
|
+
await this.releaseLock(msg.id);
|
|
13664
|
+
const [okGet, errGet, msgWithETag] = await tryFn(
|
|
13665
|
+
() => this.queueResource.get(msg.id)
|
|
13666
|
+
);
|
|
13667
|
+
if (!okGet || !msgWithETag) {
|
|
13668
|
+
this.processedCache.delete(msg.id);
|
|
13669
|
+
if (this.config.verbose) {
|
|
13670
|
+
console.log(`[attemptClaim] Message ${msg.id} not found or error: ${errGet?.message}`);
|
|
13671
|
+
}
|
|
13672
|
+
return null;
|
|
13673
|
+
}
|
|
13674
|
+
if (msgWithETag.status !== "pending" || msgWithETag.visibleAt > now) {
|
|
13675
|
+
this.processedCache.delete(msg.id);
|
|
13676
|
+
if (this.config.verbose) {
|
|
13677
|
+
console.log(`[attemptClaim] Message ${msg.id} not claimable: status=${msgWithETag.status}, visibleAt=${msgWithETag.visibleAt}, now=${now}`);
|
|
13678
|
+
}
|
|
13679
|
+
return null;
|
|
13680
|
+
}
|
|
13681
|
+
if (this.config.verbose) {
|
|
13682
|
+
console.log(`[attemptClaim] Attempting to claim ${msg.id} with ETag: ${msgWithETag._etag}`);
|
|
13683
|
+
}
|
|
13684
|
+
const [ok, err, result] = await tryFn(
|
|
13685
|
+
() => this.queueResource.updateConditional(msgWithETag.id, {
|
|
13686
|
+
status: "processing",
|
|
13687
|
+
claimedBy: this.workerId,
|
|
13688
|
+
claimedAt: now,
|
|
13689
|
+
visibleAt: now + this.config.visibilityTimeout,
|
|
13690
|
+
attempts: msgWithETag.attempts + 1
|
|
13691
|
+
}, {
|
|
13692
|
+
ifMatch: msgWithETag._etag
|
|
13693
|
+
// ← ATOMIC CLAIM using ETag!
|
|
13694
|
+
})
|
|
13695
|
+
);
|
|
13696
|
+
if (!ok || !result.success) {
|
|
13697
|
+
this.processedCache.delete(msg.id);
|
|
13698
|
+
if (this.config.verbose) {
|
|
13699
|
+
console.log(`[attemptClaim] Failed to claim ${msg.id}: ${err?.message || result.error}`);
|
|
13700
|
+
}
|
|
13701
|
+
return null;
|
|
13702
|
+
}
|
|
13703
|
+
if (this.config.verbose) {
|
|
13704
|
+
console.log(`[attemptClaim] Successfully claimed ${msg.id}`);
|
|
13705
|
+
}
|
|
13706
|
+
const [okRecord, errRecord, record] = await tryFn(
|
|
13707
|
+
() => this.targetResource.get(msgWithETag.originalId)
|
|
13708
|
+
);
|
|
13709
|
+
if (!okRecord) {
|
|
13710
|
+
await this.failMessage(msgWithETag.id, "Original record not found");
|
|
13711
|
+
return null;
|
|
13712
|
+
}
|
|
13713
|
+
return {
|
|
13714
|
+
queueId: msgWithETag.id,
|
|
13715
|
+
record,
|
|
13716
|
+
attempts: msgWithETag.attempts + 1,
|
|
13717
|
+
maxAttempts: msgWithETag.maxAttempts
|
|
13718
|
+
};
|
|
13719
|
+
}
|
|
13720
|
+
async processMessage(message, handler) {
|
|
13721
|
+
const startTime = Date.now();
|
|
13722
|
+
try {
|
|
13723
|
+
const result = await handler(message.record, {
|
|
13724
|
+
queueId: message.queueId,
|
|
13725
|
+
attempts: message.attempts,
|
|
13726
|
+
workerId: this.workerId
|
|
13727
|
+
});
|
|
13728
|
+
await this.completeMessage(message.queueId, result);
|
|
13729
|
+
const duration = Date.now() - startTime;
|
|
13730
|
+
this.emit("message.completed", {
|
|
13731
|
+
queueId: message.queueId,
|
|
13732
|
+
originalId: message.record.id,
|
|
13733
|
+
duration,
|
|
13734
|
+
attempts: message.attempts
|
|
13735
|
+
});
|
|
13736
|
+
if (this.config.onComplete) {
|
|
13737
|
+
await this.config.onComplete(message.record, result);
|
|
13738
|
+
}
|
|
13739
|
+
} catch (error) {
|
|
13740
|
+
const shouldRetry = message.attempts < message.maxAttempts;
|
|
13741
|
+
if (shouldRetry) {
|
|
13742
|
+
await this.retryMessage(message.queueId, message.attempts, error.message);
|
|
13743
|
+
this.emit("message.retry", {
|
|
13744
|
+
queueId: message.queueId,
|
|
13745
|
+
originalId: message.record.id,
|
|
13746
|
+
attempts: message.attempts,
|
|
13747
|
+
error: error.message
|
|
13748
|
+
});
|
|
13749
|
+
} else {
|
|
13750
|
+
await this.moveToDeadLetter(message.queueId, message.record, error.message);
|
|
13751
|
+
this.emit("message.dead", {
|
|
13752
|
+
queueId: message.queueId,
|
|
13753
|
+
originalId: message.record.id,
|
|
13754
|
+
error: error.message
|
|
13755
|
+
});
|
|
13756
|
+
}
|
|
13757
|
+
if (this.config.onError) {
|
|
13758
|
+
await this.config.onError(error, message.record);
|
|
13759
|
+
}
|
|
13760
|
+
}
|
|
13761
|
+
}
|
|
13762
|
+
async completeMessage(queueId, result) {
|
|
13763
|
+
await this.queueResource.update(queueId, {
|
|
13764
|
+
status: "completed",
|
|
13765
|
+
completedAt: Date.now(),
|
|
13766
|
+
result
|
|
13767
|
+
});
|
|
13768
|
+
}
|
|
13769
|
+
async failMessage(queueId, error) {
|
|
13770
|
+
await this.queueResource.update(queueId, {
|
|
13771
|
+
status: "failed",
|
|
13772
|
+
error
|
|
13773
|
+
});
|
|
13774
|
+
}
|
|
13775
|
+
async retryMessage(queueId, attempts, error) {
|
|
13776
|
+
const backoff = Math.min(Math.pow(2, attempts) * 1e3, 3e4);
|
|
13777
|
+
await this.queueResource.update(queueId, {
|
|
13778
|
+
status: "pending",
|
|
13779
|
+
visibleAt: Date.now() + backoff,
|
|
13780
|
+
error
|
|
13781
|
+
});
|
|
13782
|
+
this.processedCache.delete(queueId);
|
|
13783
|
+
}
|
|
13784
|
+
async moveToDeadLetter(queueId, record, error) {
|
|
13785
|
+
if (this.config.deadLetterResource && this.deadLetterResourceObj) {
|
|
13786
|
+
const msg = await this.queueResource.get(queueId);
|
|
13787
|
+
await this.deadLetterResourceObj.insert({
|
|
13788
|
+
id: idGenerator(),
|
|
13789
|
+
originalId: record.id,
|
|
13790
|
+
queueId,
|
|
13791
|
+
data: record,
|
|
13792
|
+
error,
|
|
13793
|
+
attempts: msg.attempts,
|
|
13794
|
+
createdAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
13795
|
+
});
|
|
13796
|
+
}
|
|
13797
|
+
await this.queueResource.update(queueId, {
|
|
13798
|
+
status: "dead",
|
|
13799
|
+
error
|
|
13800
|
+
});
|
|
13801
|
+
}
|
|
13802
|
+
async getStats() {
|
|
13803
|
+
const [ok, err, allMessages] = await tryFn(
|
|
13804
|
+
() => this.queueResource.list()
|
|
13805
|
+
);
|
|
13806
|
+
if (!ok) {
|
|
13807
|
+
if (this.config.verbose) {
|
|
13808
|
+
console.warn("[S3QueuePlugin] Failed to get stats:", err.message);
|
|
13809
|
+
}
|
|
13810
|
+
return null;
|
|
13811
|
+
}
|
|
13812
|
+
const stats = {
|
|
13813
|
+
total: allMessages.length,
|
|
13814
|
+
pending: 0,
|
|
13815
|
+
processing: 0,
|
|
13816
|
+
completed: 0,
|
|
13817
|
+
failed: 0,
|
|
13818
|
+
dead: 0
|
|
13819
|
+
};
|
|
13820
|
+
for (const msg of allMessages) {
|
|
13821
|
+
if (stats[msg.status] !== void 0) {
|
|
13822
|
+
stats[msg.status]++;
|
|
13823
|
+
}
|
|
13824
|
+
}
|
|
13825
|
+
return stats;
|
|
13826
|
+
}
|
|
13827
|
+
async createDeadLetterResource() {
|
|
13828
|
+
const [ok, err] = await tryFn(
|
|
13829
|
+
() => this.database.createResource({
|
|
13830
|
+
name: this.config.deadLetterResource,
|
|
13831
|
+
attributes: {
|
|
13832
|
+
id: "string|required",
|
|
13833
|
+
originalId: "string|required",
|
|
13834
|
+
queueId: "string|required",
|
|
13835
|
+
data: "json|required",
|
|
13836
|
+
error: "string|required",
|
|
13837
|
+
attempts: "number|required",
|
|
13838
|
+
createdAt: "string|required"
|
|
13839
|
+
},
|
|
13840
|
+
behavior: "body-overflow",
|
|
13841
|
+
timestamps: true
|
|
13842
|
+
})
|
|
13843
|
+
);
|
|
13844
|
+
if (ok || this.database.resources[this.config.deadLetterResource]) {
|
|
13845
|
+
this.deadLetterResourceObj = this.database.resources[this.config.deadLetterResource];
|
|
13846
|
+
if (this.config.verbose) {
|
|
13847
|
+
console.log(`[S3QueuePlugin] Dead letter queue created: ${this.config.deadLetterResource}`);
|
|
13848
|
+
}
|
|
13849
|
+
}
|
|
13850
|
+
}
|
|
13851
|
+
}
|
|
13852
|
+
|
|
13853
|
+
class SchedulerPlugin extends Plugin {
|
|
13854
|
+
constructor(options = {}) {
|
|
13855
|
+
super();
|
|
13856
|
+
this.config = {
|
|
13857
|
+
timezone: options.timezone || "UTC",
|
|
13858
|
+
jobs: options.jobs || {},
|
|
13859
|
+
defaultTimeout: options.defaultTimeout || 3e5,
|
|
13860
|
+
// 5 minutes
|
|
13861
|
+
defaultRetries: options.defaultRetries || 1,
|
|
13862
|
+
jobHistoryResource: options.jobHistoryResource || "plg_job_executions",
|
|
13863
|
+
persistJobs: options.persistJobs !== false,
|
|
13864
|
+
verbose: options.verbose || false,
|
|
13865
|
+
onJobStart: options.onJobStart || null,
|
|
13866
|
+
onJobComplete: options.onJobComplete || null,
|
|
13867
|
+
onJobError: options.onJobError || null,
|
|
13868
|
+
...options
|
|
13869
|
+
};
|
|
13870
|
+
this.database = null;
|
|
13871
|
+
this.lockResource = null;
|
|
13872
|
+
this.jobs = /* @__PURE__ */ new Map();
|
|
13873
|
+
this.activeJobs = /* @__PURE__ */ new Map();
|
|
13874
|
+
this.timers = /* @__PURE__ */ new Map();
|
|
13875
|
+
this.statistics = /* @__PURE__ */ new Map();
|
|
13876
|
+
this._validateConfiguration();
|
|
13877
|
+
}
|
|
13878
|
+
/**
|
|
13879
|
+
* Helper to detect test environment
|
|
13880
|
+
* @private
|
|
13881
|
+
*/
|
|
13882
|
+
_isTestEnvironment() {
|
|
13883
|
+
return process.env.NODE_ENV === "test" || process.env.JEST_WORKER_ID !== void 0 || global.expect !== void 0;
|
|
13884
|
+
}
|
|
13885
|
+
_validateConfiguration() {
|
|
13886
|
+
if (Object.keys(this.config.jobs).length === 0) {
|
|
13887
|
+
throw new Error("SchedulerPlugin: At least one job must be defined");
|
|
13888
|
+
}
|
|
13889
|
+
for (const [jobName, job] of Object.entries(this.config.jobs)) {
|
|
13890
|
+
if (!job.schedule) {
|
|
13891
|
+
throw new Error(`SchedulerPlugin: Job '${jobName}' must have a schedule`);
|
|
13892
|
+
}
|
|
13893
|
+
if (!job.action || typeof job.action !== "function") {
|
|
13894
|
+
throw new Error(`SchedulerPlugin: Job '${jobName}' must have an action function`);
|
|
13895
|
+
}
|
|
13896
|
+
if (!this._isValidCronExpression(job.schedule)) {
|
|
13897
|
+
throw new Error(`SchedulerPlugin: Job '${jobName}' has invalid cron expression: ${job.schedule}`);
|
|
13898
|
+
}
|
|
13899
|
+
}
|
|
13900
|
+
}
|
|
12371
13901
|
_isValidCronExpression(expr) {
|
|
12372
13902
|
if (typeof expr !== "string") return false;
|
|
12373
13903
|
const shortcuts = ["@yearly", "@annually", "@monthly", "@weekly", "@daily", "@hourly"];
|
|
@@ -12378,6 +13908,7 @@ class SchedulerPlugin extends Plugin {
|
|
|
12378
13908
|
}
|
|
12379
13909
|
async setup(database) {
|
|
12380
13910
|
this.database = database;
|
|
13911
|
+
await this._createLockResource();
|
|
12381
13912
|
if (this.config.persistJobs) {
|
|
12382
13913
|
await this._createJobHistoryResource();
|
|
12383
13914
|
}
|
|
@@ -12406,6 +13937,25 @@ class SchedulerPlugin extends Plugin {
|
|
|
12406
13937
|
await this._startScheduling();
|
|
12407
13938
|
this.emit("initialized", { jobs: this.jobs.size });
|
|
12408
13939
|
}
|
|
13940
|
+
async _createLockResource() {
|
|
13941
|
+
const [ok, err, lockResource] = await tryFn(
|
|
13942
|
+
() => this.database.createResource({
|
|
13943
|
+
name: "plg_scheduler_job_locks",
|
|
13944
|
+
attributes: {
|
|
13945
|
+
id: "string|required",
|
|
13946
|
+
jobName: "string|required",
|
|
13947
|
+
lockedAt: "number|required",
|
|
13948
|
+
instanceId: "string|optional"
|
|
13949
|
+
},
|
|
13950
|
+
behavior: "body-only",
|
|
13951
|
+
timestamps: false
|
|
13952
|
+
})
|
|
13953
|
+
);
|
|
13954
|
+
if (!ok && !this.database.resources.plg_scheduler_job_locks) {
|
|
13955
|
+
throw new Error(`Failed to create lock resource: ${err?.message}`);
|
|
13956
|
+
}
|
|
13957
|
+
this.lockResource = ok ? lockResource : this.database.resources.plg_scheduler_job_locks;
|
|
13958
|
+
}
|
|
12409
13959
|
async _createJobHistoryResource() {
|
|
12410
13960
|
const [ok] = await tryFn(() => this.database.createResource({
|
|
12411
13961
|
name: this.config.jobHistoryResource,
|
|
@@ -12499,18 +14049,37 @@ class SchedulerPlugin extends Plugin {
|
|
|
12499
14049
|
next.setHours(next.getHours() + 1);
|
|
12500
14050
|
}
|
|
12501
14051
|
}
|
|
12502
|
-
|
|
12503
|
-
if (isTestEnvironment) {
|
|
14052
|
+
if (this._isTestEnvironment()) {
|
|
12504
14053
|
next.setTime(next.getTime() + 1e3);
|
|
12505
14054
|
}
|
|
12506
14055
|
return next;
|
|
12507
14056
|
}
|
|
12508
14057
|
async _executeJob(jobName) {
|
|
12509
14058
|
const job = this.jobs.get(jobName);
|
|
12510
|
-
if (!job
|
|
14059
|
+
if (!job) {
|
|
14060
|
+
return;
|
|
14061
|
+
}
|
|
14062
|
+
if (this.activeJobs.has(jobName)) {
|
|
14063
|
+
return;
|
|
14064
|
+
}
|
|
14065
|
+
this.activeJobs.set(jobName, "acquiring-lock");
|
|
14066
|
+
const lockId = `lock-${jobName}`;
|
|
14067
|
+
const [lockAcquired, lockErr] = await tryFn(
|
|
14068
|
+
() => this.lockResource.insert({
|
|
14069
|
+
id: lockId,
|
|
14070
|
+
jobName,
|
|
14071
|
+
lockedAt: Date.now(),
|
|
14072
|
+
instanceId: process.pid ? String(process.pid) : "unknown"
|
|
14073
|
+
})
|
|
14074
|
+
);
|
|
14075
|
+
if (!lockAcquired) {
|
|
14076
|
+
if (this.config.verbose) {
|
|
14077
|
+
console.log(`[SchedulerPlugin] Job '${jobName}' already running on another instance`);
|
|
14078
|
+
}
|
|
14079
|
+
this.activeJobs.delete(jobName);
|
|
12511
14080
|
return;
|
|
12512
14081
|
}
|
|
12513
|
-
const executionId = `${jobName}_${
|
|
14082
|
+
const executionId = `${jobName}_${idGenerator()}`;
|
|
12514
14083
|
const startTime = Date.now();
|
|
12515
14084
|
const context = {
|
|
12516
14085
|
jobName,
|
|
@@ -12519,91 +14088,95 @@ class SchedulerPlugin extends Plugin {
|
|
|
12519
14088
|
database: this.database
|
|
12520
14089
|
};
|
|
12521
14090
|
this.activeJobs.set(jobName, executionId);
|
|
12522
|
-
|
|
12523
|
-
|
|
12524
|
-
|
|
12525
|
-
|
|
12526
|
-
|
|
12527
|
-
|
|
12528
|
-
|
|
12529
|
-
|
|
12530
|
-
|
|
12531
|
-
|
|
12532
|
-
|
|
12533
|
-
const actualTimeout = isTestEnvironment ? Math.min(job.timeout, 1e3) : job.timeout;
|
|
12534
|
-
let timeoutId;
|
|
12535
|
-
const timeoutPromise = new Promise((_, reject) => {
|
|
12536
|
-
timeoutId = setTimeout(() => reject(new Error("Job execution timeout")), actualTimeout);
|
|
12537
|
-
});
|
|
12538
|
-
const jobPromise = job.action(this.database, context, this);
|
|
14091
|
+
try {
|
|
14092
|
+
if (this.config.onJobStart) {
|
|
14093
|
+
await this._executeHook(this.config.onJobStart, jobName, context);
|
|
14094
|
+
}
|
|
14095
|
+
this.emit("job_start", { jobName, executionId, startTime });
|
|
14096
|
+
let attempt = 0;
|
|
14097
|
+
let lastError = null;
|
|
14098
|
+
let result = null;
|
|
14099
|
+
let status = "success";
|
|
14100
|
+
const isTestEnvironment = this._isTestEnvironment();
|
|
14101
|
+
while (attempt <= job.retries) {
|
|
12539
14102
|
try {
|
|
12540
|
-
|
|
12541
|
-
|
|
12542
|
-
|
|
12543
|
-
|
|
12544
|
-
|
|
12545
|
-
|
|
12546
|
-
|
|
12547
|
-
|
|
12548
|
-
|
|
12549
|
-
|
|
12550
|
-
|
|
12551
|
-
|
|
12552
|
-
|
|
12553
|
-
|
|
14103
|
+
const actualTimeout = isTestEnvironment ? Math.min(job.timeout, 1e3) : job.timeout;
|
|
14104
|
+
let timeoutId;
|
|
14105
|
+
const timeoutPromise = new Promise((_, reject) => {
|
|
14106
|
+
timeoutId = setTimeout(() => reject(new Error("Job execution timeout")), actualTimeout);
|
|
14107
|
+
});
|
|
14108
|
+
const jobPromise = job.action(this.database, context, this);
|
|
14109
|
+
try {
|
|
14110
|
+
result = await Promise.race([jobPromise, timeoutPromise]);
|
|
14111
|
+
clearTimeout(timeoutId);
|
|
14112
|
+
} catch (raceError) {
|
|
14113
|
+
clearTimeout(timeoutId);
|
|
14114
|
+
throw raceError;
|
|
14115
|
+
}
|
|
14116
|
+
status = "success";
|
|
14117
|
+
break;
|
|
14118
|
+
} catch (error) {
|
|
14119
|
+
lastError = error;
|
|
14120
|
+
attempt++;
|
|
14121
|
+
if (attempt <= job.retries) {
|
|
14122
|
+
if (this.config.verbose) {
|
|
14123
|
+
console.warn(`[SchedulerPlugin] Job '${jobName}' failed (attempt ${attempt + 1}):`, error.message);
|
|
14124
|
+
}
|
|
14125
|
+
const baseDelay = Math.min(Math.pow(2, attempt) * 1e3, 5e3);
|
|
14126
|
+
const delay = isTestEnvironment ? 1 : baseDelay;
|
|
14127
|
+
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
12554
14128
|
}
|
|
12555
|
-
const baseDelay = Math.min(Math.pow(2, attempt) * 1e3, 5e3);
|
|
12556
|
-
const delay = isTestEnvironment ? 1 : baseDelay;
|
|
12557
|
-
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
12558
14129
|
}
|
|
12559
14130
|
}
|
|
12560
|
-
|
|
12561
|
-
|
|
12562
|
-
|
|
12563
|
-
|
|
12564
|
-
|
|
12565
|
-
|
|
12566
|
-
|
|
12567
|
-
|
|
12568
|
-
|
|
12569
|
-
|
|
12570
|
-
|
|
12571
|
-
|
|
12572
|
-
|
|
12573
|
-
|
|
12574
|
-
|
|
12575
|
-
|
|
12576
|
-
|
|
12577
|
-
|
|
12578
|
-
|
|
12579
|
-
|
|
12580
|
-
|
|
12581
|
-
|
|
12582
|
-
|
|
12583
|
-
|
|
12584
|
-
|
|
12585
|
-
|
|
12586
|
-
|
|
12587
|
-
|
|
12588
|
-
|
|
12589
|
-
|
|
12590
|
-
|
|
12591
|
-
|
|
12592
|
-
|
|
12593
|
-
|
|
12594
|
-
|
|
12595
|
-
|
|
12596
|
-
|
|
12597
|
-
|
|
12598
|
-
|
|
12599
|
-
|
|
12600
|
-
|
|
12601
|
-
|
|
12602
|
-
|
|
12603
|
-
|
|
12604
|
-
|
|
12605
|
-
|
|
12606
|
-
|
|
14131
|
+
const endTime = Date.now();
|
|
14132
|
+
const duration = Math.max(1, endTime - startTime);
|
|
14133
|
+
if (lastError && attempt > job.retries) {
|
|
14134
|
+
status = lastError.message.includes("timeout") ? "timeout" : "error";
|
|
14135
|
+
}
|
|
14136
|
+
job.lastRun = new Date(endTime);
|
|
14137
|
+
job.runCount++;
|
|
14138
|
+
if (status === "success") {
|
|
14139
|
+
job.successCount++;
|
|
14140
|
+
} else {
|
|
14141
|
+
job.errorCount++;
|
|
14142
|
+
}
|
|
14143
|
+
const stats = this.statistics.get(jobName);
|
|
14144
|
+
stats.totalRuns++;
|
|
14145
|
+
stats.lastRun = new Date(endTime);
|
|
14146
|
+
if (status === "success") {
|
|
14147
|
+
stats.totalSuccesses++;
|
|
14148
|
+
stats.lastSuccess = new Date(endTime);
|
|
14149
|
+
} else {
|
|
14150
|
+
stats.totalErrors++;
|
|
14151
|
+
stats.lastError = { time: new Date(endTime), message: lastError?.message };
|
|
14152
|
+
}
|
|
14153
|
+
stats.avgDuration = (stats.avgDuration * (stats.totalRuns - 1) + duration) / stats.totalRuns;
|
|
14154
|
+
if (this.config.persistJobs) {
|
|
14155
|
+
await this._persistJobExecution(jobName, executionId, startTime, endTime, duration, status, result, lastError, attempt);
|
|
14156
|
+
}
|
|
14157
|
+
if (status === "success" && this.config.onJobComplete) {
|
|
14158
|
+
await this._executeHook(this.config.onJobComplete, jobName, result, duration);
|
|
14159
|
+
} else if (status !== "success" && this.config.onJobError) {
|
|
14160
|
+
await this._executeHook(this.config.onJobError, jobName, lastError, attempt);
|
|
14161
|
+
}
|
|
14162
|
+
this.emit("job_complete", {
|
|
14163
|
+
jobName,
|
|
14164
|
+
executionId,
|
|
14165
|
+
status,
|
|
14166
|
+
duration,
|
|
14167
|
+
result,
|
|
14168
|
+
error: lastError?.message,
|
|
14169
|
+
retryCount: attempt
|
|
14170
|
+
});
|
|
14171
|
+
this.activeJobs.delete(jobName);
|
|
14172
|
+
if (job.enabled) {
|
|
14173
|
+
this._scheduleNextExecution(jobName);
|
|
14174
|
+
}
|
|
14175
|
+
if (lastError && status !== "success") {
|
|
14176
|
+
throw lastError;
|
|
14177
|
+
}
|
|
14178
|
+
} finally {
|
|
14179
|
+
await tryFn(() => this.lockResource.delete(lockId));
|
|
12607
14180
|
}
|
|
12608
14181
|
}
|
|
12609
14182
|
async _persistJobExecution(jobName, executionId, startTime, endTime, duration, status, result, error, retryCount) {
|
|
@@ -12635,6 +14208,7 @@ class SchedulerPlugin extends Plugin {
|
|
|
12635
14208
|
}
|
|
12636
14209
|
/**
|
|
12637
14210
|
* Manually trigger a job execution
|
|
14211
|
+
* Note: Race conditions are prevented by distributed locking in _executeJob()
|
|
12638
14212
|
*/
|
|
12639
14213
|
async runJob(jobName, context = {}) {
|
|
12640
14214
|
const job = this.jobs.get(jobName);
|
|
@@ -12720,12 +14294,15 @@ class SchedulerPlugin extends Plugin {
|
|
|
12720
14294
|
return [];
|
|
12721
14295
|
}
|
|
12722
14296
|
const { limit = 50, status = null } = options;
|
|
12723
|
-
const
|
|
12724
|
-
|
|
12725
|
-
|
|
12726
|
-
|
|
12727
|
-
|
|
12728
|
-
|
|
14297
|
+
const queryParams = {
|
|
14298
|
+
jobName
|
|
14299
|
+
// Uses byJob partition for efficient lookup
|
|
14300
|
+
};
|
|
14301
|
+
if (status) {
|
|
14302
|
+
queryParams.status = status;
|
|
14303
|
+
}
|
|
14304
|
+
const [ok, err, history] = await tryFn(
|
|
14305
|
+
() => this.database.resource(this.config.jobHistoryResource).query(queryParams)
|
|
12729
14306
|
);
|
|
12730
14307
|
if (!ok) {
|
|
12731
14308
|
if (this.config.verbose) {
|
|
@@ -12733,11 +14310,7 @@ class SchedulerPlugin extends Plugin {
|
|
|
12733
14310
|
}
|
|
12734
14311
|
return [];
|
|
12735
14312
|
}
|
|
12736
|
-
let filtered =
|
|
12737
|
-
if (status) {
|
|
12738
|
-
filtered = filtered.filter((h) => h.status === status);
|
|
12739
|
-
}
|
|
12740
|
-
filtered = filtered.sort((a, b) => b.startTime - a.startTime).slice(0, limit);
|
|
14313
|
+
let filtered = history.sort((a, b) => b.startTime - a.startTime).slice(0, limit);
|
|
12741
14314
|
return filtered.map((h) => {
|
|
12742
14315
|
let result = null;
|
|
12743
14316
|
if (h.result) {
|
|
@@ -12832,8 +14405,7 @@ class SchedulerPlugin extends Plugin {
|
|
|
12832
14405
|
clearTimeout(timer);
|
|
12833
14406
|
}
|
|
12834
14407
|
this.timers.clear();
|
|
12835
|
-
|
|
12836
|
-
if (!isTestEnvironment && this.activeJobs.size > 0) {
|
|
14408
|
+
if (!this._isTestEnvironment() && this.activeJobs.size > 0) {
|
|
12837
14409
|
if (this.config.verbose) {
|
|
12838
14410
|
console.log(`[SchedulerPlugin] Waiting for ${this.activeJobs.size} active jobs to complete...`);
|
|
12839
14411
|
}
|
|
@@ -12846,7 +14418,7 @@ class SchedulerPlugin extends Plugin {
|
|
|
12846
14418
|
console.warn(`[SchedulerPlugin] ${this.activeJobs.size} jobs still running after timeout`);
|
|
12847
14419
|
}
|
|
12848
14420
|
}
|
|
12849
|
-
if (
|
|
14421
|
+
if (this._isTestEnvironment()) {
|
|
12850
14422
|
this.activeJobs.clear();
|
|
12851
14423
|
}
|
|
12852
14424
|
}
|
|
@@ -12867,14 +14439,14 @@ class StateMachinePlugin extends Plugin {
|
|
|
12867
14439
|
actions: options.actions || {},
|
|
12868
14440
|
guards: options.guards || {},
|
|
12869
14441
|
persistTransitions: options.persistTransitions !== false,
|
|
12870
|
-
transitionLogResource: options.transitionLogResource || "
|
|
12871
|
-
stateResource: options.stateResource || "
|
|
12872
|
-
|
|
12873
|
-
|
|
14442
|
+
transitionLogResource: options.transitionLogResource || "plg_state_transitions",
|
|
14443
|
+
stateResource: options.stateResource || "plg_entity_states",
|
|
14444
|
+
retryAttempts: options.retryAttempts || 3,
|
|
14445
|
+
retryDelay: options.retryDelay || 100,
|
|
14446
|
+
verbose: options.verbose || false
|
|
12874
14447
|
};
|
|
12875
14448
|
this.database = null;
|
|
12876
14449
|
this.machines = /* @__PURE__ */ new Map();
|
|
12877
|
-
this.stateStorage = /* @__PURE__ */ new Map();
|
|
12878
14450
|
this._validateConfiguration();
|
|
12879
14451
|
}
|
|
12880
14452
|
_validateConfiguration() {
|
|
@@ -13015,43 +14587,55 @@ class StateMachinePlugin extends Plugin {
|
|
|
13015
14587
|
machine.currentStates.set(entityId, toState);
|
|
13016
14588
|
if (this.config.persistTransitions) {
|
|
13017
14589
|
const transitionId = `${machineId}_${entityId}_${timestamp}`;
|
|
13018
|
-
|
|
13019
|
-
|
|
13020
|
-
|
|
13021
|
-
|
|
13022
|
-
|
|
13023
|
-
|
|
13024
|
-
|
|
13025
|
-
|
|
13026
|
-
|
|
13027
|
-
|
|
13028
|
-
|
|
13029
|
-
|
|
13030
|
-
|
|
13031
|
-
|
|
14590
|
+
let logOk = false;
|
|
14591
|
+
let lastLogErr;
|
|
14592
|
+
for (let attempt = 0; attempt < this.config.retryAttempts; attempt++) {
|
|
14593
|
+
const [ok, err] = await tryFn(
|
|
14594
|
+
() => this.database.resource(this.config.transitionLogResource).insert({
|
|
14595
|
+
id: transitionId,
|
|
14596
|
+
machineId,
|
|
14597
|
+
entityId,
|
|
14598
|
+
fromState,
|
|
14599
|
+
toState,
|
|
14600
|
+
event,
|
|
14601
|
+
context,
|
|
14602
|
+
timestamp,
|
|
14603
|
+
createdAt: now.slice(0, 10)
|
|
14604
|
+
// YYYY-MM-DD for partitioning
|
|
14605
|
+
})
|
|
14606
|
+
);
|
|
14607
|
+
if (ok) {
|
|
14608
|
+
logOk = true;
|
|
14609
|
+
break;
|
|
14610
|
+
}
|
|
14611
|
+
lastLogErr = err;
|
|
14612
|
+
if (attempt < this.config.retryAttempts - 1) {
|
|
14613
|
+
const delay = this.config.retryDelay * Math.pow(2, attempt);
|
|
14614
|
+
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
14615
|
+
}
|
|
14616
|
+
}
|
|
13032
14617
|
if (!logOk && this.config.verbose) {
|
|
13033
|
-
console.warn(`[StateMachinePlugin] Failed to log transition:`,
|
|
14618
|
+
console.warn(`[StateMachinePlugin] Failed to log transition after ${this.config.retryAttempts} attempts:`, lastLogErr.message);
|
|
13034
14619
|
}
|
|
13035
14620
|
const stateId = `${machineId}_${entityId}`;
|
|
13036
|
-
const
|
|
13037
|
-
|
|
13038
|
-
|
|
13039
|
-
|
|
13040
|
-
|
|
13041
|
-
|
|
13042
|
-
|
|
13043
|
-
|
|
13044
|
-
|
|
13045
|
-
|
|
13046
|
-
|
|
13047
|
-
|
|
13048
|
-
|
|
13049
|
-
|
|
13050
|
-
|
|
14621
|
+
const stateData = {
|
|
14622
|
+
machineId,
|
|
14623
|
+
entityId,
|
|
14624
|
+
currentState: toState,
|
|
14625
|
+
context,
|
|
14626
|
+
lastTransition: transitionId,
|
|
14627
|
+
updatedAt: now
|
|
14628
|
+
};
|
|
14629
|
+
const [updateOk] = await tryFn(
|
|
14630
|
+
() => this.database.resource(this.config.stateResource).update(stateId, stateData)
|
|
14631
|
+
);
|
|
14632
|
+
if (!updateOk) {
|
|
14633
|
+
const [insertOk, insertErr] = await tryFn(
|
|
14634
|
+
() => this.database.resource(this.config.stateResource).insert({ id: stateId, ...stateData })
|
|
14635
|
+
);
|
|
14636
|
+
if (!insertOk && this.config.verbose) {
|
|
14637
|
+
console.warn(`[StateMachinePlugin] Failed to upsert state:`, insertErr.message);
|
|
13051
14638
|
}
|
|
13052
|
-
});
|
|
13053
|
-
if (!stateOk && this.config.verbose) {
|
|
13054
|
-
console.warn(`[StateMachinePlugin] Failed to update state:`, stateErr.message);
|
|
13055
14639
|
}
|
|
13056
14640
|
}
|
|
13057
14641
|
}
|
|
@@ -13082,8 +14666,9 @@ class StateMachinePlugin extends Plugin {
|
|
|
13082
14666
|
}
|
|
13083
14667
|
/**
|
|
13084
14668
|
* Get valid events for current state
|
|
14669
|
+
* Can accept either a state name (sync) or entityId (async to fetch latest state)
|
|
13085
14670
|
*/
|
|
13086
|
-
getValidEvents(machineId, stateOrEntityId) {
|
|
14671
|
+
async getValidEvents(machineId, stateOrEntityId) {
|
|
13087
14672
|
const machine = this.machines.get(machineId);
|
|
13088
14673
|
if (!machine) {
|
|
13089
14674
|
throw new Error(`State machine '${machineId}' not found`);
|
|
@@ -13092,7 +14677,7 @@ class StateMachinePlugin extends Plugin {
|
|
|
13092
14677
|
if (machine.config.states[stateOrEntityId]) {
|
|
13093
14678
|
state = stateOrEntityId;
|
|
13094
14679
|
} else {
|
|
13095
|
-
state =
|
|
14680
|
+
state = await this.getState(machineId, stateOrEntityId);
|
|
13096
14681
|
}
|
|
13097
14682
|
const stateConfig = machine.config.states[state];
|
|
13098
14683
|
return stateConfig && stateConfig.on ? Object.keys(stateConfig.on) : [];
|
|
@@ -13106,9 +14691,10 @@ class StateMachinePlugin extends Plugin {
|
|
|
13106
14691
|
}
|
|
13107
14692
|
const { limit = 50, offset = 0 } = options;
|
|
13108
14693
|
const [ok, err, transitions] = await tryFn(
|
|
13109
|
-
() => this.database.resource(this.config.transitionLogResource).
|
|
13110
|
-
|
|
13111
|
-
|
|
14694
|
+
() => this.database.resource(this.config.transitionLogResource).query({
|
|
14695
|
+
machineId,
|
|
14696
|
+
entityId
|
|
14697
|
+
}, {
|
|
13112
14698
|
limit,
|
|
13113
14699
|
offset
|
|
13114
14700
|
})
|
|
@@ -13119,8 +14705,8 @@ class StateMachinePlugin extends Plugin {
|
|
|
13119
14705
|
}
|
|
13120
14706
|
return [];
|
|
13121
14707
|
}
|
|
13122
|
-
const
|
|
13123
|
-
return
|
|
14708
|
+
const sorted = (transitions || []).sort((a, b) => b.timestamp - a.timestamp);
|
|
14709
|
+
return sorted.map((t) => ({
|
|
13124
14710
|
from: t.fromState,
|
|
13125
14711
|
to: t.toState,
|
|
13126
14712
|
event: t.event,
|
|
@@ -13141,15 +14727,20 @@ class StateMachinePlugin extends Plugin {
|
|
|
13141
14727
|
if (this.config.persistTransitions) {
|
|
13142
14728
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
13143
14729
|
const stateId = `${machineId}_${entityId}`;
|
|
13144
|
-
await
|
|
13145
|
-
|
|
13146
|
-
|
|
13147
|
-
|
|
13148
|
-
|
|
13149
|
-
|
|
13150
|
-
|
|
13151
|
-
|
|
13152
|
-
|
|
14730
|
+
const [ok, err] = await tryFn(
|
|
14731
|
+
() => this.database.resource(this.config.stateResource).insert({
|
|
14732
|
+
id: stateId,
|
|
14733
|
+
machineId,
|
|
14734
|
+
entityId,
|
|
14735
|
+
currentState: initialState,
|
|
14736
|
+
context,
|
|
14737
|
+
lastTransition: null,
|
|
14738
|
+
updatedAt: now
|
|
14739
|
+
})
|
|
14740
|
+
);
|
|
14741
|
+
if (!ok && err && !err.message?.includes("already exists")) {
|
|
14742
|
+
throw new Error(`Failed to initialize entity state: ${err.message}`);
|
|
14743
|
+
}
|
|
13153
14744
|
}
|
|
13154
14745
|
const initialStateConfig = machine.config.states[initialState];
|
|
13155
14746
|
if (initialStateConfig && initialStateConfig.entry) {
|
|
@@ -13214,7 +14805,6 @@ class StateMachinePlugin extends Plugin {
|
|
|
13214
14805
|
}
|
|
13215
14806
|
async stop() {
|
|
13216
14807
|
this.machines.clear();
|
|
13217
|
-
this.stateStorage.clear();
|
|
13218
14808
|
}
|
|
13219
14809
|
async cleanup() {
|
|
13220
14810
|
await this.stop();
|
|
@@ -13222,5 +14812,5 @@ class StateMachinePlugin extends Plugin {
|
|
|
13222
14812
|
}
|
|
13223
14813
|
}
|
|
13224
14814
|
|
|
13225
|
-
export { AVAILABLE_BEHAVIORS, AuditPlugin, AuthenticationError, BackupPlugin, BaseError, CachePlugin, Client, ConnectionString, ConnectionStringError, CostsPlugin, CryptoError, DEFAULT_BEHAVIOR, Database, DatabaseError, EncryptionError, ErrorMap, EventualConsistencyPlugin, FullTextPlugin, InvalidResourceItem, MetricsPlugin, MissingMetadata, NoSuchBucket, NoSuchKey, NotFound, PartitionError, PermissionError, Plugin, PluginObject, ReplicatorPlugin, Resource, ResourceError, ResourceIdsPageReader, ResourceIdsReader, ResourceNotFound, ResourceReader, ResourceWriter, Database as S3db, S3dbError, SchedulerPlugin, Schema, SchemaError, StateMachinePlugin, UnknownError, ValidationError, Validator, behaviors, calculateAttributeNamesSize, calculateAttributeSizes, calculateEffectiveLimit, calculateSystemOverhead, calculateTotalSize, calculateUTF8Bytes, clearUTF8Cache, clearUTF8Memo, clearUTF8Memory, decode, decodeDecimal, decrypt, S3db as default, encode, encodeDecimal, encrypt, getBehavior, getSizeBreakdown, idGenerator, mapAwsError, md5, passwordGenerator, sha256, streamToString, transformValue, tryFn, tryFnSync };
|
|
14815
|
+
export { AVAILABLE_BEHAVIORS, AuditPlugin, AuthenticationError, BackupPlugin, BaseError, CachePlugin, Client, ConnectionString, ConnectionStringError, CostsPlugin, CryptoError, DEFAULT_BEHAVIOR, Database, DatabaseError, EncryptionError, ErrorMap, EventualConsistencyPlugin, FullTextPlugin, InvalidResourceItem, MetricsPlugin, MissingMetadata, NoSuchBucket, NoSuchKey, NotFound, PartitionError, PermissionError, Plugin, PluginObject, QueueConsumerPlugin, ReplicatorPlugin, Resource, ResourceError, ResourceIdsPageReader, ResourceIdsReader, ResourceNotFound, ResourceReader, ResourceWriter, S3QueuePlugin, Database as S3db, S3dbError, SchedulerPlugin, Schema, SchemaError, StateMachinePlugin, UnknownError, ValidationError, Validator, behaviors, calculateAttributeNamesSize, calculateAttributeSizes, calculateEffectiveLimit, calculateSystemOverhead, calculateTotalSize, calculateUTF8Bytes, clearUTF8Cache, clearUTF8Memo, clearUTF8Memory, decode, decodeDecimal, decrypt, S3db as default, encode, encodeDecimal, encrypt, getBehavior, getSizeBreakdown, idGenerator, mapAwsError, md5, passwordGenerator, sha256, streamToString, transformValue, tryFn, tryFnSync };
|
|
13226
14816
|
//# sourceMappingURL=s3db.es.js.map
|