s3db.js 9.3.0 → 10.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +72 -13
- package/dist/s3db.cjs.js +2342 -540
- package/dist/s3db.cjs.js.map +1 -1
- package/dist/s3db.es.js +2341 -541
- package/dist/s3db.es.js.map +1 -1
- package/package.json +1 -1
- package/src/client.class.js +8 -7
- package/src/concerns/high-performance-inserter.js +285 -0
- package/src/concerns/partition-queue.js +171 -0
- package/src/errors.js +10 -2
- package/src/partition-drivers/base-partition-driver.js +96 -0
- package/src/partition-drivers/index.js +60 -0
- package/src/partition-drivers/memory-partition-driver.js +274 -0
- package/src/partition-drivers/sqs-partition-driver.js +332 -0
- package/src/partition-drivers/sync-partition-driver.js +38 -0
- package/src/plugins/audit.plugin.js +4 -4
- package/src/plugins/backup.plugin.js +380 -105
- package/src/plugins/backup.plugin.js.backup +1 -1
- package/src/plugins/cache.plugin.js +203 -150
- package/src/plugins/eventual-consistency.plugin.js +1012 -0
- package/src/plugins/fulltext.plugin.js +6 -6
- package/src/plugins/index.js +2 -0
- package/src/plugins/metrics.plugin.js +13 -13
- package/src/plugins/replicator.plugin.js +108 -70
- package/src/plugins/replicators/s3db-replicator.class.js +7 -3
- package/src/plugins/replicators/sqs-replicator.class.js +11 -3
- package/src/plugins/s3-queue.plugin.js +776 -0
- package/src/plugins/scheduler.plugin.js +226 -164
- package/src/plugins/state-machine.plugin.js +109 -81
- package/src/resource.class.js +205 -0
- package/PLUGINS.md +0 -5036
package/dist/s3db.es.js
CHANGED
|
@@ -6,11 +6,12 @@ import { pipeline } from 'stream/promises';
|
|
|
6
6
|
import path, { join } from 'path';
|
|
7
7
|
import crypto, { createHash } from 'crypto';
|
|
8
8
|
import zlib from 'node:zlib';
|
|
9
|
+
import os from 'os';
|
|
10
|
+
import jsonStableStringify from 'json-stable-stringify';
|
|
9
11
|
import { Transform, Writable } from 'stream';
|
|
10
12
|
import { PromisePool } from '@supercharge/promise-pool';
|
|
11
13
|
import { ReadableStream } from 'node:stream/web';
|
|
12
14
|
import { chunk, merge, isString, isEmpty, invert, uniq, cloneDeep, get, set, isObject, isFunction } from 'lodash-es';
|
|
13
|
-
import jsonStableStringify from 'json-stable-stringify';
|
|
14
15
|
import { Agent } from 'http';
|
|
15
16
|
import { Agent as Agent$1 } from 'https';
|
|
16
17
|
import { NodeHttpHandler } from '@smithy/node-http-handler';
|
|
@@ -419,8 +420,14 @@ function mapAwsError(err, context = {}) {
|
|
|
419
420
|
suggestion = "Check if the object metadata is present and valid.";
|
|
420
421
|
return new MissingMetadata({ ...context, original: err, metadata, commandName, commandInput, suggestion });
|
|
421
422
|
}
|
|
422
|
-
|
|
423
|
-
|
|
423
|
+
const errorDetails = [
|
|
424
|
+
`Unknown error: ${err.message || err.toString()}`,
|
|
425
|
+
err.code && `Code: ${err.code}`,
|
|
426
|
+
err.statusCode && `Status: ${err.statusCode}`,
|
|
427
|
+
err.stack && `Stack: ${err.stack.split("\n")[0]}`
|
|
428
|
+
].filter(Boolean).join(" | ");
|
|
429
|
+
suggestion = `Check the error details and AWS documentation. Original error: ${err.message || err.toString()}`;
|
|
430
|
+
return new UnknownError(errorDetails, { ...context, original: err, metadata, commandName, commandInput, suggestion });
|
|
424
431
|
}
|
|
425
432
|
class ConnectionStringError extends S3dbError {
|
|
426
433
|
constructor(message, details = {}) {
|
|
@@ -832,7 +839,7 @@ class AuditPlugin extends Plugin {
|
|
|
832
839
|
}
|
|
833
840
|
async onSetup() {
|
|
834
841
|
const [ok, err, auditResource] = await tryFn(() => this.database.createResource({
|
|
835
|
-
name: "
|
|
842
|
+
name: "plg_audits",
|
|
836
843
|
attributes: {
|
|
837
844
|
id: "string|required",
|
|
838
845
|
resourceName: "string|required",
|
|
@@ -848,15 +855,15 @@ class AuditPlugin extends Plugin {
|
|
|
848
855
|
},
|
|
849
856
|
behavior: "body-overflow"
|
|
850
857
|
}));
|
|
851
|
-
this.auditResource = ok ? auditResource : this.database.resources.
|
|
858
|
+
this.auditResource = ok ? auditResource : this.database.resources.plg_audits || null;
|
|
852
859
|
if (!ok && !this.auditResource) return;
|
|
853
860
|
this.database.addHook("afterCreateResource", (context) => {
|
|
854
|
-
if (context.resource.name !== "
|
|
861
|
+
if (context.resource.name !== "plg_audits") {
|
|
855
862
|
this.setupResourceAuditing(context.resource);
|
|
856
863
|
}
|
|
857
864
|
});
|
|
858
865
|
for (const resource of Object.values(this.database.resources)) {
|
|
859
|
-
if (resource.name !== "
|
|
866
|
+
if (resource.name !== "plg_audits") {
|
|
860
867
|
this.setupResourceAuditing(resource);
|
|
861
868
|
}
|
|
862
869
|
}
|
|
@@ -1876,11 +1883,10 @@ function validateBackupConfig(driver, config = {}) {
|
|
|
1876
1883
|
class BackupPlugin extends Plugin {
|
|
1877
1884
|
constructor(options = {}) {
|
|
1878
1885
|
super();
|
|
1879
|
-
this.driverName = options.driver || "filesystem";
|
|
1880
|
-
this.driverConfig = options.config || {};
|
|
1881
1886
|
this.config = {
|
|
1882
|
-
//
|
|
1883
|
-
|
|
1887
|
+
// Driver configuration
|
|
1888
|
+
driver: options.driver || "filesystem",
|
|
1889
|
+
driverConfig: options.config || {},
|
|
1884
1890
|
// Scheduling configuration
|
|
1885
1891
|
schedule: options.schedule || {},
|
|
1886
1892
|
// Retention policy (Grandfather-Father-Son)
|
|
@@ -1898,8 +1904,8 @@ class BackupPlugin extends Plugin {
|
|
|
1898
1904
|
parallelism: options.parallelism || 4,
|
|
1899
1905
|
include: options.include || null,
|
|
1900
1906
|
exclude: options.exclude || [],
|
|
1901
|
-
backupMetadataResource: options.backupMetadataResource || "
|
|
1902
|
-
tempDir: options.tempDir || "
|
|
1907
|
+
backupMetadataResource: options.backupMetadataResource || "plg_backup_metadata",
|
|
1908
|
+
tempDir: options.tempDir || path.join(os.tmpdir(), "s3db", "backups"),
|
|
1903
1909
|
verbose: options.verbose || false,
|
|
1904
1910
|
// Hooks
|
|
1905
1911
|
onBackupStart: options.onBackupStart || null,
|
|
@@ -1911,32 +1917,9 @@ class BackupPlugin extends Plugin {
|
|
|
1911
1917
|
};
|
|
1912
1918
|
this.driver = null;
|
|
1913
1919
|
this.activeBackups = /* @__PURE__ */ new Set();
|
|
1914
|
-
this.
|
|
1915
|
-
validateBackupConfig(this.driverName, this.driverConfig);
|
|
1920
|
+
validateBackupConfig(this.config.driver, this.config.driverConfig);
|
|
1916
1921
|
this._validateConfiguration();
|
|
1917
1922
|
}
|
|
1918
|
-
/**
|
|
1919
|
-
* Convert legacy destinations format to multi driver format
|
|
1920
|
-
*/
|
|
1921
|
-
_handleLegacyDestinations() {
|
|
1922
|
-
if (this.config.destinations && Array.isArray(this.config.destinations)) {
|
|
1923
|
-
this.driverName = "multi";
|
|
1924
|
-
this.driverConfig = {
|
|
1925
|
-
strategy: "all",
|
|
1926
|
-
destinations: this.config.destinations.map((dest) => {
|
|
1927
|
-
const { type, ...config } = dest;
|
|
1928
|
-
return {
|
|
1929
|
-
driver: type,
|
|
1930
|
-
config
|
|
1931
|
-
};
|
|
1932
|
-
})
|
|
1933
|
-
};
|
|
1934
|
-
this.config.destinations = null;
|
|
1935
|
-
if (this.config.verbose) {
|
|
1936
|
-
console.log("[BackupPlugin] Converted legacy destinations format to multi driver");
|
|
1937
|
-
}
|
|
1938
|
-
}
|
|
1939
|
-
}
|
|
1940
1923
|
_validateConfiguration() {
|
|
1941
1924
|
if (this.config.encryption && (!this.config.encryption.key || !this.config.encryption.algorithm)) {
|
|
1942
1925
|
throw new Error("BackupPlugin: Encryption requires both key and algorithm");
|
|
@@ -1946,7 +1929,7 @@ class BackupPlugin extends Plugin {
|
|
|
1946
1929
|
}
|
|
1947
1930
|
}
|
|
1948
1931
|
async onSetup() {
|
|
1949
|
-
this.driver = createBackupDriver(this.
|
|
1932
|
+
this.driver = createBackupDriver(this.config.driver, this.config.driverConfig);
|
|
1950
1933
|
await this.driver.setup(this.database);
|
|
1951
1934
|
await mkdir(this.config.tempDir, { recursive: true });
|
|
1952
1935
|
await this._createBackupMetadataResource();
|
|
@@ -1994,6 +1977,9 @@ class BackupPlugin extends Plugin {
|
|
|
1994
1977
|
async backup(type = "full", options = {}) {
|
|
1995
1978
|
const backupId = this._generateBackupId(type);
|
|
1996
1979
|
const startTime = Date.now();
|
|
1980
|
+
if (this.activeBackups.has(backupId)) {
|
|
1981
|
+
throw new Error(`Backup '${backupId}' is already in progress`);
|
|
1982
|
+
}
|
|
1997
1983
|
try {
|
|
1998
1984
|
this.activeBackups.add(backupId);
|
|
1999
1985
|
if (this.config.onBackupStart) {
|
|
@@ -2009,16 +1995,9 @@ class BackupPlugin extends Plugin {
|
|
|
2009
1995
|
if (exportedFiles.length === 0) {
|
|
2010
1996
|
throw new Error("No resources were exported for backup");
|
|
2011
1997
|
}
|
|
2012
|
-
|
|
2013
|
-
|
|
2014
|
-
|
|
2015
|
-
finalPath = path.join(tempBackupDir, `${backupId}.tar.gz`);
|
|
2016
|
-
totalSize = await this._createCompressedArchive(exportedFiles, finalPath);
|
|
2017
|
-
} else {
|
|
2018
|
-
finalPath = exportedFiles[0];
|
|
2019
|
-
const [statOk, , stats] = await tryFn(() => stat(finalPath));
|
|
2020
|
-
totalSize = statOk ? stats.size : 0;
|
|
2021
|
-
}
|
|
1998
|
+
const archiveExtension = this.config.compression !== "none" ? ".tar.gz" : ".json";
|
|
1999
|
+
const finalPath = path.join(tempBackupDir, `${backupId}${archiveExtension}`);
|
|
2000
|
+
const totalSize = await this._createArchive(exportedFiles, finalPath, this.config.compression);
|
|
2022
2001
|
const checksum = await this._generateChecksum(finalPath);
|
|
2023
2002
|
const uploadResult = await this.driver.upload(finalPath, backupId, manifest);
|
|
2024
2003
|
if (this.config.verification) {
|
|
@@ -2127,15 +2106,35 @@ class BackupPlugin extends Plugin {
|
|
|
2127
2106
|
for (const resourceName of resourceNames) {
|
|
2128
2107
|
const resource = this.database.resources[resourceName];
|
|
2129
2108
|
if (!resource) {
|
|
2130
|
-
|
|
2109
|
+
if (this.config.verbose) {
|
|
2110
|
+
console.warn(`[BackupPlugin] Resource '${resourceName}' not found, skipping`);
|
|
2111
|
+
}
|
|
2131
2112
|
continue;
|
|
2132
2113
|
}
|
|
2133
2114
|
const exportPath = path.join(tempDir, `${resourceName}.json`);
|
|
2134
2115
|
let records;
|
|
2135
2116
|
if (type === "incremental") {
|
|
2136
|
-
const
|
|
2117
|
+
const [lastBackupOk, , lastBackups] = await tryFn(
|
|
2118
|
+
() => this.database.resource(this.config.backupMetadataResource).list({
|
|
2119
|
+
filter: {
|
|
2120
|
+
status: "completed",
|
|
2121
|
+
type: { $in: ["full", "incremental"] }
|
|
2122
|
+
},
|
|
2123
|
+
sort: { timestamp: -1 },
|
|
2124
|
+
limit: 1
|
|
2125
|
+
})
|
|
2126
|
+
);
|
|
2127
|
+
let sinceTimestamp;
|
|
2128
|
+
if (lastBackupOk && lastBackups && lastBackups.length > 0) {
|
|
2129
|
+
sinceTimestamp = new Date(lastBackups[0].timestamp);
|
|
2130
|
+
} else {
|
|
2131
|
+
sinceTimestamp = new Date(Date.now() - 24 * 60 * 60 * 1e3);
|
|
2132
|
+
}
|
|
2133
|
+
if (this.config.verbose) {
|
|
2134
|
+
console.log(`[BackupPlugin] Incremental backup for '${resourceName}' since ${sinceTimestamp.toISOString()}`);
|
|
2135
|
+
}
|
|
2137
2136
|
records = await resource.list({
|
|
2138
|
-
filter: { updatedAt: { ">":
|
|
2137
|
+
filter: { updatedAt: { ">": sinceTimestamp.toISOString() } }
|
|
2139
2138
|
});
|
|
2140
2139
|
} else {
|
|
2141
2140
|
records = await resource.list();
|
|
@@ -2155,29 +2154,57 @@ class BackupPlugin extends Plugin {
|
|
|
2155
2154
|
}
|
|
2156
2155
|
return exportedFiles;
|
|
2157
2156
|
}
|
|
2158
|
-
async
|
|
2159
|
-
const
|
|
2160
|
-
|
|
2157
|
+
async _createArchive(files, targetPath, compressionType) {
|
|
2158
|
+
const archive = {
|
|
2159
|
+
version: "1.0",
|
|
2160
|
+
created: (/* @__PURE__ */ new Date()).toISOString(),
|
|
2161
|
+
files: []
|
|
2162
|
+
};
|
|
2161
2163
|
let totalSize = 0;
|
|
2162
|
-
|
|
2163
|
-
|
|
2164
|
-
|
|
2165
|
-
|
|
2166
|
-
|
|
2167
|
-
yield content;
|
|
2164
|
+
for (const filePath of files) {
|
|
2165
|
+
const [readOk, readErr, content] = await tryFn(() => readFile(filePath, "utf8"));
|
|
2166
|
+
if (!readOk) {
|
|
2167
|
+
if (this.config.verbose) {
|
|
2168
|
+
console.warn(`[BackupPlugin] Failed to read ${filePath}: ${readErr?.message}`);
|
|
2168
2169
|
}
|
|
2169
|
-
|
|
2170
|
-
|
|
2171
|
-
|
|
2172
|
-
|
|
2170
|
+
continue;
|
|
2171
|
+
}
|
|
2172
|
+
const fileName = path.basename(filePath);
|
|
2173
|
+
totalSize += content.length;
|
|
2174
|
+
archive.files.push({
|
|
2175
|
+
name: fileName,
|
|
2176
|
+
size: content.length,
|
|
2177
|
+
content
|
|
2178
|
+
});
|
|
2179
|
+
}
|
|
2180
|
+
const archiveJson = JSON.stringify(archive);
|
|
2181
|
+
if (compressionType === "none") {
|
|
2182
|
+
await writeFile(targetPath, archiveJson, "utf8");
|
|
2183
|
+
} else {
|
|
2184
|
+
const output = createWriteStream(targetPath);
|
|
2185
|
+
const gzip = zlib.createGzip({ level: 6 });
|
|
2186
|
+
await pipeline(
|
|
2187
|
+
async function* () {
|
|
2188
|
+
yield Buffer.from(archiveJson, "utf8");
|
|
2189
|
+
},
|
|
2190
|
+
gzip,
|
|
2191
|
+
output
|
|
2192
|
+
);
|
|
2193
|
+
}
|
|
2173
2194
|
const [statOk, , stats] = await tryFn(() => stat(targetPath));
|
|
2174
2195
|
return statOk ? stats.size : totalSize;
|
|
2175
2196
|
}
|
|
2176
2197
|
async _generateChecksum(filePath) {
|
|
2177
|
-
const
|
|
2178
|
-
|
|
2179
|
-
|
|
2180
|
-
|
|
2198
|
+
const [ok, err, result] = await tryFn(async () => {
|
|
2199
|
+
const hash = crypto.createHash("sha256");
|
|
2200
|
+
const stream = createReadStream(filePath);
|
|
2201
|
+
await pipeline(stream, hash);
|
|
2202
|
+
return hash.digest("hex");
|
|
2203
|
+
});
|
|
2204
|
+
if (!ok) {
|
|
2205
|
+
throw new Error(`Failed to generate checksum for ${filePath}: ${err?.message}`);
|
|
2206
|
+
}
|
|
2207
|
+
return result;
|
|
2181
2208
|
}
|
|
2182
2209
|
async _cleanupTempFiles(tempDir) {
|
|
2183
2210
|
const [ok] = await tryFn(
|
|
@@ -2239,7 +2266,109 @@ class BackupPlugin extends Plugin {
|
|
|
2239
2266
|
}
|
|
2240
2267
|
async _restoreFromBackup(backupPath, options) {
|
|
2241
2268
|
const restoredResources = [];
|
|
2242
|
-
|
|
2269
|
+
try {
|
|
2270
|
+
let archiveData = "";
|
|
2271
|
+
if (this.config.compression !== "none") {
|
|
2272
|
+
const input = createReadStream(backupPath);
|
|
2273
|
+
const gunzip = zlib.createGunzip();
|
|
2274
|
+
const chunks = [];
|
|
2275
|
+
await new Promise((resolve, reject) => {
|
|
2276
|
+
input.pipe(gunzip).on("data", (chunk) => chunks.push(chunk)).on("end", resolve).on("error", reject);
|
|
2277
|
+
});
|
|
2278
|
+
archiveData = Buffer.concat(chunks).toString("utf8");
|
|
2279
|
+
} else {
|
|
2280
|
+
archiveData = await readFile(backupPath, "utf8");
|
|
2281
|
+
}
|
|
2282
|
+
let archive;
|
|
2283
|
+
try {
|
|
2284
|
+
archive = JSON.parse(archiveData);
|
|
2285
|
+
} catch (parseError) {
|
|
2286
|
+
throw new Error(`Failed to parse backup archive: ${parseError.message}`);
|
|
2287
|
+
}
|
|
2288
|
+
if (!archive || typeof archive !== "object") {
|
|
2289
|
+
throw new Error("Invalid backup archive: not a valid JSON object");
|
|
2290
|
+
}
|
|
2291
|
+
if (!archive.version || !archive.files) {
|
|
2292
|
+
throw new Error("Invalid backup archive format: missing version or files array");
|
|
2293
|
+
}
|
|
2294
|
+
if (this.config.verbose) {
|
|
2295
|
+
console.log(`[BackupPlugin] Restoring ${archive.files.length} files from backup`);
|
|
2296
|
+
}
|
|
2297
|
+
for (const file of archive.files) {
|
|
2298
|
+
try {
|
|
2299
|
+
const resourceData = JSON.parse(file.content);
|
|
2300
|
+
if (!resourceData.resourceName || !resourceData.definition) {
|
|
2301
|
+
if (this.config.verbose) {
|
|
2302
|
+
console.warn(`[BackupPlugin] Skipping invalid file: ${file.name}`);
|
|
2303
|
+
}
|
|
2304
|
+
continue;
|
|
2305
|
+
}
|
|
2306
|
+
const resourceName = resourceData.resourceName;
|
|
2307
|
+
if (options.resources && !options.resources.includes(resourceName)) {
|
|
2308
|
+
continue;
|
|
2309
|
+
}
|
|
2310
|
+
let resource = this.database.resources[resourceName];
|
|
2311
|
+
if (!resource) {
|
|
2312
|
+
if (this.config.verbose) {
|
|
2313
|
+
console.log(`[BackupPlugin] Creating resource '${resourceName}'`);
|
|
2314
|
+
}
|
|
2315
|
+
const [createOk, createErr] = await tryFn(
|
|
2316
|
+
() => this.database.createResource(resourceData.definition)
|
|
2317
|
+
);
|
|
2318
|
+
if (!createOk) {
|
|
2319
|
+
if (this.config.verbose) {
|
|
2320
|
+
console.warn(`[BackupPlugin] Failed to create resource '${resourceName}': ${createErr?.message}`);
|
|
2321
|
+
}
|
|
2322
|
+
continue;
|
|
2323
|
+
}
|
|
2324
|
+
resource = this.database.resources[resourceName];
|
|
2325
|
+
}
|
|
2326
|
+
if (resourceData.records && Array.isArray(resourceData.records)) {
|
|
2327
|
+
const mode = options.mode || "merge";
|
|
2328
|
+
if (mode === "replace") {
|
|
2329
|
+
const ids = await resource.listIds();
|
|
2330
|
+
for (const id of ids) {
|
|
2331
|
+
await resource.delete(id);
|
|
2332
|
+
}
|
|
2333
|
+
}
|
|
2334
|
+
let insertedCount = 0;
|
|
2335
|
+
for (const record of resourceData.records) {
|
|
2336
|
+
const [insertOk] = await tryFn(async () => {
|
|
2337
|
+
if (mode === "skip") {
|
|
2338
|
+
const existing = await resource.get(record.id);
|
|
2339
|
+
if (existing) {
|
|
2340
|
+
return false;
|
|
2341
|
+
}
|
|
2342
|
+
}
|
|
2343
|
+
await resource.insert(record);
|
|
2344
|
+
return true;
|
|
2345
|
+
});
|
|
2346
|
+
if (insertOk) {
|
|
2347
|
+
insertedCount++;
|
|
2348
|
+
}
|
|
2349
|
+
}
|
|
2350
|
+
restoredResources.push({
|
|
2351
|
+
name: resourceName,
|
|
2352
|
+
recordsRestored: insertedCount,
|
|
2353
|
+
totalRecords: resourceData.records.length
|
|
2354
|
+
});
|
|
2355
|
+
if (this.config.verbose) {
|
|
2356
|
+
console.log(`[BackupPlugin] Restored ${insertedCount}/${resourceData.records.length} records to '${resourceName}'`);
|
|
2357
|
+
}
|
|
2358
|
+
}
|
|
2359
|
+
} catch (fileError) {
|
|
2360
|
+
if (this.config.verbose) {
|
|
2361
|
+
console.warn(`[BackupPlugin] Error processing file ${file.name}: ${fileError.message}`);
|
|
2362
|
+
}
|
|
2363
|
+
}
|
|
2364
|
+
}
|
|
2365
|
+
return restoredResources;
|
|
2366
|
+
} catch (error) {
|
|
2367
|
+
if (this.config.verbose) {
|
|
2368
|
+
console.error(`[BackupPlugin] Error restoring backup: ${error.message}`);
|
|
2369
|
+
}
|
|
2370
|
+
throw new Error(`Failed to restore backup: ${error.message}`);
|
|
2371
|
+
}
|
|
2243
2372
|
}
|
|
2244
2373
|
/**
|
|
2245
2374
|
* List available backups
|
|
@@ -2283,6 +2412,90 @@ class BackupPlugin extends Plugin {
|
|
|
2283
2412
|
return ok ? backup : null;
|
|
2284
2413
|
}
|
|
2285
2414
|
async _cleanupOldBackups() {
|
|
2415
|
+
try {
|
|
2416
|
+
const [listOk, , allBackups] = await tryFn(
|
|
2417
|
+
() => this.database.resource(this.config.backupMetadataResource).list({
|
|
2418
|
+
filter: { status: "completed" },
|
|
2419
|
+
sort: { timestamp: -1 }
|
|
2420
|
+
})
|
|
2421
|
+
);
|
|
2422
|
+
if (!listOk || !allBackups || allBackups.length === 0) {
|
|
2423
|
+
return;
|
|
2424
|
+
}
|
|
2425
|
+
const now = Date.now();
|
|
2426
|
+
const msPerDay = 24 * 60 * 60 * 1e3;
|
|
2427
|
+
const msPerWeek = 7 * msPerDay;
|
|
2428
|
+
const msPerMonth = 30 * msPerDay;
|
|
2429
|
+
const msPerYear = 365 * msPerDay;
|
|
2430
|
+
const categorized = {
|
|
2431
|
+
daily: [],
|
|
2432
|
+
weekly: [],
|
|
2433
|
+
monthly: [],
|
|
2434
|
+
yearly: []
|
|
2435
|
+
};
|
|
2436
|
+
for (const backup of allBackups) {
|
|
2437
|
+
const age = now - backup.timestamp;
|
|
2438
|
+
if (age <= msPerDay * this.config.retention.daily) {
|
|
2439
|
+
categorized.daily.push(backup);
|
|
2440
|
+
} else if (age <= msPerWeek * this.config.retention.weekly) {
|
|
2441
|
+
categorized.weekly.push(backup);
|
|
2442
|
+
} else if (age <= msPerMonth * this.config.retention.monthly) {
|
|
2443
|
+
categorized.monthly.push(backup);
|
|
2444
|
+
} else if (age <= msPerYear * this.config.retention.yearly) {
|
|
2445
|
+
categorized.yearly.push(backup);
|
|
2446
|
+
}
|
|
2447
|
+
}
|
|
2448
|
+
const toKeep = /* @__PURE__ */ new Set();
|
|
2449
|
+
categorized.daily.forEach((b) => toKeep.add(b.id));
|
|
2450
|
+
const weeklyByWeek = /* @__PURE__ */ new Map();
|
|
2451
|
+
for (const backup of categorized.weekly) {
|
|
2452
|
+
const weekNum = Math.floor((now - backup.timestamp) / msPerWeek);
|
|
2453
|
+
if (!weeklyByWeek.has(weekNum)) {
|
|
2454
|
+
weeklyByWeek.set(weekNum, backup);
|
|
2455
|
+
toKeep.add(backup.id);
|
|
2456
|
+
}
|
|
2457
|
+
}
|
|
2458
|
+
const monthlyByMonth = /* @__PURE__ */ new Map();
|
|
2459
|
+
for (const backup of categorized.monthly) {
|
|
2460
|
+
const monthNum = Math.floor((now - backup.timestamp) / msPerMonth);
|
|
2461
|
+
if (!monthlyByMonth.has(monthNum)) {
|
|
2462
|
+
monthlyByMonth.set(monthNum, backup);
|
|
2463
|
+
toKeep.add(backup.id);
|
|
2464
|
+
}
|
|
2465
|
+
}
|
|
2466
|
+
const yearlyByYear = /* @__PURE__ */ new Map();
|
|
2467
|
+
for (const backup of categorized.yearly) {
|
|
2468
|
+
const yearNum = Math.floor((now - backup.timestamp) / msPerYear);
|
|
2469
|
+
if (!yearlyByYear.has(yearNum)) {
|
|
2470
|
+
yearlyByYear.set(yearNum, backup);
|
|
2471
|
+
toKeep.add(backup.id);
|
|
2472
|
+
}
|
|
2473
|
+
}
|
|
2474
|
+
const backupsToDelete = allBackups.filter((b) => !toKeep.has(b.id));
|
|
2475
|
+
if (backupsToDelete.length === 0) {
|
|
2476
|
+
return;
|
|
2477
|
+
}
|
|
2478
|
+
if (this.config.verbose) {
|
|
2479
|
+
console.log(`[BackupPlugin] Cleaning up ${backupsToDelete.length} old backups (keeping ${toKeep.size})`);
|
|
2480
|
+
}
|
|
2481
|
+
for (const backup of backupsToDelete) {
|
|
2482
|
+
try {
|
|
2483
|
+
await this.driver.delete(backup.id, backup.driverInfo);
|
|
2484
|
+
await this.database.resource(this.config.backupMetadataResource).delete(backup.id);
|
|
2485
|
+
if (this.config.verbose) {
|
|
2486
|
+
console.log(`[BackupPlugin] Deleted old backup: ${backup.id}`);
|
|
2487
|
+
}
|
|
2488
|
+
} catch (deleteError) {
|
|
2489
|
+
if (this.config.verbose) {
|
|
2490
|
+
console.warn(`[BackupPlugin] Failed to delete backup ${backup.id}: ${deleteError.message}`);
|
|
2491
|
+
}
|
|
2492
|
+
}
|
|
2493
|
+
}
|
|
2494
|
+
} catch (error) {
|
|
2495
|
+
if (this.config.verbose) {
|
|
2496
|
+
console.warn(`[BackupPlugin] Error during cleanup: ${error.message}`);
|
|
2497
|
+
}
|
|
2498
|
+
}
|
|
2286
2499
|
}
|
|
2287
2500
|
async _executeHook(hook, ...args) {
|
|
2288
2501
|
if (typeof hook === "function") {
|
|
@@ -3572,81 +3785,58 @@ class PartitionAwareFilesystemCache extends FilesystemCache {
|
|
|
3572
3785
|
class CachePlugin extends Plugin {
|
|
3573
3786
|
constructor(options = {}) {
|
|
3574
3787
|
super(options);
|
|
3575
|
-
this.
|
|
3576
|
-
|
|
3577
|
-
|
|
3578
|
-
|
|
3579
|
-
|
|
3580
|
-
|
|
3581
|
-
|
|
3582
|
-
|
|
3583
|
-
|
|
3584
|
-
|
|
3585
|
-
|
|
3586
|
-
|
|
3587
|
-
|
|
3588
|
-
|
|
3788
|
+
this.config = {
|
|
3789
|
+
// Driver configuration
|
|
3790
|
+
driver: options.driver || "s3",
|
|
3791
|
+
config: {
|
|
3792
|
+
ttl: options.ttl,
|
|
3793
|
+
maxSize: options.maxSize,
|
|
3794
|
+
...options.config
|
|
3795
|
+
// Driver-specific config (can override ttl/maxSize)
|
|
3796
|
+
},
|
|
3797
|
+
// Resource filtering
|
|
3798
|
+
include: options.include || null,
|
|
3799
|
+
// Array of resource names to cache (null = all)
|
|
3800
|
+
exclude: options.exclude || [],
|
|
3801
|
+
// Array of resource names to exclude
|
|
3802
|
+
// Partition settings
|
|
3803
|
+
includePartitions: options.includePartitions !== false,
|
|
3804
|
+
partitionStrategy: options.partitionStrategy || "hierarchical",
|
|
3805
|
+
partitionAware: options.partitionAware !== false,
|
|
3806
|
+
trackUsage: options.trackUsage !== false,
|
|
3807
|
+
preloadRelated: options.preloadRelated !== false,
|
|
3808
|
+
// Retry configuration
|
|
3809
|
+
retryAttempts: options.retryAttempts || 3,
|
|
3810
|
+
retryDelay: options.retryDelay || 100,
|
|
3811
|
+
// ms
|
|
3812
|
+
// Logging
|
|
3813
|
+
verbose: options.verbose || false
|
|
3589
3814
|
};
|
|
3590
3815
|
}
|
|
3591
3816
|
async setup(database) {
|
|
3592
3817
|
await super.setup(database);
|
|
3593
3818
|
}
|
|
3594
3819
|
async onSetup() {
|
|
3595
|
-
if (this.
|
|
3596
|
-
this.driver = this.
|
|
3597
|
-
} else if (this.
|
|
3598
|
-
|
|
3599
|
-
|
|
3600
|
-
|
|
3601
|
-
...this.config
|
|
3602
|
-
// New config format (medium priority)
|
|
3603
|
-
};
|
|
3604
|
-
if (this.ttl !== void 0) {
|
|
3605
|
-
driverConfig.ttl = this.ttl;
|
|
3606
|
-
}
|
|
3607
|
-
if (this.maxSize !== void 0) {
|
|
3608
|
-
driverConfig.maxSize = this.maxSize;
|
|
3609
|
-
}
|
|
3610
|
-
this.driver = new MemoryCache(driverConfig);
|
|
3611
|
-
} else if (this.driverName === "filesystem") {
|
|
3612
|
-
const driverConfig = {
|
|
3613
|
-
...this.legacyConfig.filesystemOptions,
|
|
3614
|
-
// Legacy support (lowest priority)
|
|
3615
|
-
...this.config
|
|
3616
|
-
// New config format (medium priority)
|
|
3617
|
-
};
|
|
3618
|
-
if (this.ttl !== void 0) {
|
|
3619
|
-
driverConfig.ttl = this.ttl;
|
|
3620
|
-
}
|
|
3621
|
-
if (this.maxSize !== void 0) {
|
|
3622
|
-
driverConfig.maxSize = this.maxSize;
|
|
3623
|
-
}
|
|
3624
|
-
if (this.partitionAware) {
|
|
3820
|
+
if (this.config.driver && typeof this.config.driver === "object") {
|
|
3821
|
+
this.driver = this.config.driver;
|
|
3822
|
+
} else if (this.config.driver === "memory") {
|
|
3823
|
+
this.driver = new MemoryCache(this.config.config);
|
|
3824
|
+
} else if (this.config.driver === "filesystem") {
|
|
3825
|
+
if (this.config.partitionAware) {
|
|
3625
3826
|
this.driver = new PartitionAwareFilesystemCache({
|
|
3626
|
-
partitionStrategy: this.partitionStrategy,
|
|
3627
|
-
trackUsage: this.trackUsage,
|
|
3628
|
-
preloadRelated: this.preloadRelated,
|
|
3629
|
-
...
|
|
3827
|
+
partitionStrategy: this.config.partitionStrategy,
|
|
3828
|
+
trackUsage: this.config.trackUsage,
|
|
3829
|
+
preloadRelated: this.config.preloadRelated,
|
|
3830
|
+
...this.config.config
|
|
3630
3831
|
});
|
|
3631
3832
|
} else {
|
|
3632
|
-
this.driver = new FilesystemCache(
|
|
3833
|
+
this.driver = new FilesystemCache(this.config.config);
|
|
3633
3834
|
}
|
|
3634
3835
|
} else {
|
|
3635
|
-
|
|
3836
|
+
this.driver = new S3Cache({
|
|
3636
3837
|
client: this.database.client,
|
|
3637
|
-
|
|
3638
|
-
|
|
3639
|
-
// Legacy support (lowest priority)
|
|
3640
|
-
...this.config
|
|
3641
|
-
// New config format (medium priority)
|
|
3642
|
-
};
|
|
3643
|
-
if (this.ttl !== void 0) {
|
|
3644
|
-
driverConfig.ttl = this.ttl;
|
|
3645
|
-
}
|
|
3646
|
-
if (this.maxSize !== void 0) {
|
|
3647
|
-
driverConfig.maxSize = this.maxSize;
|
|
3648
|
-
}
|
|
3649
|
-
this.driver = new S3Cache(driverConfig);
|
|
3838
|
+
...this.config.config
|
|
3839
|
+
});
|
|
3650
3840
|
}
|
|
3651
3841
|
this.installDatabaseHooks();
|
|
3652
3842
|
this.installResourceHooks();
|
|
@@ -3656,7 +3846,9 @@ class CachePlugin extends Plugin {
|
|
|
3656
3846
|
*/
|
|
3657
3847
|
installDatabaseHooks() {
|
|
3658
3848
|
this.database.addHook("afterCreateResource", async ({ resource }) => {
|
|
3659
|
-
this.
|
|
3849
|
+
if (this.shouldCacheResource(resource.name)) {
|
|
3850
|
+
this.installResourceHooksForResource(resource);
|
|
3851
|
+
}
|
|
3660
3852
|
});
|
|
3661
3853
|
}
|
|
3662
3854
|
async onStart() {
|
|
@@ -3666,9 +3858,24 @@ class CachePlugin extends Plugin {
|
|
|
3666
3858
|
// Remove the old installDatabaseProxy method
|
|
3667
3859
|
installResourceHooks() {
|
|
3668
3860
|
for (const resource of Object.values(this.database.resources)) {
|
|
3861
|
+
if (!this.shouldCacheResource(resource.name)) {
|
|
3862
|
+
continue;
|
|
3863
|
+
}
|
|
3669
3864
|
this.installResourceHooksForResource(resource);
|
|
3670
3865
|
}
|
|
3671
3866
|
}
|
|
3867
|
+
shouldCacheResource(resourceName) {
|
|
3868
|
+
if (resourceName.startsWith("plg_") && !this.config.include) {
|
|
3869
|
+
return false;
|
|
3870
|
+
}
|
|
3871
|
+
if (this.config.exclude.includes(resourceName)) {
|
|
3872
|
+
return false;
|
|
3873
|
+
}
|
|
3874
|
+
if (this.config.include && !this.config.include.includes(resourceName)) {
|
|
3875
|
+
return false;
|
|
3876
|
+
}
|
|
3877
|
+
return true;
|
|
3878
|
+
}
|
|
3672
3879
|
installResourceHooksForResource(resource) {
|
|
3673
3880
|
if (!this.driver) return;
|
|
3674
3881
|
Object.defineProperty(resource, "cache", {
|
|
@@ -3817,37 +4024,74 @@ class CachePlugin extends Plugin {
|
|
|
3817
4024
|
if (data && data.id) {
|
|
3818
4025
|
const itemSpecificMethods = ["get", "exists", "content", "hasContent"];
|
|
3819
4026
|
for (const method of itemSpecificMethods) {
|
|
3820
|
-
|
|
3821
|
-
|
|
3822
|
-
|
|
3823
|
-
|
|
4027
|
+
const specificKey = await this.generateCacheKey(resource, method, { id: data.id });
|
|
4028
|
+
const [ok2, err2] = await this.clearCacheWithRetry(resource.cache, specificKey);
|
|
4029
|
+
if (!ok2) {
|
|
4030
|
+
this.emit("cache_clear_error", {
|
|
4031
|
+
resource: resource.name,
|
|
4032
|
+
method,
|
|
4033
|
+
id: data.id,
|
|
4034
|
+
error: err2.message
|
|
4035
|
+
});
|
|
4036
|
+
if (this.config.verbose) {
|
|
4037
|
+
console.warn(`[CachePlugin] Failed to clear ${method} cache for ${resource.name}:${data.id}:`, err2.message);
|
|
4038
|
+
}
|
|
3824
4039
|
}
|
|
3825
4040
|
}
|
|
3826
4041
|
if (this.config.includePartitions === true && resource.config?.partitions && Object.keys(resource.config.partitions).length > 0) {
|
|
3827
4042
|
const partitionValues = this.getPartitionValues(data, resource);
|
|
3828
4043
|
for (const [partitionName, values] of Object.entries(partitionValues)) {
|
|
3829
4044
|
if (values && Object.keys(values).length > 0 && Object.values(values).some((v) => v !== null && v !== void 0)) {
|
|
3830
|
-
|
|
3831
|
-
|
|
3832
|
-
|
|
3833
|
-
|
|
4045
|
+
const partitionKeyPrefix = join(keyPrefix, `partition=${partitionName}`);
|
|
4046
|
+
const [ok2, err2] = await this.clearCacheWithRetry(resource.cache, partitionKeyPrefix);
|
|
4047
|
+
if (!ok2) {
|
|
4048
|
+
this.emit("cache_clear_error", {
|
|
4049
|
+
resource: resource.name,
|
|
4050
|
+
partition: partitionName,
|
|
4051
|
+
error: err2.message
|
|
4052
|
+
});
|
|
4053
|
+
if (this.config.verbose) {
|
|
4054
|
+
console.warn(`[CachePlugin] Failed to clear partition cache for ${resource.name}/${partitionName}:`, err2.message);
|
|
4055
|
+
}
|
|
3834
4056
|
}
|
|
3835
4057
|
}
|
|
3836
4058
|
}
|
|
3837
4059
|
}
|
|
3838
4060
|
}
|
|
3839
|
-
|
|
3840
|
-
|
|
3841
|
-
|
|
4061
|
+
const [ok, err] = await this.clearCacheWithRetry(resource.cache, keyPrefix);
|
|
4062
|
+
if (!ok) {
|
|
4063
|
+
this.emit("cache_clear_error", {
|
|
4064
|
+
resource: resource.name,
|
|
4065
|
+
type: "broad",
|
|
4066
|
+
error: err.message
|
|
4067
|
+
});
|
|
4068
|
+
if (this.config.verbose) {
|
|
4069
|
+
console.warn(`[CachePlugin] Failed to clear broad cache for ${resource.name}, trying specific methods:`, err.message);
|
|
4070
|
+
}
|
|
3842
4071
|
const aggregateMethods = ["count", "list", "listIds", "getAll", "page", "query"];
|
|
3843
4072
|
for (const method of aggregateMethods) {
|
|
3844
|
-
|
|
3845
|
-
|
|
3846
|
-
|
|
3847
|
-
|
|
3848
|
-
|
|
4073
|
+
await this.clearCacheWithRetry(resource.cache, `${keyPrefix}/action=${method}`);
|
|
4074
|
+
await this.clearCacheWithRetry(resource.cache, `resource=${resource.name}/action=${method}`);
|
|
4075
|
+
}
|
|
4076
|
+
}
|
|
4077
|
+
}
|
|
4078
|
+
async clearCacheWithRetry(cache, key) {
|
|
4079
|
+
let lastError;
|
|
4080
|
+
for (let attempt = 0; attempt < this.config.retryAttempts; attempt++) {
|
|
4081
|
+
const [ok, err] = await tryFn(() => cache.clear(key));
|
|
4082
|
+
if (ok) {
|
|
4083
|
+
return [true, null];
|
|
4084
|
+
}
|
|
4085
|
+
lastError = err;
|
|
4086
|
+
if (err.name === "NoSuchKey" || err.code === "NoSuchKey") {
|
|
4087
|
+
return [true, null];
|
|
4088
|
+
}
|
|
4089
|
+
if (attempt < this.config.retryAttempts - 1) {
|
|
4090
|
+
const delay = this.config.retryDelay * Math.pow(2, attempt);
|
|
4091
|
+
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
3849
4092
|
}
|
|
3850
4093
|
}
|
|
4094
|
+
return [false, lastError];
|
|
3851
4095
|
}
|
|
3852
4096
|
async generateCacheKey(resource, action, params = {}, partition = null, partitionValues = null) {
|
|
3853
4097
|
const keyParts = [
|
|
@@ -3863,14 +4107,14 @@ class CachePlugin extends Plugin {
|
|
|
3863
4107
|
}
|
|
3864
4108
|
}
|
|
3865
4109
|
if (Object.keys(params).length > 0) {
|
|
3866
|
-
const paramsHash =
|
|
4110
|
+
const paramsHash = this.hashParams(params);
|
|
3867
4111
|
keyParts.push(paramsHash);
|
|
3868
4112
|
}
|
|
3869
4113
|
return join(...keyParts) + ".json.gz";
|
|
3870
4114
|
}
|
|
3871
|
-
|
|
3872
|
-
const
|
|
3873
|
-
return
|
|
4115
|
+
hashParams(params) {
|
|
4116
|
+
const serialized = jsonStableStringify(params) || "empty";
|
|
4117
|
+
return crypto.createHash("md5").update(serialized).digest("hex").substring(0, 16);
|
|
3874
4118
|
}
|
|
3875
4119
|
// Utility methods
|
|
3876
4120
|
async getCacheStats() {
|
|
@@ -3895,50 +4139,48 @@ class CachePlugin extends Plugin {
|
|
|
3895
4139
|
if (!resource) {
|
|
3896
4140
|
throw new Error(`Resource '${resourceName}' not found`);
|
|
3897
4141
|
}
|
|
3898
|
-
const { includePartitions = true } = options;
|
|
4142
|
+
const { includePartitions = true, sampleSize = 100 } = options;
|
|
3899
4143
|
if (this.driver instanceof PartitionAwareFilesystemCache && resource.warmPartitionCache) {
|
|
3900
4144
|
const partitionNames = resource.config.partitions ? Object.keys(resource.config.partitions) : [];
|
|
3901
4145
|
return await resource.warmPartitionCache(partitionNames, options);
|
|
3902
4146
|
}
|
|
3903
|
-
|
|
3904
|
-
|
|
4147
|
+
let offset = 0;
|
|
4148
|
+
const pageSize = 100;
|
|
4149
|
+
const sampledRecords = [];
|
|
4150
|
+
while (sampledRecords.length < sampleSize) {
|
|
4151
|
+
const [ok, err, pageResult] = await tryFn(() => resource.page({ offset, size: pageSize }));
|
|
4152
|
+
if (!ok || !pageResult) {
|
|
4153
|
+
break;
|
|
4154
|
+
}
|
|
4155
|
+
const pageItems = Array.isArray(pageResult) ? pageResult : pageResult.items || [];
|
|
4156
|
+
if (pageItems.length === 0) {
|
|
4157
|
+
break;
|
|
4158
|
+
}
|
|
4159
|
+
sampledRecords.push(...pageItems);
|
|
4160
|
+
offset += pageSize;
|
|
4161
|
+
}
|
|
4162
|
+
if (includePartitions && resource.config.partitions && sampledRecords.length > 0) {
|
|
3905
4163
|
for (const [partitionName, partitionDef] of Object.entries(resource.config.partitions)) {
|
|
3906
4164
|
if (partitionDef.fields) {
|
|
3907
|
-
const
|
|
3908
|
-
const
|
|
3909
|
-
const partitionValues = /* @__PURE__ */ new Set();
|
|
3910
|
-
for (const record of recordsArray.slice(0, 10)) {
|
|
4165
|
+
const partitionValuesSet = /* @__PURE__ */ new Set();
|
|
4166
|
+
for (const record of sampledRecords) {
|
|
3911
4167
|
const values = this.getPartitionValues(record, resource);
|
|
3912
4168
|
if (values[partitionName]) {
|
|
3913
|
-
|
|
4169
|
+
partitionValuesSet.add(JSON.stringify(values[partitionName]));
|
|
3914
4170
|
}
|
|
3915
4171
|
}
|
|
3916
|
-
for (const partitionValueStr of
|
|
3917
|
-
const
|
|
3918
|
-
await resource.list({ partition: partitionName, partitionValues
|
|
4172
|
+
for (const partitionValueStr of partitionValuesSet) {
|
|
4173
|
+
const partitionValues = JSON.parse(partitionValueStr);
|
|
4174
|
+
await tryFn(() => resource.list({ partition: partitionName, partitionValues }));
|
|
3919
4175
|
}
|
|
3920
4176
|
}
|
|
3921
4177
|
}
|
|
3922
4178
|
}
|
|
3923
|
-
|
|
3924
|
-
|
|
3925
|
-
|
|
3926
|
-
|
|
3927
|
-
|
|
3928
|
-
}
|
|
3929
|
-
return await this.driver.getPartitionStats(resourceName, partition);
|
|
3930
|
-
}
|
|
3931
|
-
async getCacheRecommendations(resourceName) {
|
|
3932
|
-
if (!(this.driver instanceof PartitionAwareFilesystemCache)) {
|
|
3933
|
-
throw new Error("Cache recommendations are only available with PartitionAwareFilesystemCache");
|
|
3934
|
-
}
|
|
3935
|
-
return await this.driver.getCacheRecommendations(resourceName);
|
|
3936
|
-
}
|
|
3937
|
-
async clearPartitionCache(resourceName, partition, partitionValues = {}) {
|
|
3938
|
-
if (!(this.driver instanceof PartitionAwareFilesystemCache)) {
|
|
3939
|
-
throw new Error("Partition cache clearing is only available with PartitionAwareFilesystemCache");
|
|
3940
|
-
}
|
|
3941
|
-
return await this.driver.clearPartition(resourceName, partition, partitionValues);
|
|
4179
|
+
return {
|
|
4180
|
+
resourceName,
|
|
4181
|
+
recordsSampled: sampledRecords.length,
|
|
4182
|
+
partitionsWarmed: includePartitions && resource.config.partitions ? Object.keys(resource.config.partitions).length : 0
|
|
4183
|
+
};
|
|
3942
4184
|
}
|
|
3943
4185
|
async analyzeCacheUsage() {
|
|
3944
4186
|
if (!(this.driver instanceof PartitionAwareFilesystemCache)) {
|
|
@@ -3955,6 +4197,9 @@ class CachePlugin extends Plugin {
|
|
|
3955
4197
|
}
|
|
3956
4198
|
};
|
|
3957
4199
|
for (const [resourceName, resource] of Object.entries(this.database.resources)) {
|
|
4200
|
+
if (!this.shouldCacheResource(resourceName)) {
|
|
4201
|
+
continue;
|
|
4202
|
+
}
|
|
3958
4203
|
try {
|
|
3959
4204
|
analysis.resourceStats[resourceName] = await this.driver.getPartitionStats(resourceName);
|
|
3960
4205
|
analysis.recommendations[resourceName] = await this.driver.getCacheRecommendations(resourceName);
|
|
@@ -4046,62 +4291,807 @@ const CostsPlugin = {
|
|
|
4046
4291
|
}
|
|
4047
4292
|
};
|
|
4048
4293
|
|
|
4049
|
-
class
|
|
4294
|
+
class EventualConsistencyPlugin extends Plugin {
|
|
4050
4295
|
constructor(options = {}) {
|
|
4051
|
-
super();
|
|
4052
|
-
|
|
4296
|
+
super(options);
|
|
4297
|
+
if (!options.resource) {
|
|
4298
|
+
throw new Error("EventualConsistencyPlugin requires 'resource' option");
|
|
4299
|
+
}
|
|
4300
|
+
if (!options.field) {
|
|
4301
|
+
throw new Error("EventualConsistencyPlugin requires 'field' option");
|
|
4302
|
+
}
|
|
4303
|
+
const detectedTimezone = this._detectTimezone();
|
|
4053
4304
|
this.config = {
|
|
4054
|
-
|
|
4055
|
-
|
|
4056
|
-
|
|
4305
|
+
resource: options.resource,
|
|
4306
|
+
field: options.field,
|
|
4307
|
+
cohort: {
|
|
4308
|
+
timezone: options.cohort?.timezone || detectedTimezone
|
|
4309
|
+
},
|
|
4310
|
+
reducer: options.reducer || ((transactions) => {
|
|
4311
|
+
let baseValue = 0;
|
|
4312
|
+
for (const t of transactions) {
|
|
4313
|
+
if (t.operation === "set") {
|
|
4314
|
+
baseValue = t.value;
|
|
4315
|
+
} else if (t.operation === "add") {
|
|
4316
|
+
baseValue += t.value;
|
|
4317
|
+
} else if (t.operation === "sub") {
|
|
4318
|
+
baseValue -= t.value;
|
|
4319
|
+
}
|
|
4320
|
+
}
|
|
4321
|
+
return baseValue;
|
|
4322
|
+
}),
|
|
4323
|
+
consolidationInterval: options.consolidationInterval ?? 300,
|
|
4324
|
+
// 5 minutes (in seconds)
|
|
4325
|
+
consolidationConcurrency: options.consolidationConcurrency || 5,
|
|
4326
|
+
consolidationWindow: options.consolidationWindow || 24,
|
|
4327
|
+
// Hours to look back for pending transactions (watermark)
|
|
4328
|
+
autoConsolidate: options.autoConsolidate !== false,
|
|
4329
|
+
lateArrivalStrategy: options.lateArrivalStrategy || "warn",
|
|
4330
|
+
// 'ignore', 'warn', 'process'
|
|
4331
|
+
batchTransactions: options.batchTransactions || false,
|
|
4332
|
+
// CAUTION: Not safe in distributed environments! Loses data on container crash
|
|
4333
|
+
batchSize: options.batchSize || 100,
|
|
4334
|
+
mode: options.mode || "async",
|
|
4335
|
+
// 'async' or 'sync'
|
|
4336
|
+
lockTimeout: options.lockTimeout || 300,
|
|
4337
|
+
// 5 minutes (in seconds, configurable)
|
|
4338
|
+
transactionRetention: options.transactionRetention || 30,
|
|
4339
|
+
// Days to keep applied transactions
|
|
4340
|
+
gcInterval: options.gcInterval || 86400,
|
|
4341
|
+
// 24 hours (in seconds)
|
|
4342
|
+
verbose: options.verbose || false
|
|
4057
4343
|
};
|
|
4058
|
-
this.
|
|
4059
|
-
|
|
4060
|
-
|
|
4061
|
-
this.
|
|
4062
|
-
|
|
4063
|
-
|
|
4064
|
-
|
|
4065
|
-
|
|
4066
|
-
|
|
4067
|
-
|
|
4068
|
-
|
|
4069
|
-
|
|
4070
|
-
|
|
4071
|
-
|
|
4072
|
-
|
|
4073
|
-
}
|
|
4074
|
-
}));
|
|
4075
|
-
this.indexResource = ok ? indexResource : database.resources.fulltext_indexes;
|
|
4076
|
-
await this.loadIndexes();
|
|
4077
|
-
this.installDatabaseHooks();
|
|
4078
|
-
this.installIndexingHooks();
|
|
4079
|
-
}
|
|
4080
|
-
async start() {
|
|
4344
|
+
this.transactionResource = null;
|
|
4345
|
+
this.targetResource = null;
|
|
4346
|
+
this.consolidationTimer = null;
|
|
4347
|
+
this.gcTimer = null;
|
|
4348
|
+
this.pendingTransactions = /* @__PURE__ */ new Map();
|
|
4349
|
+
if (this.config.batchTransactions && !this.config.verbose) {
|
|
4350
|
+
console.warn(
|
|
4351
|
+
`[EventualConsistency] WARNING: batchTransactions is enabled. This stores transactions in memory and will lose data if container crashes. Not recommended for distributed/production environments. Set verbose: true to suppress this warning.`
|
|
4352
|
+
);
|
|
4353
|
+
}
|
|
4354
|
+
if (this.config.verbose && !options.cohort?.timezone) {
|
|
4355
|
+
console.log(
|
|
4356
|
+
`[EventualConsistency] Auto-detected timezone: ${this.config.cohort.timezone} (from ${process.env.TZ ? "TZ env var" : "system Intl API"})`
|
|
4357
|
+
);
|
|
4358
|
+
}
|
|
4081
4359
|
}
|
|
4082
|
-
async
|
|
4083
|
-
|
|
4084
|
-
this.
|
|
4360
|
+
async onSetup() {
|
|
4361
|
+
this.targetResource = this.database.resources[this.config.resource];
|
|
4362
|
+
if (!this.targetResource) {
|
|
4363
|
+
this.deferredSetup = true;
|
|
4364
|
+
this.watchForResource();
|
|
4365
|
+
return;
|
|
4366
|
+
}
|
|
4367
|
+
await this.completeSetup();
|
|
4085
4368
|
}
|
|
4086
|
-
|
|
4087
|
-
|
|
4088
|
-
|
|
4089
|
-
|
|
4090
|
-
|
|
4091
|
-
|
|
4092
|
-
this.indexes.set(key, {
|
|
4093
|
-
recordIds: indexRecord.recordIds || [],
|
|
4094
|
-
count: indexRecord.count || 0
|
|
4095
|
-
});
|
|
4369
|
+
watchForResource() {
|
|
4370
|
+
const hookCallback = async ({ resource, config }) => {
|
|
4371
|
+
if (config.name === this.config.resource && this.deferredSetup) {
|
|
4372
|
+
this.targetResource = resource;
|
|
4373
|
+
this.deferredSetup = false;
|
|
4374
|
+
await this.completeSetup();
|
|
4096
4375
|
}
|
|
4376
|
+
};
|
|
4377
|
+
this.database.addHook("afterCreateResource", hookCallback);
|
|
4378
|
+
}
|
|
4379
|
+
async completeSetup() {
|
|
4380
|
+
if (!this.targetResource) return;
|
|
4381
|
+
const transactionResourceName = `${this.config.resource}_transactions_${this.config.field}`;
|
|
4382
|
+
const partitionConfig = this.createPartitionConfig();
|
|
4383
|
+
const [ok, err, transactionResource] = await tryFn(
|
|
4384
|
+
() => this.database.createResource({
|
|
4385
|
+
name: transactionResourceName,
|
|
4386
|
+
attributes: {
|
|
4387
|
+
id: "string|required",
|
|
4388
|
+
originalId: "string|required",
|
|
4389
|
+
field: "string|required",
|
|
4390
|
+
value: "number|required",
|
|
4391
|
+
operation: "string|required",
|
|
4392
|
+
// 'set', 'add', or 'sub'
|
|
4393
|
+
timestamp: "string|required",
|
|
4394
|
+
cohortDate: "string|required",
|
|
4395
|
+
// For daily partitioning
|
|
4396
|
+
cohortHour: "string|required",
|
|
4397
|
+
// For hourly partitioning
|
|
4398
|
+
cohortMonth: "string|optional",
|
|
4399
|
+
// For monthly partitioning
|
|
4400
|
+
source: "string|optional",
|
|
4401
|
+
applied: "boolean|optional"
|
|
4402
|
+
// Track if transaction was applied
|
|
4403
|
+
},
|
|
4404
|
+
behavior: "body-overflow",
|
|
4405
|
+
timestamps: true,
|
|
4406
|
+
partitions: partitionConfig,
|
|
4407
|
+
asyncPartitions: true
|
|
4408
|
+
// Use async partitions for better performance
|
|
4409
|
+
})
|
|
4410
|
+
);
|
|
4411
|
+
if (!ok && !this.database.resources[transactionResourceName]) {
|
|
4412
|
+
throw new Error(`Failed to create transaction resource: ${err?.message}`);
|
|
4413
|
+
}
|
|
4414
|
+
this.transactionResource = ok ? transactionResource : this.database.resources[transactionResourceName];
|
|
4415
|
+
const lockResourceName = `${this.config.resource}_consolidation_locks_${this.config.field}`;
|
|
4416
|
+
const [lockOk, lockErr, lockResource] = await tryFn(
|
|
4417
|
+
() => this.database.createResource({
|
|
4418
|
+
name: lockResourceName,
|
|
4419
|
+
attributes: {
|
|
4420
|
+
id: "string|required",
|
|
4421
|
+
lockedAt: "number|required",
|
|
4422
|
+
workerId: "string|optional"
|
|
4423
|
+
},
|
|
4424
|
+
behavior: "body-only",
|
|
4425
|
+
timestamps: false
|
|
4426
|
+
})
|
|
4427
|
+
);
|
|
4428
|
+
if (!lockOk && !this.database.resources[lockResourceName]) {
|
|
4429
|
+
throw new Error(`Failed to create lock resource: ${lockErr?.message}`);
|
|
4430
|
+
}
|
|
4431
|
+
this.lockResource = lockOk ? lockResource : this.database.resources[lockResourceName];
|
|
4432
|
+
this.addHelperMethods();
|
|
4433
|
+
if (this.config.autoConsolidate) {
|
|
4434
|
+
this.startConsolidationTimer();
|
|
4097
4435
|
}
|
|
4436
|
+
this.startGarbageCollectionTimer();
|
|
4098
4437
|
}
|
|
4099
|
-
async
|
|
4100
|
-
if (
|
|
4101
|
-
|
|
4102
|
-
|
|
4103
|
-
|
|
4104
|
-
|
|
4438
|
+
async onStart() {
|
|
4439
|
+
if (this.deferredSetup) {
|
|
4440
|
+
return;
|
|
4441
|
+
}
|
|
4442
|
+
this.emit("eventual-consistency.started", {
|
|
4443
|
+
resource: this.config.resource,
|
|
4444
|
+
field: this.config.field,
|
|
4445
|
+
cohort: this.config.cohort
|
|
4446
|
+
});
|
|
4447
|
+
}
|
|
4448
|
+
async onStop() {
|
|
4449
|
+
if (this.consolidationTimer) {
|
|
4450
|
+
clearInterval(this.consolidationTimer);
|
|
4451
|
+
this.consolidationTimer = null;
|
|
4452
|
+
}
|
|
4453
|
+
if (this.gcTimer) {
|
|
4454
|
+
clearInterval(this.gcTimer);
|
|
4455
|
+
this.gcTimer = null;
|
|
4456
|
+
}
|
|
4457
|
+
await this.flushPendingTransactions();
|
|
4458
|
+
this.emit("eventual-consistency.stopped", {
|
|
4459
|
+
resource: this.config.resource,
|
|
4460
|
+
field: this.config.field
|
|
4461
|
+
});
|
|
4462
|
+
}
|
|
4463
|
+
createPartitionConfig() {
|
|
4464
|
+
const partitions = {
|
|
4465
|
+
byHour: {
|
|
4466
|
+
fields: {
|
|
4467
|
+
cohortHour: "string"
|
|
4468
|
+
}
|
|
4469
|
+
},
|
|
4470
|
+
byDay: {
|
|
4471
|
+
fields: {
|
|
4472
|
+
cohortDate: "string"
|
|
4473
|
+
}
|
|
4474
|
+
},
|
|
4475
|
+
byMonth: {
|
|
4476
|
+
fields: {
|
|
4477
|
+
cohortMonth: "string"
|
|
4478
|
+
}
|
|
4479
|
+
}
|
|
4480
|
+
};
|
|
4481
|
+
return partitions;
|
|
4482
|
+
}
|
|
4483
|
+
/**
|
|
4484
|
+
* Auto-detect timezone from environment or system
|
|
4485
|
+
* @private
|
|
4486
|
+
*/
|
|
4487
|
+
_detectTimezone() {
|
|
4488
|
+
if (process.env.TZ) {
|
|
4489
|
+
return process.env.TZ;
|
|
4490
|
+
}
|
|
4491
|
+
try {
|
|
4492
|
+
const systemTimezone = Intl.DateTimeFormat().resolvedOptions().timeZone;
|
|
4493
|
+
if (systemTimezone) {
|
|
4494
|
+
return systemTimezone;
|
|
4495
|
+
}
|
|
4496
|
+
} catch (err) {
|
|
4497
|
+
}
|
|
4498
|
+
return "UTC";
|
|
4499
|
+
}
|
|
4500
|
+
/**
|
|
4501
|
+
* Helper method to resolve field and plugin from arguments
|
|
4502
|
+
* Supports both single-field (field, value) and multi-field (field, value) signatures
|
|
4503
|
+
* @private
|
|
4504
|
+
*/
|
|
4505
|
+
_resolveFieldAndPlugin(resource, fieldOrValue, value) {
|
|
4506
|
+
const hasMultipleFields = Object.keys(resource._eventualConsistencyPlugins).length > 1;
|
|
4507
|
+
if (hasMultipleFields && value === void 0) {
|
|
4508
|
+
throw new Error(`Multiple fields have eventual consistency. Please specify the field explicitly.`);
|
|
4509
|
+
}
|
|
4510
|
+
const field = value !== void 0 ? fieldOrValue : this.config.field;
|
|
4511
|
+
const actualValue = value !== void 0 ? value : fieldOrValue;
|
|
4512
|
+
const fieldPlugin = resource._eventualConsistencyPlugins[field];
|
|
4513
|
+
if (!fieldPlugin) {
|
|
4514
|
+
throw new Error(`No eventual consistency plugin found for field "${field}"`);
|
|
4515
|
+
}
|
|
4516
|
+
return { field, value: actualValue, plugin: fieldPlugin };
|
|
4517
|
+
}
|
|
4518
|
+
/**
|
|
4519
|
+
* Helper method to perform atomic consolidation in sync mode
|
|
4520
|
+
* @private
|
|
4521
|
+
*/
|
|
4522
|
+
async _syncModeConsolidate(id, field) {
|
|
4523
|
+
const consolidatedValue = await this.consolidateRecord(id);
|
|
4524
|
+
await this.targetResource.update(id, {
|
|
4525
|
+
[field]: consolidatedValue
|
|
4526
|
+
});
|
|
4527
|
+
return consolidatedValue;
|
|
4528
|
+
}
|
|
4529
|
+
/**
|
|
4530
|
+
* Create synthetic 'set' transaction from current value
|
|
4531
|
+
* @private
|
|
4532
|
+
*/
|
|
4533
|
+
_createSyntheticSetTransaction(currentValue) {
|
|
4534
|
+
return {
|
|
4535
|
+
id: "__synthetic__",
|
|
4536
|
+
operation: "set",
|
|
4537
|
+
value: currentValue,
|
|
4538
|
+
timestamp: (/* @__PURE__ */ new Date(0)).toISOString(),
|
|
4539
|
+
synthetic: true
|
|
4540
|
+
};
|
|
4541
|
+
}
|
|
4542
|
+
addHelperMethods() {
|
|
4543
|
+
const resource = this.targetResource;
|
|
4544
|
+
const defaultField = this.config.field;
|
|
4545
|
+
const plugin = this;
|
|
4546
|
+
if (!resource._eventualConsistencyPlugins) {
|
|
4547
|
+
resource._eventualConsistencyPlugins = {};
|
|
4548
|
+
}
|
|
4549
|
+
resource._eventualConsistencyPlugins[defaultField] = plugin;
|
|
4550
|
+
resource.set = async (id, fieldOrValue, value) => {
|
|
4551
|
+
const { field, value: actualValue, plugin: fieldPlugin } = plugin._resolveFieldAndPlugin(resource, fieldOrValue, value);
|
|
4552
|
+
await fieldPlugin.createTransaction({
|
|
4553
|
+
originalId: id,
|
|
4554
|
+
operation: "set",
|
|
4555
|
+
value: actualValue,
|
|
4556
|
+
source: "set"
|
|
4557
|
+
});
|
|
4558
|
+
if (fieldPlugin.config.mode === "sync") {
|
|
4559
|
+
return await fieldPlugin._syncModeConsolidate(id, field);
|
|
4560
|
+
}
|
|
4561
|
+
return actualValue;
|
|
4562
|
+
};
|
|
4563
|
+
resource.add = async (id, fieldOrAmount, amount) => {
|
|
4564
|
+
const { field, value: actualAmount, plugin: fieldPlugin } = plugin._resolveFieldAndPlugin(resource, fieldOrAmount, amount);
|
|
4565
|
+
await fieldPlugin.createTransaction({
|
|
4566
|
+
originalId: id,
|
|
4567
|
+
operation: "add",
|
|
4568
|
+
value: actualAmount,
|
|
4569
|
+
source: "add"
|
|
4570
|
+
});
|
|
4571
|
+
if (fieldPlugin.config.mode === "sync") {
|
|
4572
|
+
return await fieldPlugin._syncModeConsolidate(id, field);
|
|
4573
|
+
}
|
|
4574
|
+
const currentValue = await fieldPlugin.getConsolidatedValue(id);
|
|
4575
|
+
return currentValue + actualAmount;
|
|
4576
|
+
};
|
|
4577
|
+
resource.sub = async (id, fieldOrAmount, amount) => {
|
|
4578
|
+
const { field, value: actualAmount, plugin: fieldPlugin } = plugin._resolveFieldAndPlugin(resource, fieldOrAmount, amount);
|
|
4579
|
+
await fieldPlugin.createTransaction({
|
|
4580
|
+
originalId: id,
|
|
4581
|
+
operation: "sub",
|
|
4582
|
+
value: actualAmount,
|
|
4583
|
+
source: "sub"
|
|
4584
|
+
});
|
|
4585
|
+
if (fieldPlugin.config.mode === "sync") {
|
|
4586
|
+
return await fieldPlugin._syncModeConsolidate(id, field);
|
|
4587
|
+
}
|
|
4588
|
+
const currentValue = await fieldPlugin.getConsolidatedValue(id);
|
|
4589
|
+
return currentValue - actualAmount;
|
|
4590
|
+
};
|
|
4591
|
+
resource.consolidate = async (id, field) => {
|
|
4592
|
+
const hasMultipleFields = Object.keys(resource._eventualConsistencyPlugins).length > 1;
|
|
4593
|
+
if (hasMultipleFields && !field) {
|
|
4594
|
+
throw new Error(`Multiple fields have eventual consistency. Please specify the field: consolidate(id, field)`);
|
|
4595
|
+
}
|
|
4596
|
+
const actualField = field || defaultField;
|
|
4597
|
+
const fieldPlugin = resource._eventualConsistencyPlugins[actualField];
|
|
4598
|
+
if (!fieldPlugin) {
|
|
4599
|
+
throw new Error(`No eventual consistency plugin found for field "${actualField}"`);
|
|
4600
|
+
}
|
|
4601
|
+
return await fieldPlugin.consolidateRecord(id);
|
|
4602
|
+
};
|
|
4603
|
+
resource.getConsolidatedValue = async (id, fieldOrOptions, options) => {
|
|
4604
|
+
if (typeof fieldOrOptions === "string") {
|
|
4605
|
+
const field = fieldOrOptions;
|
|
4606
|
+
const fieldPlugin = resource._eventualConsistencyPlugins[field] || plugin;
|
|
4607
|
+
return await fieldPlugin.getConsolidatedValue(id, options || {});
|
|
4608
|
+
} else {
|
|
4609
|
+
return await plugin.getConsolidatedValue(id, fieldOrOptions || {});
|
|
4610
|
+
}
|
|
4611
|
+
};
|
|
4612
|
+
}
|
|
4613
|
+
async createTransaction(data) {
|
|
4614
|
+
const now = /* @__PURE__ */ new Date();
|
|
4615
|
+
const cohortInfo = this.getCohortInfo(now);
|
|
4616
|
+
const watermarkMs = this.config.consolidationWindow * 60 * 60 * 1e3;
|
|
4617
|
+
const watermarkTime = now.getTime() - watermarkMs;
|
|
4618
|
+
const cohortHourDate = /* @__PURE__ */ new Date(cohortInfo.hour + ":00:00Z");
|
|
4619
|
+
if (cohortHourDate.getTime() < watermarkTime) {
|
|
4620
|
+
const hoursLate = Math.floor((now.getTime() - cohortHourDate.getTime()) / (60 * 60 * 1e3));
|
|
4621
|
+
if (this.config.lateArrivalStrategy === "ignore") {
|
|
4622
|
+
if (this.config.verbose) {
|
|
4623
|
+
console.warn(
|
|
4624
|
+
`[EventualConsistency] Late arrival ignored: transaction for ${cohortInfo.hour} is ${hoursLate}h late (watermark: ${this.config.consolidationWindow}h)`
|
|
4625
|
+
);
|
|
4626
|
+
}
|
|
4627
|
+
return null;
|
|
4628
|
+
} else if (this.config.lateArrivalStrategy === "warn") {
|
|
4629
|
+
console.warn(
|
|
4630
|
+
`[EventualConsistency] Late arrival detected: transaction for ${cohortInfo.hour} is ${hoursLate}h late (watermark: ${this.config.consolidationWindow}h). Processing anyway, but consolidation may not pick it up.`
|
|
4631
|
+
);
|
|
4632
|
+
}
|
|
4633
|
+
}
|
|
4634
|
+
const transaction = {
|
|
4635
|
+
id: idGenerator(),
|
|
4636
|
+
// Use nanoid for guaranteed uniqueness
|
|
4637
|
+
originalId: data.originalId,
|
|
4638
|
+
field: this.config.field,
|
|
4639
|
+
value: data.value || 0,
|
|
4640
|
+
operation: data.operation || "set",
|
|
4641
|
+
timestamp: now.toISOString(),
|
|
4642
|
+
cohortDate: cohortInfo.date,
|
|
4643
|
+
cohortHour: cohortInfo.hour,
|
|
4644
|
+
cohortMonth: cohortInfo.month,
|
|
4645
|
+
source: data.source || "unknown",
|
|
4646
|
+
applied: false
|
|
4647
|
+
};
|
|
4648
|
+
if (this.config.batchTransactions) {
|
|
4649
|
+
this.pendingTransactions.set(transaction.id, transaction);
|
|
4650
|
+
if (this.pendingTransactions.size >= this.config.batchSize) {
|
|
4651
|
+
await this.flushPendingTransactions();
|
|
4652
|
+
}
|
|
4653
|
+
} else {
|
|
4654
|
+
await this.transactionResource.insert(transaction);
|
|
4655
|
+
}
|
|
4656
|
+
return transaction;
|
|
4657
|
+
}
|
|
4658
|
+
async flushPendingTransactions() {
|
|
4659
|
+
if (this.pendingTransactions.size === 0) return;
|
|
4660
|
+
const transactions = Array.from(this.pendingTransactions.values());
|
|
4661
|
+
try {
|
|
4662
|
+
await Promise.all(
|
|
4663
|
+
transactions.map(
|
|
4664
|
+
(transaction) => this.transactionResource.insert(transaction)
|
|
4665
|
+
)
|
|
4666
|
+
);
|
|
4667
|
+
this.pendingTransactions.clear();
|
|
4668
|
+
} catch (error) {
|
|
4669
|
+
console.error("Failed to flush pending transactions:", error);
|
|
4670
|
+
throw error;
|
|
4671
|
+
}
|
|
4672
|
+
}
|
|
4673
|
+
getCohortInfo(date) {
|
|
4674
|
+
const tz = this.config.cohort.timezone;
|
|
4675
|
+
const offset = this.getTimezoneOffset(tz);
|
|
4676
|
+
const localDate = new Date(date.getTime() + offset);
|
|
4677
|
+
const year = localDate.getFullYear();
|
|
4678
|
+
const month = String(localDate.getMonth() + 1).padStart(2, "0");
|
|
4679
|
+
const day = String(localDate.getDate()).padStart(2, "0");
|
|
4680
|
+
const hour = String(localDate.getHours()).padStart(2, "0");
|
|
4681
|
+
return {
|
|
4682
|
+
date: `${year}-${month}-${day}`,
|
|
4683
|
+
hour: `${year}-${month}-${day}T${hour}`,
|
|
4684
|
+
// ISO-like format for hour partition
|
|
4685
|
+
month: `${year}-${month}`
|
|
4686
|
+
};
|
|
4687
|
+
}
|
|
4688
|
+
getTimezoneOffset(timezone) {
|
|
4689
|
+
try {
|
|
4690
|
+
const now = /* @__PURE__ */ new Date();
|
|
4691
|
+
const utcDate = new Date(now.toLocaleString("en-US", { timeZone: "UTC" }));
|
|
4692
|
+
const tzDate = new Date(now.toLocaleString("en-US", { timeZone: timezone }));
|
|
4693
|
+
return tzDate.getTime() - utcDate.getTime();
|
|
4694
|
+
} catch (err) {
|
|
4695
|
+
const offsets = {
|
|
4696
|
+
"UTC": 0,
|
|
4697
|
+
"America/New_York": -5 * 36e5,
|
|
4698
|
+
"America/Chicago": -6 * 36e5,
|
|
4699
|
+
"America/Denver": -7 * 36e5,
|
|
4700
|
+
"America/Los_Angeles": -8 * 36e5,
|
|
4701
|
+
"America/Sao_Paulo": -3 * 36e5,
|
|
4702
|
+
"Europe/London": 0,
|
|
4703
|
+
"Europe/Paris": 1 * 36e5,
|
|
4704
|
+
"Europe/Berlin": 1 * 36e5,
|
|
4705
|
+
"Asia/Tokyo": 9 * 36e5,
|
|
4706
|
+
"Asia/Shanghai": 8 * 36e5,
|
|
4707
|
+
"Australia/Sydney": 10 * 36e5
|
|
4708
|
+
};
|
|
4709
|
+
if (this.config.verbose && !offsets[timezone]) {
|
|
4710
|
+
console.warn(
|
|
4711
|
+
`[EventualConsistency] Unknown timezone '${timezone}', using UTC. Consider using a valid IANA timezone (e.g., 'America/New_York')`
|
|
4712
|
+
);
|
|
4713
|
+
}
|
|
4714
|
+
return offsets[timezone] || 0;
|
|
4715
|
+
}
|
|
4716
|
+
}
|
|
4717
|
+
startConsolidationTimer() {
|
|
4718
|
+
const intervalMs = this.config.consolidationInterval * 1e3;
|
|
4719
|
+
this.consolidationTimer = setInterval(async () => {
|
|
4720
|
+
await this.runConsolidation();
|
|
4721
|
+
}, intervalMs);
|
|
4722
|
+
}
|
|
4723
|
+
async runConsolidation() {
|
|
4724
|
+
try {
|
|
4725
|
+
const now = /* @__PURE__ */ new Date();
|
|
4726
|
+
const hoursToCheck = this.config.consolidationWindow || 24;
|
|
4727
|
+
const cohortHours = [];
|
|
4728
|
+
for (let i = 0; i < hoursToCheck; i++) {
|
|
4729
|
+
const date = new Date(now.getTime() - i * 60 * 60 * 1e3);
|
|
4730
|
+
const cohortInfo = this.getCohortInfo(date);
|
|
4731
|
+
cohortHours.push(cohortInfo.hour);
|
|
4732
|
+
}
|
|
4733
|
+
const transactionsByHour = await Promise.all(
|
|
4734
|
+
cohortHours.map(async (cohortHour) => {
|
|
4735
|
+
const [ok, err, txns] = await tryFn(
|
|
4736
|
+
() => this.transactionResource.query({
|
|
4737
|
+
cohortHour,
|
|
4738
|
+
applied: false
|
|
4739
|
+
})
|
|
4740
|
+
);
|
|
4741
|
+
return ok ? txns : [];
|
|
4742
|
+
})
|
|
4743
|
+
);
|
|
4744
|
+
const transactions = transactionsByHour.flat();
|
|
4745
|
+
if (transactions.length === 0) {
|
|
4746
|
+
if (this.config.verbose) {
|
|
4747
|
+
console.log(`[EventualConsistency] No pending transactions to consolidate`);
|
|
4748
|
+
}
|
|
4749
|
+
return;
|
|
4750
|
+
}
|
|
4751
|
+
const uniqueIds = [...new Set(transactions.map((t) => t.originalId))];
|
|
4752
|
+
const { results, errors } = await PromisePool.for(uniqueIds).withConcurrency(this.config.consolidationConcurrency).process(async (id) => {
|
|
4753
|
+
return await this.consolidateRecord(id);
|
|
4754
|
+
});
|
|
4755
|
+
if (errors && errors.length > 0) {
|
|
4756
|
+
console.error(`Consolidation completed with ${errors.length} errors:`, errors);
|
|
4757
|
+
}
|
|
4758
|
+
this.emit("eventual-consistency.consolidated", {
|
|
4759
|
+
resource: this.config.resource,
|
|
4760
|
+
field: this.config.field,
|
|
4761
|
+
recordCount: uniqueIds.length,
|
|
4762
|
+
successCount: results.length,
|
|
4763
|
+
errorCount: errors.length
|
|
4764
|
+
});
|
|
4765
|
+
} catch (error) {
|
|
4766
|
+
console.error("Consolidation error:", error);
|
|
4767
|
+
this.emit("eventual-consistency.consolidation-error", error);
|
|
4768
|
+
}
|
|
4769
|
+
}
|
|
4770
|
+
async consolidateRecord(originalId) {
|
|
4771
|
+
await this.cleanupStaleLocks();
|
|
4772
|
+
const lockId = `lock-${originalId}`;
|
|
4773
|
+
const [lockAcquired, lockErr, lock] = await tryFn(
|
|
4774
|
+
() => this.lockResource.insert({
|
|
4775
|
+
id: lockId,
|
|
4776
|
+
lockedAt: Date.now(),
|
|
4777
|
+
workerId: process.pid ? String(process.pid) : "unknown"
|
|
4778
|
+
})
|
|
4779
|
+
);
|
|
4780
|
+
if (!lockAcquired) {
|
|
4781
|
+
if (this.config.verbose) {
|
|
4782
|
+
console.log(`[EventualConsistency] Lock for ${originalId} already held, skipping`);
|
|
4783
|
+
}
|
|
4784
|
+
const [recordOk, recordErr, record] = await tryFn(
|
|
4785
|
+
() => this.targetResource.get(originalId)
|
|
4786
|
+
);
|
|
4787
|
+
return recordOk && record ? record[this.config.field] || 0 : 0;
|
|
4788
|
+
}
|
|
4789
|
+
try {
|
|
4790
|
+
const [recordOk, recordErr, record] = await tryFn(
|
|
4791
|
+
() => this.targetResource.get(originalId)
|
|
4792
|
+
);
|
|
4793
|
+
const currentValue = recordOk && record ? record[this.config.field] || 0 : 0;
|
|
4794
|
+
const [ok, err, transactions] = await tryFn(
|
|
4795
|
+
() => this.transactionResource.query({
|
|
4796
|
+
originalId,
|
|
4797
|
+
applied: false
|
|
4798
|
+
})
|
|
4799
|
+
);
|
|
4800
|
+
if (!ok || !transactions || transactions.length === 0) {
|
|
4801
|
+
return currentValue;
|
|
4802
|
+
}
|
|
4803
|
+
transactions.sort(
|
|
4804
|
+
(a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()
|
|
4805
|
+
);
|
|
4806
|
+
const hasSetOperation = transactions.some((t) => t.operation === "set");
|
|
4807
|
+
if (currentValue !== 0 && !hasSetOperation) {
|
|
4808
|
+
transactions.unshift(this._createSyntheticSetTransaction(currentValue));
|
|
4809
|
+
}
|
|
4810
|
+
const consolidatedValue = this.config.reducer(transactions);
|
|
4811
|
+
const [updateOk, updateErr] = await tryFn(
|
|
4812
|
+
() => this.targetResource.update(originalId, {
|
|
4813
|
+
[this.config.field]: consolidatedValue
|
|
4814
|
+
})
|
|
4815
|
+
);
|
|
4816
|
+
if (updateOk) {
|
|
4817
|
+
const transactionsToUpdate = transactions.filter((txn) => txn.id !== "__synthetic__");
|
|
4818
|
+
const { results, errors } = await PromisePool.for(transactionsToUpdate).withConcurrency(10).process(async (txn) => {
|
|
4819
|
+
const [ok2, err2] = await tryFn(
|
|
4820
|
+
() => this.transactionResource.update(txn.id, { applied: true })
|
|
4821
|
+
);
|
|
4822
|
+
if (!ok2 && this.config.verbose) {
|
|
4823
|
+
console.warn(`[EventualConsistency] Failed to mark transaction ${txn.id} as applied:`, err2?.message);
|
|
4824
|
+
}
|
|
4825
|
+
return ok2;
|
|
4826
|
+
});
|
|
4827
|
+
if (errors && errors.length > 0 && this.config.verbose) {
|
|
4828
|
+
console.warn(`[EventualConsistency] ${errors.length} transactions failed to mark as applied`);
|
|
4829
|
+
}
|
|
4830
|
+
}
|
|
4831
|
+
return consolidatedValue;
|
|
4832
|
+
} finally {
|
|
4833
|
+
const [lockReleased, lockReleaseErr] = await tryFn(() => this.lockResource.delete(lockId));
|
|
4834
|
+
if (!lockReleased && this.config.verbose) {
|
|
4835
|
+
console.warn(`[EventualConsistency] Failed to release lock ${lockId}:`, lockReleaseErr?.message);
|
|
4836
|
+
}
|
|
4837
|
+
}
|
|
4838
|
+
}
|
|
4839
|
+
async getConsolidatedValue(originalId, options = {}) {
|
|
4840
|
+
const includeApplied = options.includeApplied || false;
|
|
4841
|
+
const startDate = options.startDate;
|
|
4842
|
+
const endDate = options.endDate;
|
|
4843
|
+
const query = { originalId };
|
|
4844
|
+
if (!includeApplied) {
|
|
4845
|
+
query.applied = false;
|
|
4846
|
+
}
|
|
4847
|
+
const [ok, err, transactions] = await tryFn(
|
|
4848
|
+
() => this.transactionResource.query(query)
|
|
4849
|
+
);
|
|
4850
|
+
if (!ok || !transactions || transactions.length === 0) {
|
|
4851
|
+
const [recordOk2, recordErr2, record2] = await tryFn(
|
|
4852
|
+
() => this.targetResource.get(originalId)
|
|
4853
|
+
);
|
|
4854
|
+
if (recordOk2 && record2) {
|
|
4855
|
+
return record2[this.config.field] || 0;
|
|
4856
|
+
}
|
|
4857
|
+
return 0;
|
|
4858
|
+
}
|
|
4859
|
+
let filtered = transactions;
|
|
4860
|
+
if (startDate || endDate) {
|
|
4861
|
+
filtered = transactions.filter((t) => {
|
|
4862
|
+
const timestamp = new Date(t.timestamp);
|
|
4863
|
+
if (startDate && timestamp < new Date(startDate)) return false;
|
|
4864
|
+
if (endDate && timestamp > new Date(endDate)) return false;
|
|
4865
|
+
return true;
|
|
4866
|
+
});
|
|
4867
|
+
}
|
|
4868
|
+
const [recordOk, recordErr, record] = await tryFn(
|
|
4869
|
+
() => this.targetResource.get(originalId)
|
|
4870
|
+
);
|
|
4871
|
+
const currentValue = recordOk && record ? record[this.config.field] || 0 : 0;
|
|
4872
|
+
const hasSetOperation = filtered.some((t) => t.operation === "set");
|
|
4873
|
+
if (currentValue !== 0 && !hasSetOperation) {
|
|
4874
|
+
filtered.unshift(this._createSyntheticSetTransaction(currentValue));
|
|
4875
|
+
}
|
|
4876
|
+
filtered.sort(
|
|
4877
|
+
(a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()
|
|
4878
|
+
);
|
|
4879
|
+
return this.config.reducer(filtered);
|
|
4880
|
+
}
|
|
4881
|
+
// Helper method to get cohort statistics
|
|
4882
|
+
async getCohortStats(cohortDate) {
|
|
4883
|
+
const [ok, err, transactions] = await tryFn(
|
|
4884
|
+
() => this.transactionResource.query({
|
|
4885
|
+
cohortDate
|
|
4886
|
+
})
|
|
4887
|
+
);
|
|
4888
|
+
if (!ok) return null;
|
|
4889
|
+
const stats = {
|
|
4890
|
+
date: cohortDate,
|
|
4891
|
+
transactionCount: transactions.length,
|
|
4892
|
+
totalValue: 0,
|
|
4893
|
+
byOperation: { set: 0, add: 0, sub: 0 },
|
|
4894
|
+
byOriginalId: {}
|
|
4895
|
+
};
|
|
4896
|
+
for (const txn of transactions) {
|
|
4897
|
+
stats.totalValue += txn.value || 0;
|
|
4898
|
+
stats.byOperation[txn.operation] = (stats.byOperation[txn.operation] || 0) + 1;
|
|
4899
|
+
if (!stats.byOriginalId[txn.originalId]) {
|
|
4900
|
+
stats.byOriginalId[txn.originalId] = {
|
|
4901
|
+
count: 0,
|
|
4902
|
+
value: 0
|
|
4903
|
+
};
|
|
4904
|
+
}
|
|
4905
|
+
stats.byOriginalId[txn.originalId].count++;
|
|
4906
|
+
stats.byOriginalId[txn.originalId].value += txn.value || 0;
|
|
4907
|
+
}
|
|
4908
|
+
return stats;
|
|
4909
|
+
}
|
|
4910
|
+
/**
|
|
4911
|
+
* Clean up stale locks that exceed the configured timeout
|
|
4912
|
+
* Uses distributed locking to prevent multiple containers from cleaning simultaneously
|
|
4913
|
+
*/
|
|
4914
|
+
async cleanupStaleLocks() {
|
|
4915
|
+
const now = Date.now();
|
|
4916
|
+
const lockTimeoutMs = this.config.lockTimeout * 1e3;
|
|
4917
|
+
const cutoffTime = now - lockTimeoutMs;
|
|
4918
|
+
const cleanupLockId = `lock-cleanup-${this.config.resource}-${this.config.field}`;
|
|
4919
|
+
const [lockAcquired] = await tryFn(
|
|
4920
|
+
() => this.lockResource.insert({
|
|
4921
|
+
id: cleanupLockId,
|
|
4922
|
+
lockedAt: Date.now(),
|
|
4923
|
+
workerId: process.pid ? String(process.pid) : "unknown"
|
|
4924
|
+
})
|
|
4925
|
+
);
|
|
4926
|
+
if (!lockAcquired) {
|
|
4927
|
+
if (this.config.verbose) {
|
|
4928
|
+
console.log(`[EventualConsistency] Lock cleanup already running in another container`);
|
|
4929
|
+
}
|
|
4930
|
+
return;
|
|
4931
|
+
}
|
|
4932
|
+
try {
|
|
4933
|
+
const [ok, err, locks] = await tryFn(() => this.lockResource.list());
|
|
4934
|
+
if (!ok || !locks || locks.length === 0) return;
|
|
4935
|
+
const staleLocks = locks.filter(
|
|
4936
|
+
(lock) => lock.id !== cleanupLockId && lock.lockedAt < cutoffTime
|
|
4937
|
+
);
|
|
4938
|
+
if (staleLocks.length === 0) return;
|
|
4939
|
+
if (this.config.verbose) {
|
|
4940
|
+
console.log(`[EventualConsistency] Cleaning up ${staleLocks.length} stale locks`);
|
|
4941
|
+
}
|
|
4942
|
+
const { results, errors } = await PromisePool.for(staleLocks).withConcurrency(5).process(async (lock) => {
|
|
4943
|
+
const [deleted] = await tryFn(() => this.lockResource.delete(lock.id));
|
|
4944
|
+
return deleted;
|
|
4945
|
+
});
|
|
4946
|
+
if (errors && errors.length > 0 && this.config.verbose) {
|
|
4947
|
+
console.warn(`[EventualConsistency] ${errors.length} stale locks failed to delete`);
|
|
4948
|
+
}
|
|
4949
|
+
} catch (error) {
|
|
4950
|
+
if (this.config.verbose) {
|
|
4951
|
+
console.warn(`[EventualConsistency] Error cleaning up stale locks:`, error.message);
|
|
4952
|
+
}
|
|
4953
|
+
} finally {
|
|
4954
|
+
await tryFn(() => this.lockResource.delete(cleanupLockId));
|
|
4955
|
+
}
|
|
4956
|
+
}
|
|
4957
|
+
/**
|
|
4958
|
+
* Start garbage collection timer for old applied transactions
|
|
4959
|
+
*/
|
|
4960
|
+
startGarbageCollectionTimer() {
|
|
4961
|
+
const gcIntervalMs = this.config.gcInterval * 1e3;
|
|
4962
|
+
this.gcTimer = setInterval(async () => {
|
|
4963
|
+
await this.runGarbageCollection();
|
|
4964
|
+
}, gcIntervalMs);
|
|
4965
|
+
}
|
|
4966
|
+
/**
|
|
4967
|
+
* Delete old applied transactions based on retention policy
|
|
4968
|
+
* Uses distributed locking to prevent multiple containers from running GC simultaneously
|
|
4969
|
+
*/
|
|
4970
|
+
async runGarbageCollection() {
|
|
4971
|
+
const gcLockId = `lock-gc-${this.config.resource}-${this.config.field}`;
|
|
4972
|
+
const [lockAcquired] = await tryFn(
|
|
4973
|
+
() => this.lockResource.insert({
|
|
4974
|
+
id: gcLockId,
|
|
4975
|
+
lockedAt: Date.now(),
|
|
4976
|
+
workerId: process.pid ? String(process.pid) : "unknown"
|
|
4977
|
+
})
|
|
4978
|
+
);
|
|
4979
|
+
if (!lockAcquired) {
|
|
4980
|
+
if (this.config.verbose) {
|
|
4981
|
+
console.log(`[EventualConsistency] GC already running in another container`);
|
|
4982
|
+
}
|
|
4983
|
+
return;
|
|
4984
|
+
}
|
|
4985
|
+
try {
|
|
4986
|
+
const now = Date.now();
|
|
4987
|
+
const retentionMs = this.config.transactionRetention * 24 * 60 * 60 * 1e3;
|
|
4988
|
+
const cutoffDate = new Date(now - retentionMs);
|
|
4989
|
+
const cutoffIso = cutoffDate.toISOString();
|
|
4990
|
+
if (this.config.verbose) {
|
|
4991
|
+
console.log(`[EventualConsistency] Running GC for transactions older than ${cutoffIso} (${this.config.transactionRetention} days)`);
|
|
4992
|
+
}
|
|
4993
|
+
const cutoffMonth = cutoffDate.toISOString().substring(0, 7);
|
|
4994
|
+
const [ok, err, oldTransactions] = await tryFn(
|
|
4995
|
+
() => this.transactionResource.query({
|
|
4996
|
+
applied: true,
|
|
4997
|
+
timestamp: { "<": cutoffIso }
|
|
4998
|
+
})
|
|
4999
|
+
);
|
|
5000
|
+
if (!ok) {
|
|
5001
|
+
if (this.config.verbose) {
|
|
5002
|
+
console.warn(`[EventualConsistency] GC failed to query transactions:`, err?.message);
|
|
5003
|
+
}
|
|
5004
|
+
return;
|
|
5005
|
+
}
|
|
5006
|
+
if (!oldTransactions || oldTransactions.length === 0) {
|
|
5007
|
+
if (this.config.verbose) {
|
|
5008
|
+
console.log(`[EventualConsistency] No old transactions to clean up`);
|
|
5009
|
+
}
|
|
5010
|
+
return;
|
|
5011
|
+
}
|
|
5012
|
+
if (this.config.verbose) {
|
|
5013
|
+
console.log(`[EventualConsistency] Deleting ${oldTransactions.length} old transactions`);
|
|
5014
|
+
}
|
|
5015
|
+
const { results, errors } = await PromisePool.for(oldTransactions).withConcurrency(10).process(async (txn) => {
|
|
5016
|
+
const [deleted] = await tryFn(() => this.transactionResource.delete(txn.id));
|
|
5017
|
+
return deleted;
|
|
5018
|
+
});
|
|
5019
|
+
if (this.config.verbose) {
|
|
5020
|
+
console.log(`[EventualConsistency] GC completed: ${results.length} deleted, ${errors.length} errors`);
|
|
5021
|
+
}
|
|
5022
|
+
this.emit("eventual-consistency.gc-completed", {
|
|
5023
|
+
resource: this.config.resource,
|
|
5024
|
+
field: this.config.field,
|
|
5025
|
+
deletedCount: results.length,
|
|
5026
|
+
errorCount: errors.length
|
|
5027
|
+
});
|
|
5028
|
+
} catch (error) {
|
|
5029
|
+
if (this.config.verbose) {
|
|
5030
|
+
console.warn(`[EventualConsistency] GC error:`, error.message);
|
|
5031
|
+
}
|
|
5032
|
+
this.emit("eventual-consistency.gc-error", error);
|
|
5033
|
+
} finally {
|
|
5034
|
+
await tryFn(() => this.lockResource.delete(gcLockId));
|
|
5035
|
+
}
|
|
5036
|
+
}
|
|
5037
|
+
}
|
|
5038
|
+
|
|
5039
|
+
class FullTextPlugin extends Plugin {
|
|
5040
|
+
constructor(options = {}) {
|
|
5041
|
+
super();
|
|
5042
|
+
this.indexResource = null;
|
|
5043
|
+
this.config = {
|
|
5044
|
+
minWordLength: options.minWordLength || 3,
|
|
5045
|
+
maxResults: options.maxResults || 100,
|
|
5046
|
+
...options
|
|
5047
|
+
};
|
|
5048
|
+
this.indexes = /* @__PURE__ */ new Map();
|
|
5049
|
+
}
|
|
5050
|
+
async setup(database) {
|
|
5051
|
+
this.database = database;
|
|
5052
|
+
const [ok, err, indexResource] = await tryFn(() => database.createResource({
|
|
5053
|
+
name: "plg_fulltext_indexes",
|
|
5054
|
+
attributes: {
|
|
5055
|
+
id: "string|required",
|
|
5056
|
+
resourceName: "string|required",
|
|
5057
|
+
fieldName: "string|required",
|
|
5058
|
+
word: "string|required",
|
|
5059
|
+
recordIds: "json|required",
|
|
5060
|
+
// Array of record IDs containing this word
|
|
5061
|
+
count: "number|required",
|
|
5062
|
+
lastUpdated: "string|required"
|
|
5063
|
+
}
|
|
5064
|
+
}));
|
|
5065
|
+
this.indexResource = ok ? indexResource : database.resources.fulltext_indexes;
|
|
5066
|
+
await this.loadIndexes();
|
|
5067
|
+
this.installDatabaseHooks();
|
|
5068
|
+
this.installIndexingHooks();
|
|
5069
|
+
}
|
|
5070
|
+
async start() {
|
|
5071
|
+
}
|
|
5072
|
+
async stop() {
|
|
5073
|
+
await this.saveIndexes();
|
|
5074
|
+
this.removeDatabaseHooks();
|
|
5075
|
+
}
|
|
5076
|
+
async loadIndexes() {
|
|
5077
|
+
if (!this.indexResource) return;
|
|
5078
|
+
const [ok, err, allIndexes] = await tryFn(() => this.indexResource.getAll());
|
|
5079
|
+
if (ok) {
|
|
5080
|
+
for (const indexRecord of allIndexes) {
|
|
5081
|
+
const key = `${indexRecord.resourceName}:${indexRecord.fieldName}:${indexRecord.word}`;
|
|
5082
|
+
this.indexes.set(key, {
|
|
5083
|
+
recordIds: indexRecord.recordIds || [],
|
|
5084
|
+
count: indexRecord.count || 0
|
|
5085
|
+
});
|
|
5086
|
+
}
|
|
5087
|
+
}
|
|
5088
|
+
}
|
|
5089
|
+
async saveIndexes() {
|
|
5090
|
+
if (!this.indexResource) return;
|
|
5091
|
+
const [ok, err] = await tryFn(async () => {
|
|
5092
|
+
const existingIndexes = await this.indexResource.getAll();
|
|
5093
|
+
for (const index of existingIndexes) {
|
|
5094
|
+
await this.indexResource.delete(index.id);
|
|
4105
5095
|
}
|
|
4106
5096
|
for (const [key, data] of this.indexes.entries()) {
|
|
4107
5097
|
const [resourceName, fieldName, word] = key.split(":");
|
|
@@ -4119,7 +5109,7 @@ class FullTextPlugin extends Plugin {
|
|
|
4119
5109
|
}
|
|
4120
5110
|
installDatabaseHooks() {
|
|
4121
5111
|
this.database.addHook("afterCreateResource", (resource) => {
|
|
4122
|
-
if (resource.name !== "
|
|
5112
|
+
if (resource.name !== "plg_fulltext_indexes") {
|
|
4123
5113
|
this.installResourceHooks(resource);
|
|
4124
5114
|
}
|
|
4125
5115
|
});
|
|
@@ -4133,14 +5123,14 @@ class FullTextPlugin extends Plugin {
|
|
|
4133
5123
|
}
|
|
4134
5124
|
this.database.plugins.fulltext = this;
|
|
4135
5125
|
for (const resource of Object.values(this.database.resources)) {
|
|
4136
|
-
if (resource.name === "
|
|
5126
|
+
if (resource.name === "plg_fulltext_indexes") continue;
|
|
4137
5127
|
this.installResourceHooks(resource);
|
|
4138
5128
|
}
|
|
4139
5129
|
if (!this.database._fulltextProxyInstalled) {
|
|
4140
5130
|
this.database._previousCreateResourceForFullText = this.database.createResource;
|
|
4141
5131
|
this.database.createResource = async function(...args) {
|
|
4142
5132
|
const resource = await this._previousCreateResourceForFullText(...args);
|
|
4143
|
-
if (this.plugins?.fulltext && resource.name !== "
|
|
5133
|
+
if (this.plugins?.fulltext && resource.name !== "plg_fulltext_indexes") {
|
|
4144
5134
|
this.plugins.fulltext.installResourceHooks(resource);
|
|
4145
5135
|
}
|
|
4146
5136
|
return resource;
|
|
@@ -4148,7 +5138,7 @@ class FullTextPlugin extends Plugin {
|
|
|
4148
5138
|
this.database._fulltextProxyInstalled = true;
|
|
4149
5139
|
}
|
|
4150
5140
|
for (const resource of Object.values(this.database.resources)) {
|
|
4151
|
-
if (resource.name !== "
|
|
5141
|
+
if (resource.name !== "plg_fulltext_indexes") {
|
|
4152
5142
|
this.installResourceHooks(resource);
|
|
4153
5143
|
}
|
|
4154
5144
|
}
|
|
@@ -4391,7 +5381,7 @@ class FullTextPlugin extends Plugin {
|
|
|
4391
5381
|
return this._rebuildAllIndexesInternal();
|
|
4392
5382
|
}
|
|
4393
5383
|
async _rebuildAllIndexesInternal() {
|
|
4394
|
-
const resourceNames = Object.keys(this.database.resources).filter((name) => name !== "
|
|
5384
|
+
const resourceNames = Object.keys(this.database.resources).filter((name) => name !== "plg_fulltext_indexes");
|
|
4395
5385
|
for (const resourceName of resourceNames) {
|
|
4396
5386
|
const [ok, err] = await tryFn(() => this.rebuildIndex(resourceName));
|
|
4397
5387
|
}
|
|
@@ -4443,7 +5433,7 @@ class MetricsPlugin extends Plugin {
|
|
|
4443
5433
|
if (typeof process !== "undefined" && process.env.NODE_ENV === "test") return;
|
|
4444
5434
|
const [ok, err] = await tryFn(async () => {
|
|
4445
5435
|
const [ok1, err1, metricsResource] = await tryFn(() => database.createResource({
|
|
4446
|
-
name: "
|
|
5436
|
+
name: "plg_metrics",
|
|
4447
5437
|
attributes: {
|
|
4448
5438
|
id: "string|required",
|
|
4449
5439
|
type: "string|required",
|
|
@@ -4458,9 +5448,9 @@ class MetricsPlugin extends Plugin {
|
|
|
4458
5448
|
metadata: "json"
|
|
4459
5449
|
}
|
|
4460
5450
|
}));
|
|
4461
|
-
this.metricsResource = ok1 ? metricsResource : database.resources.
|
|
5451
|
+
this.metricsResource = ok1 ? metricsResource : database.resources.plg_metrics;
|
|
4462
5452
|
const [ok2, err2, errorsResource] = await tryFn(() => database.createResource({
|
|
4463
|
-
name: "
|
|
5453
|
+
name: "plg_error_logs",
|
|
4464
5454
|
attributes: {
|
|
4465
5455
|
id: "string|required",
|
|
4466
5456
|
resourceName: "string|required",
|
|
@@ -4470,9 +5460,9 @@ class MetricsPlugin extends Plugin {
|
|
|
4470
5460
|
metadata: "json"
|
|
4471
5461
|
}
|
|
4472
5462
|
}));
|
|
4473
|
-
this.errorsResource = ok2 ? errorsResource : database.resources.
|
|
5463
|
+
this.errorsResource = ok2 ? errorsResource : database.resources.plg_error_logs;
|
|
4474
5464
|
const [ok3, err3, performanceResource] = await tryFn(() => database.createResource({
|
|
4475
|
-
name: "
|
|
5465
|
+
name: "plg_performance_logs",
|
|
4476
5466
|
attributes: {
|
|
4477
5467
|
id: "string|required",
|
|
4478
5468
|
resourceName: "string|required",
|
|
@@ -4482,12 +5472,12 @@ class MetricsPlugin extends Plugin {
|
|
|
4482
5472
|
metadata: "json"
|
|
4483
5473
|
}
|
|
4484
5474
|
}));
|
|
4485
|
-
this.performanceResource = ok3 ? performanceResource : database.resources.
|
|
5475
|
+
this.performanceResource = ok3 ? performanceResource : database.resources.plg_performance_logs;
|
|
4486
5476
|
});
|
|
4487
5477
|
if (!ok) {
|
|
4488
|
-
this.metricsResource = database.resources.
|
|
4489
|
-
this.errorsResource = database.resources.
|
|
4490
|
-
this.performanceResource = database.resources.
|
|
5478
|
+
this.metricsResource = database.resources.plg_metrics;
|
|
5479
|
+
this.errorsResource = database.resources.plg_error_logs;
|
|
5480
|
+
this.performanceResource = database.resources.plg_performance_logs;
|
|
4491
5481
|
}
|
|
4492
5482
|
this.installDatabaseHooks();
|
|
4493
5483
|
this.installMetricsHooks();
|
|
@@ -4506,7 +5496,7 @@ class MetricsPlugin extends Plugin {
|
|
|
4506
5496
|
}
|
|
4507
5497
|
installDatabaseHooks() {
|
|
4508
5498
|
this.database.addHook("afterCreateResource", (resource) => {
|
|
4509
|
-
if (resource.name !== "
|
|
5499
|
+
if (resource.name !== "plg_metrics" && resource.name !== "plg_error_logs" && resource.name !== "plg_performance_logs") {
|
|
4510
5500
|
this.installResourceHooks(resource);
|
|
4511
5501
|
}
|
|
4512
5502
|
});
|
|
@@ -4516,7 +5506,7 @@ class MetricsPlugin extends Plugin {
|
|
|
4516
5506
|
}
|
|
4517
5507
|
installMetricsHooks() {
|
|
4518
5508
|
for (const resource of Object.values(this.database.resources)) {
|
|
4519
|
-
if (["
|
|
5509
|
+
if (["plg_metrics", "plg_error_logs", "plg_performance_logs"].includes(resource.name)) {
|
|
4520
5510
|
continue;
|
|
4521
5511
|
}
|
|
4522
5512
|
this.installResourceHooks(resource);
|
|
@@ -4524,7 +5514,7 @@ class MetricsPlugin extends Plugin {
|
|
|
4524
5514
|
this.database._createResource = this.database.createResource;
|
|
4525
5515
|
this.database.createResource = async function(...args) {
|
|
4526
5516
|
const resource = await this._createResource(...args);
|
|
4527
|
-
if (this.plugins?.metrics && !["
|
|
5517
|
+
if (this.plugins?.metrics && !["plg_metrics", "plg_error_logs", "plg_performance_logs"].includes(resource.name)) {
|
|
4528
5518
|
this.plugins.metrics.installResourceHooks(resource);
|
|
4529
5519
|
}
|
|
4530
5520
|
return resource;
|
|
@@ -5830,10 +6820,10 @@ class Client extends EventEmitter {
|
|
|
5830
6820
|
// Enabled for better performance
|
|
5831
6821
|
keepAliveMsecs: 1e3,
|
|
5832
6822
|
// 1 second keep-alive
|
|
5833
|
-
maxSockets:
|
|
5834
|
-
//
|
|
5835
|
-
maxFreeSockets:
|
|
5836
|
-
//
|
|
6823
|
+
maxSockets: httpClientOptions.maxSockets || 500,
|
|
6824
|
+
// High concurrency support
|
|
6825
|
+
maxFreeSockets: httpClientOptions.maxFreeSockets || 100,
|
|
6826
|
+
// Better connection reuse
|
|
5837
6827
|
timeout: 6e4,
|
|
5838
6828
|
// 60 second timeout
|
|
5839
6829
|
...httpClientOptions
|
|
@@ -5895,7 +6885,7 @@ class Client extends EventEmitter {
|
|
|
5895
6885
|
this.emit("command.response", command.constructor.name, response, command.input);
|
|
5896
6886
|
return response;
|
|
5897
6887
|
}
|
|
5898
|
-
async putObject({ key, metadata, contentType, body, contentEncoding, contentLength }) {
|
|
6888
|
+
async putObject({ key, metadata, contentType, body, contentEncoding, contentLength, ifMatch }) {
|
|
5899
6889
|
const keyPrefix = typeof this.config.keyPrefix === "string" ? this.config.keyPrefix : "";
|
|
5900
6890
|
keyPrefix ? path.join(keyPrefix, key) : key;
|
|
5901
6891
|
const stringMetadata = {};
|
|
@@ -5915,6 +6905,7 @@ class Client extends EventEmitter {
|
|
|
5915
6905
|
if (contentType !== void 0) options.ContentType = contentType;
|
|
5916
6906
|
if (contentEncoding !== void 0) options.ContentEncoding = contentEncoding;
|
|
5917
6907
|
if (contentLength !== void 0) options.ContentLength = contentLength;
|
|
6908
|
+
if (ifMatch !== void 0) options.IfMatch = ifMatch;
|
|
5918
6909
|
let response, error;
|
|
5919
6910
|
try {
|
|
5920
6911
|
response = await this.sendCommand(new PutObjectCommand(options));
|
|
@@ -8078,6 +9069,7 @@ ${errorDetails}`,
|
|
|
8078
9069
|
data._lastModified = request.LastModified;
|
|
8079
9070
|
data._hasContent = request.ContentLength > 0;
|
|
8080
9071
|
data._mimeType = request.ContentType || null;
|
|
9072
|
+
data._etag = request.ETag;
|
|
8081
9073
|
data._v = objectVersion;
|
|
8082
9074
|
if (request.VersionId) data._versionId = request.VersionId;
|
|
8083
9075
|
if (request.Expiration) data._expiresAt = request.Expiration;
|
|
@@ -8289,6 +9281,172 @@ ${errorDetails}`,
|
|
|
8289
9281
|
return finalResult;
|
|
8290
9282
|
}
|
|
8291
9283
|
}
|
|
9284
|
+
/**
|
|
9285
|
+
* Update with conditional check (If-Match ETag)
|
|
9286
|
+
* @param {string} id - Resource ID
|
|
9287
|
+
* @param {Object} attributes - Attributes to update
|
|
9288
|
+
* @param {Object} options - Options including ifMatch (ETag)
|
|
9289
|
+
* @returns {Promise<Object>} { success: boolean, data?: Object, etag?: string, error?: string }
|
|
9290
|
+
* @example
|
|
9291
|
+
* const msg = await resource.get('msg-123');
|
|
9292
|
+
* const result = await resource.updateConditional('msg-123', { status: 'processing' }, { ifMatch: msg._etag });
|
|
9293
|
+
* if (!result.success) {
|
|
9294
|
+
* console.log('Update failed - object was modified by another process');
|
|
9295
|
+
* }
|
|
9296
|
+
*/
|
|
9297
|
+
async updateConditional(id, attributes, options = {}) {
|
|
9298
|
+
if (isEmpty(id)) {
|
|
9299
|
+
throw new Error("id cannot be empty");
|
|
9300
|
+
}
|
|
9301
|
+
const { ifMatch } = options;
|
|
9302
|
+
if (!ifMatch) {
|
|
9303
|
+
throw new Error("updateConditional requires ifMatch option with ETag value");
|
|
9304
|
+
}
|
|
9305
|
+
const exists = await this.exists(id);
|
|
9306
|
+
if (!exists) {
|
|
9307
|
+
return {
|
|
9308
|
+
success: false,
|
|
9309
|
+
error: `Resource with id '${id}' does not exist`
|
|
9310
|
+
};
|
|
9311
|
+
}
|
|
9312
|
+
const originalData = await this.get(id);
|
|
9313
|
+
const attributesClone = cloneDeep(attributes);
|
|
9314
|
+
let mergedData = cloneDeep(originalData);
|
|
9315
|
+
for (const [key2, value] of Object.entries(attributesClone)) {
|
|
9316
|
+
if (key2.includes(".")) {
|
|
9317
|
+
let ref = mergedData;
|
|
9318
|
+
const parts = key2.split(".");
|
|
9319
|
+
for (let i = 0; i < parts.length - 1; i++) {
|
|
9320
|
+
if (typeof ref[parts[i]] !== "object" || ref[parts[i]] === null) {
|
|
9321
|
+
ref[parts[i]] = {};
|
|
9322
|
+
}
|
|
9323
|
+
ref = ref[parts[i]];
|
|
9324
|
+
}
|
|
9325
|
+
ref[parts[parts.length - 1]] = cloneDeep(value);
|
|
9326
|
+
} else if (typeof value === "object" && value !== null && !Array.isArray(value)) {
|
|
9327
|
+
mergedData[key2] = merge({}, mergedData[key2], value);
|
|
9328
|
+
} else {
|
|
9329
|
+
mergedData[key2] = cloneDeep(value);
|
|
9330
|
+
}
|
|
9331
|
+
}
|
|
9332
|
+
if (this.config.timestamps) {
|
|
9333
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
9334
|
+
mergedData.updatedAt = now;
|
|
9335
|
+
if (!mergedData.metadata) mergedData.metadata = {};
|
|
9336
|
+
mergedData.metadata.updatedAt = now;
|
|
9337
|
+
}
|
|
9338
|
+
const preProcessedData = await this.executeHooks("beforeUpdate", cloneDeep(mergedData));
|
|
9339
|
+
const completeData = { ...originalData, ...preProcessedData, id };
|
|
9340
|
+
const { isValid, errors, data } = await this.validate(cloneDeep(completeData));
|
|
9341
|
+
if (!isValid) {
|
|
9342
|
+
return {
|
|
9343
|
+
success: false,
|
|
9344
|
+
error: "Validation failed: " + (errors && errors.length ? JSON.stringify(errors) : "unknown"),
|
|
9345
|
+
validationErrors: errors
|
|
9346
|
+
};
|
|
9347
|
+
}
|
|
9348
|
+
const { id: validatedId, ...validatedAttributes } = data;
|
|
9349
|
+
const mappedData = await this.schema.mapper(validatedAttributes);
|
|
9350
|
+
mappedData._v = String(this.version);
|
|
9351
|
+
const behaviorImpl = getBehavior(this.behavior);
|
|
9352
|
+
const { mappedData: processedMetadata, body } = await behaviorImpl.handleUpdate({
|
|
9353
|
+
resource: this,
|
|
9354
|
+
id,
|
|
9355
|
+
data: validatedAttributes,
|
|
9356
|
+
mappedData,
|
|
9357
|
+
originalData: { ...attributesClone, id }
|
|
9358
|
+
});
|
|
9359
|
+
const key = this.getResourceKey(id);
|
|
9360
|
+
let existingContentType = void 0;
|
|
9361
|
+
let finalBody = body;
|
|
9362
|
+
if (body === "" && this.behavior !== "body-overflow") {
|
|
9363
|
+
const [ok2, err2, existingObject] = await tryFn(() => this.client.getObject(key));
|
|
9364
|
+
if (ok2 && existingObject.ContentLength > 0) {
|
|
9365
|
+
const existingBodyBuffer = Buffer.from(await existingObject.Body.transformToByteArray());
|
|
9366
|
+
const existingBodyString = existingBodyBuffer.toString();
|
|
9367
|
+
const [okParse, errParse] = await tryFn(() => Promise.resolve(JSON.parse(existingBodyString)));
|
|
9368
|
+
if (!okParse) {
|
|
9369
|
+
finalBody = existingBodyBuffer;
|
|
9370
|
+
existingContentType = existingObject.ContentType;
|
|
9371
|
+
}
|
|
9372
|
+
}
|
|
9373
|
+
}
|
|
9374
|
+
let finalContentType = existingContentType;
|
|
9375
|
+
if (finalBody && finalBody !== "" && !finalContentType) {
|
|
9376
|
+
const [okParse, errParse] = await tryFn(() => Promise.resolve(JSON.parse(finalBody)));
|
|
9377
|
+
if (okParse) finalContentType = "application/json";
|
|
9378
|
+
}
|
|
9379
|
+
const [ok, err, response] = await tryFn(() => this.client.putObject({
|
|
9380
|
+
key,
|
|
9381
|
+
body: finalBody,
|
|
9382
|
+
contentType: finalContentType,
|
|
9383
|
+
metadata: processedMetadata,
|
|
9384
|
+
ifMatch
|
|
9385
|
+
// ← Conditional write with ETag
|
|
9386
|
+
}));
|
|
9387
|
+
if (!ok) {
|
|
9388
|
+
if (err.name === "PreconditionFailed" || err.$metadata?.httpStatusCode === 412) {
|
|
9389
|
+
return {
|
|
9390
|
+
success: false,
|
|
9391
|
+
error: "ETag mismatch - object was modified by another process"
|
|
9392
|
+
};
|
|
9393
|
+
}
|
|
9394
|
+
return {
|
|
9395
|
+
success: false,
|
|
9396
|
+
error: err.message || "Update failed"
|
|
9397
|
+
};
|
|
9398
|
+
}
|
|
9399
|
+
const updatedData = await this.composeFullObjectFromWrite({
|
|
9400
|
+
id,
|
|
9401
|
+
metadata: processedMetadata,
|
|
9402
|
+
body: finalBody,
|
|
9403
|
+
behavior: this.behavior
|
|
9404
|
+
});
|
|
9405
|
+
const oldData = { ...originalData, id };
|
|
9406
|
+
const newData = { ...validatedAttributes, id };
|
|
9407
|
+
if (this.config.asyncPartitions && this.config.partitions && Object.keys(this.config.partitions).length > 0) {
|
|
9408
|
+
setImmediate(() => {
|
|
9409
|
+
this.handlePartitionReferenceUpdates(oldData, newData).catch((err2) => {
|
|
9410
|
+
this.emit("partitionIndexError", {
|
|
9411
|
+
operation: "updateConditional",
|
|
9412
|
+
id,
|
|
9413
|
+
error: err2,
|
|
9414
|
+
message: err2.message
|
|
9415
|
+
});
|
|
9416
|
+
});
|
|
9417
|
+
});
|
|
9418
|
+
const nonPartitionHooks = this.hooks.afterUpdate.filter(
|
|
9419
|
+
(hook) => !hook.toString().includes("handlePartitionReferenceUpdates")
|
|
9420
|
+
);
|
|
9421
|
+
let finalResult = updatedData;
|
|
9422
|
+
for (const hook of nonPartitionHooks) {
|
|
9423
|
+
finalResult = await hook(finalResult);
|
|
9424
|
+
}
|
|
9425
|
+
this.emit("update", {
|
|
9426
|
+
...updatedData,
|
|
9427
|
+
$before: { ...originalData },
|
|
9428
|
+
$after: { ...finalResult }
|
|
9429
|
+
});
|
|
9430
|
+
return {
|
|
9431
|
+
success: true,
|
|
9432
|
+
data: finalResult,
|
|
9433
|
+
etag: response.ETag
|
|
9434
|
+
};
|
|
9435
|
+
} else {
|
|
9436
|
+
await this.handlePartitionReferenceUpdates(oldData, newData);
|
|
9437
|
+
const finalResult = await this.executeHooks("afterUpdate", updatedData);
|
|
9438
|
+
this.emit("update", {
|
|
9439
|
+
...updatedData,
|
|
9440
|
+
$before: { ...originalData },
|
|
9441
|
+
$after: { ...finalResult }
|
|
9442
|
+
});
|
|
9443
|
+
return {
|
|
9444
|
+
success: true,
|
|
9445
|
+
data: finalResult,
|
|
9446
|
+
etag: response.ETag
|
|
9447
|
+
};
|
|
9448
|
+
}
|
|
9449
|
+
}
|
|
8292
9450
|
/**
|
|
8293
9451
|
* Delete a resource object by ID
|
|
8294
9452
|
* @param {string} id - Resource ID
|
|
@@ -9696,7 +10854,7 @@ class Database extends EventEmitter {
|
|
|
9696
10854
|
this.id = idGenerator(7);
|
|
9697
10855
|
this.version = "1";
|
|
9698
10856
|
this.s3dbVersion = (() => {
|
|
9699
|
-
const [ok, err, version] = tryFn(() => true ? "
|
|
10857
|
+
const [ok, err, version] = tryFn(() => true ? "10.0.1" : "latest");
|
|
9700
10858
|
return ok ? version : "latest";
|
|
9701
10859
|
})();
|
|
9702
10860
|
this.resources = {};
|
|
@@ -10949,16 +12107,20 @@ class S3dbReplicator extends BaseReplicator {
|
|
|
10949
12107
|
return resource;
|
|
10950
12108
|
}
|
|
10951
12109
|
_getDestResourceObj(resource) {
|
|
10952
|
-
const
|
|
12110
|
+
const db = this.targetDatabase || this.client;
|
|
12111
|
+
const available = Object.keys(db.resources || {});
|
|
10953
12112
|
const norm = normalizeResourceName$1(resource);
|
|
10954
12113
|
const found = available.find((r) => normalizeResourceName$1(r) === norm);
|
|
10955
12114
|
if (!found) {
|
|
10956
12115
|
throw new Error(`[S3dbReplicator] Destination resource not found: ${resource}. Available: ${available.join(", ")}`);
|
|
10957
12116
|
}
|
|
10958
|
-
return
|
|
12117
|
+
return db.resources[found];
|
|
10959
12118
|
}
|
|
10960
12119
|
async replicateBatch(resourceName, records) {
|
|
10961
|
-
if (
|
|
12120
|
+
if (this.enabled === false) {
|
|
12121
|
+
return { skipped: true, reason: "replicator_disabled" };
|
|
12122
|
+
}
|
|
12123
|
+
if (!this.shouldReplicateResource(resourceName)) {
|
|
10962
12124
|
return { skipped: true, reason: "resource_not_included" };
|
|
10963
12125
|
}
|
|
10964
12126
|
const results = [];
|
|
@@ -11069,11 +12231,12 @@ class SqsReplicator extends BaseReplicator {
|
|
|
11069
12231
|
this.client = client;
|
|
11070
12232
|
this.queueUrl = config.queueUrl;
|
|
11071
12233
|
this.queues = config.queues || {};
|
|
11072
|
-
this.defaultQueue = config.defaultQueue || config.defaultQueueUrl || config.queueUrlDefault;
|
|
12234
|
+
this.defaultQueue = config.defaultQueue || config.defaultQueueUrl || config.queueUrlDefault || null;
|
|
11073
12235
|
this.region = config.region || "us-east-1";
|
|
11074
12236
|
this.sqsClient = client || null;
|
|
11075
12237
|
this.messageGroupId = config.messageGroupId;
|
|
11076
12238
|
this.deduplicationId = config.deduplicationId;
|
|
12239
|
+
this.resourceQueueMap = config.resourceQueueMap || null;
|
|
11077
12240
|
if (Array.isArray(resources)) {
|
|
11078
12241
|
this.resources = {};
|
|
11079
12242
|
for (const resource of resources) {
|
|
@@ -11204,7 +12367,10 @@ class SqsReplicator extends BaseReplicator {
|
|
|
11204
12367
|
}
|
|
11205
12368
|
}
|
|
11206
12369
|
async replicate(resource, operation, data, id, beforeData = null) {
|
|
11207
|
-
if (
|
|
12370
|
+
if (this.enabled === false) {
|
|
12371
|
+
return { skipped: true, reason: "replicator_disabled" };
|
|
12372
|
+
}
|
|
12373
|
+
if (!this.shouldReplicateResource(resource)) {
|
|
11208
12374
|
return { skipped: true, reason: "resource_not_included" };
|
|
11209
12375
|
}
|
|
11210
12376
|
const [ok, err, result] = await tryFn(async () => {
|
|
@@ -11248,7 +12414,10 @@ class SqsReplicator extends BaseReplicator {
|
|
|
11248
12414
|
return { success: false, error: err.message };
|
|
11249
12415
|
}
|
|
11250
12416
|
async replicateBatch(resource, records) {
|
|
11251
|
-
if (
|
|
12417
|
+
if (this.enabled === false) {
|
|
12418
|
+
return { skipped: true, reason: "replicator_disabled" };
|
|
12419
|
+
}
|
|
12420
|
+
if (!this.shouldReplicateResource(resource)) {
|
|
11252
12421
|
return { skipped: true, reason: "resource_not_included" };
|
|
11253
12422
|
}
|
|
11254
12423
|
const [ok, err, result] = await tryFn(async () => {
|
|
@@ -11402,22 +12571,23 @@ class ReplicatorPlugin extends Plugin {
|
|
|
11402
12571
|
replicators: options.replicators || [],
|
|
11403
12572
|
logErrors: options.logErrors !== false,
|
|
11404
12573
|
replicatorLogResource: options.replicatorLogResource || "replicator_log",
|
|
12574
|
+
persistReplicatorLog: options.persistReplicatorLog || false,
|
|
11405
12575
|
enabled: options.enabled !== false,
|
|
11406
12576
|
batchSize: options.batchSize || 100,
|
|
11407
12577
|
maxRetries: options.maxRetries || 3,
|
|
11408
12578
|
timeout: options.timeout || 3e4,
|
|
11409
|
-
verbose: options.verbose || false
|
|
11410
|
-
...options
|
|
12579
|
+
verbose: options.verbose || false
|
|
11411
12580
|
};
|
|
11412
12581
|
this.replicators = [];
|
|
11413
12582
|
this.database = null;
|
|
11414
12583
|
this.eventListenersInstalled = /* @__PURE__ */ new Set();
|
|
11415
|
-
|
|
11416
|
-
|
|
11417
|
-
|
|
11418
|
-
|
|
11419
|
-
|
|
11420
|
-
|
|
12584
|
+
this.eventHandlers = /* @__PURE__ */ new Map();
|
|
12585
|
+
this.stats = {
|
|
12586
|
+
totalReplications: 0,
|
|
12587
|
+
totalErrors: 0,
|
|
12588
|
+
lastSync: null
|
|
12589
|
+
};
|
|
12590
|
+
this._afterCreateResourceHook = null;
|
|
11421
12591
|
}
|
|
11422
12592
|
// Helper to filter out internal S3DB fields
|
|
11423
12593
|
filterInternalFields(obj) {
|
|
@@ -11438,7 +12608,7 @@ class ReplicatorPlugin extends Plugin {
|
|
|
11438
12608
|
if (!resource || this.eventListenersInstalled.has(resource.name) || resource.name === this.config.replicatorLogResource) {
|
|
11439
12609
|
return;
|
|
11440
12610
|
}
|
|
11441
|
-
|
|
12611
|
+
const insertHandler = async (data) => {
|
|
11442
12612
|
const [ok, error] = await tryFn(async () => {
|
|
11443
12613
|
const completeData = { ...data, createdAt: (/* @__PURE__ */ new Date()).toISOString() };
|
|
11444
12614
|
await plugin.processReplicatorEvent("insert", resource.name, completeData.id, completeData);
|
|
@@ -11449,8 +12619,8 @@ class ReplicatorPlugin extends Plugin {
|
|
|
11449
12619
|
}
|
|
11450
12620
|
this.emit("error", { operation: "insert", error: error.message, resource: resource.name });
|
|
11451
12621
|
}
|
|
11452
|
-
}
|
|
11453
|
-
|
|
12622
|
+
};
|
|
12623
|
+
const updateHandler = async (data, beforeData) => {
|
|
11454
12624
|
const [ok, error] = await tryFn(async () => {
|
|
11455
12625
|
const completeData = await plugin.getCompleteData(resource, data);
|
|
11456
12626
|
const dataWithTimestamp = { ...completeData, updatedAt: (/* @__PURE__ */ new Date()).toISOString() };
|
|
@@ -11462,8 +12632,8 @@ class ReplicatorPlugin extends Plugin {
|
|
|
11462
12632
|
}
|
|
11463
12633
|
this.emit("error", { operation: "update", error: error.message, resource: resource.name });
|
|
11464
12634
|
}
|
|
11465
|
-
}
|
|
11466
|
-
|
|
12635
|
+
};
|
|
12636
|
+
const deleteHandler = async (data) => {
|
|
11467
12637
|
const [ok, error] = await tryFn(async () => {
|
|
11468
12638
|
await plugin.processReplicatorEvent("delete", resource.name, data.id, data);
|
|
11469
12639
|
});
|
|
@@ -11473,14 +12643,22 @@ class ReplicatorPlugin extends Plugin {
|
|
|
11473
12643
|
}
|
|
11474
12644
|
this.emit("error", { operation: "delete", error: error.message, resource: resource.name });
|
|
11475
12645
|
}
|
|
11476
|
-
}
|
|
12646
|
+
};
|
|
12647
|
+
this.eventHandlers.set(resource.name, {
|
|
12648
|
+
insert: insertHandler,
|
|
12649
|
+
update: updateHandler,
|
|
12650
|
+
delete: deleteHandler
|
|
12651
|
+
});
|
|
12652
|
+
resource.on("insert", insertHandler);
|
|
12653
|
+
resource.on("update", updateHandler);
|
|
12654
|
+
resource.on("delete", deleteHandler);
|
|
11477
12655
|
this.eventListenersInstalled.add(resource.name);
|
|
11478
12656
|
}
|
|
11479
12657
|
async setup(database) {
|
|
11480
12658
|
this.database = database;
|
|
11481
12659
|
if (this.config.persistReplicatorLog) {
|
|
11482
12660
|
const [ok, err, logResource] = await tryFn(() => database.createResource({
|
|
11483
|
-
name: this.config.replicatorLogResource || "
|
|
12661
|
+
name: this.config.replicatorLogResource || "plg_replicator_logs",
|
|
11484
12662
|
attributes: {
|
|
11485
12663
|
id: "string|required",
|
|
11486
12664
|
resource: "string|required",
|
|
@@ -11494,13 +12672,13 @@ class ReplicatorPlugin extends Plugin {
|
|
|
11494
12672
|
if (ok) {
|
|
11495
12673
|
this.replicatorLogResource = logResource;
|
|
11496
12674
|
} else {
|
|
11497
|
-
this.replicatorLogResource = database.resources[this.config.replicatorLogResource || "
|
|
12675
|
+
this.replicatorLogResource = database.resources[this.config.replicatorLogResource || "plg_replicator_logs"];
|
|
11498
12676
|
}
|
|
11499
12677
|
}
|
|
11500
12678
|
await this.initializeReplicators(database);
|
|
11501
12679
|
this.installDatabaseHooks();
|
|
11502
12680
|
for (const resource of Object.values(database.resources)) {
|
|
11503
|
-
if (resource.name !== (this.config.replicatorLogResource || "
|
|
12681
|
+
if (resource.name !== (this.config.replicatorLogResource || "plg_replicator_logs")) {
|
|
11504
12682
|
this.installEventListeners(resource, database, this);
|
|
11505
12683
|
}
|
|
11506
12684
|
}
|
|
@@ -11516,14 +12694,18 @@ class ReplicatorPlugin extends Plugin {
|
|
|
11516
12694
|
this.removeDatabaseHooks();
|
|
11517
12695
|
}
|
|
11518
12696
|
installDatabaseHooks() {
|
|
11519
|
-
this.
|
|
11520
|
-
if (resource.name !== (this.config.replicatorLogResource || "
|
|
12697
|
+
this._afterCreateResourceHook = (resource) => {
|
|
12698
|
+
if (resource.name !== (this.config.replicatorLogResource || "plg_replicator_logs")) {
|
|
11521
12699
|
this.installEventListeners(resource, this.database, this);
|
|
11522
12700
|
}
|
|
11523
|
-
}
|
|
12701
|
+
};
|
|
12702
|
+
this.database.addHook("afterCreateResource", this._afterCreateResourceHook);
|
|
11524
12703
|
}
|
|
11525
12704
|
removeDatabaseHooks() {
|
|
11526
|
-
|
|
12705
|
+
if (this._afterCreateResourceHook) {
|
|
12706
|
+
this.database.removeHook("afterCreateResource", this._afterCreateResourceHook);
|
|
12707
|
+
this._afterCreateResourceHook = null;
|
|
12708
|
+
}
|
|
11527
12709
|
}
|
|
11528
12710
|
createReplicator(driver, config, resources, client) {
|
|
11529
12711
|
return createReplicator(driver, config, resources, client);
|
|
@@ -11548,9 +12730,9 @@ class ReplicatorPlugin extends Plugin {
|
|
|
11548
12730
|
async retryWithBackoff(operation, maxRetries = 3) {
|
|
11549
12731
|
let lastError;
|
|
11550
12732
|
for (let attempt = 1; attempt <= maxRetries; attempt++) {
|
|
11551
|
-
const [ok, error] = await tryFn(operation);
|
|
12733
|
+
const [ok, error, result] = await tryFn(operation);
|
|
11552
12734
|
if (ok) {
|
|
11553
|
-
return
|
|
12735
|
+
return result;
|
|
11554
12736
|
} else {
|
|
11555
12737
|
lastError = error;
|
|
11556
12738
|
if (this.config.verbose) {
|
|
@@ -11645,7 +12827,7 @@ class ReplicatorPlugin extends Plugin {
|
|
|
11645
12827
|
});
|
|
11646
12828
|
return Promise.allSettled(promises);
|
|
11647
12829
|
}
|
|
11648
|
-
async
|
|
12830
|
+
async processReplicatorItem(item) {
|
|
11649
12831
|
const applicableReplicators = this.replicators.filter((replicator) => {
|
|
11650
12832
|
const should = replicator.shouldReplicateResource && replicator.shouldReplicateResource(item.resourceName, item.operation);
|
|
11651
12833
|
return should;
|
|
@@ -11705,12 +12887,9 @@ class ReplicatorPlugin extends Plugin {
|
|
|
11705
12887
|
});
|
|
11706
12888
|
return Promise.allSettled(promises);
|
|
11707
12889
|
}
|
|
11708
|
-
async
|
|
12890
|
+
async logReplicator(item) {
|
|
11709
12891
|
const logRes = this.replicatorLog || this.database.resources[normalizeResourceName(this.config.replicatorLogResource)];
|
|
11710
12892
|
if (!logRes) {
|
|
11711
|
-
if (this.database) {
|
|
11712
|
-
if (this.database.options && this.database.options.connectionString) ;
|
|
11713
|
-
}
|
|
11714
12893
|
this.emit("replicator.log.failed", { error: "replicator log resource not found", item });
|
|
11715
12894
|
return;
|
|
11716
12895
|
}
|
|
@@ -11732,7 +12911,7 @@ class ReplicatorPlugin extends Plugin {
|
|
|
11732
12911
|
this.emit("replicator.log.failed", { error: err, item });
|
|
11733
12912
|
}
|
|
11734
12913
|
}
|
|
11735
|
-
async
|
|
12914
|
+
async updateReplicatorLog(logId, updates) {
|
|
11736
12915
|
if (!this.replicatorLog) return;
|
|
11737
12916
|
const [ok, err] = await tryFn(async () => {
|
|
11738
12917
|
await this.replicatorLog.update(logId, {
|
|
@@ -11745,7 +12924,7 @@ class ReplicatorPlugin extends Plugin {
|
|
|
11745
12924
|
}
|
|
11746
12925
|
}
|
|
11747
12926
|
// Utility methods
|
|
11748
|
-
async
|
|
12927
|
+
async getReplicatorStats() {
|
|
11749
12928
|
const replicatorStats = await Promise.all(
|
|
11750
12929
|
this.replicators.map(async (replicator) => {
|
|
11751
12930
|
const status = await replicator.getStatus();
|
|
@@ -11757,116 +12936,669 @@ class ReplicatorPlugin extends Plugin {
|
|
|
11757
12936
|
};
|
|
11758
12937
|
})
|
|
11759
12938
|
);
|
|
11760
|
-
return {
|
|
11761
|
-
replicators: replicatorStats,
|
|
11762
|
-
|
|
11763
|
-
|
|
11764
|
-
|
|
11765
|
-
|
|
11766
|
-
|
|
11767
|
-
|
|
11768
|
-
|
|
11769
|
-
|
|
11770
|
-
|
|
11771
|
-
|
|
11772
|
-
|
|
11773
|
-
|
|
11774
|
-
|
|
11775
|
-
|
|
11776
|
-
|
|
11777
|
-
|
|
11778
|
-
|
|
11779
|
-
|
|
11780
|
-
}
|
|
11781
|
-
|
|
11782
|
-
|
|
11783
|
-
|
|
12939
|
+
return {
|
|
12940
|
+
replicators: replicatorStats,
|
|
12941
|
+
stats: this.stats,
|
|
12942
|
+
lastSync: this.stats.lastSync
|
|
12943
|
+
};
|
|
12944
|
+
}
|
|
12945
|
+
async getReplicatorLogs(options = {}) {
|
|
12946
|
+
if (!this.replicatorLog) {
|
|
12947
|
+
return [];
|
|
12948
|
+
}
|
|
12949
|
+
const {
|
|
12950
|
+
resourceName,
|
|
12951
|
+
operation,
|
|
12952
|
+
status,
|
|
12953
|
+
limit = 100,
|
|
12954
|
+
offset = 0
|
|
12955
|
+
} = options;
|
|
12956
|
+
const filter = {};
|
|
12957
|
+
if (resourceName) {
|
|
12958
|
+
filter.resourceName = resourceName;
|
|
12959
|
+
}
|
|
12960
|
+
if (operation) {
|
|
12961
|
+
filter.operation = operation;
|
|
12962
|
+
}
|
|
12963
|
+
if (status) {
|
|
12964
|
+
filter.status = status;
|
|
12965
|
+
}
|
|
12966
|
+
const logs = await this.replicatorLog.query(filter, { limit, offset });
|
|
12967
|
+
return logs || [];
|
|
12968
|
+
}
|
|
12969
|
+
async retryFailedReplicators() {
|
|
12970
|
+
if (!this.replicatorLog) {
|
|
12971
|
+
return { retried: 0 };
|
|
12972
|
+
}
|
|
12973
|
+
const failedLogs = await this.replicatorLog.query({
|
|
12974
|
+
status: "failed"
|
|
12975
|
+
});
|
|
12976
|
+
let retried = 0;
|
|
12977
|
+
for (const log of failedLogs || []) {
|
|
12978
|
+
const [ok, err] = await tryFn(async () => {
|
|
12979
|
+
await this.processReplicatorEvent(
|
|
12980
|
+
log.operation,
|
|
12981
|
+
log.resourceName,
|
|
12982
|
+
log.recordId,
|
|
12983
|
+
log.data
|
|
12984
|
+
);
|
|
12985
|
+
});
|
|
12986
|
+
if (ok) {
|
|
12987
|
+
retried++;
|
|
12988
|
+
}
|
|
12989
|
+
}
|
|
12990
|
+
return { retried };
|
|
12991
|
+
}
|
|
12992
|
+
async syncAllData(replicatorId) {
|
|
12993
|
+
const replicator = this.replicators.find((r) => r.id === replicatorId);
|
|
12994
|
+
if (!replicator) {
|
|
12995
|
+
throw new Error(`Replicator not found: ${replicatorId}`);
|
|
12996
|
+
}
|
|
12997
|
+
this.stats.lastSync = (/* @__PURE__ */ new Date()).toISOString();
|
|
12998
|
+
for (const resourceName in this.database.resources) {
|
|
12999
|
+
if (normalizeResourceName(resourceName) === normalizeResourceName("plg_replicator_logs")) continue;
|
|
13000
|
+
if (replicator.shouldReplicateResource(resourceName)) {
|
|
13001
|
+
this.emit("replicator.sync.resource", { resourceName, replicatorId });
|
|
13002
|
+
const resource = this.database.resources[resourceName];
|
|
13003
|
+
let offset = 0;
|
|
13004
|
+
const pageSize = this.config.batchSize || 100;
|
|
13005
|
+
while (true) {
|
|
13006
|
+
const [ok, err, page] = await tryFn(() => resource.page({ offset, size: pageSize }));
|
|
13007
|
+
if (!ok || !page) break;
|
|
13008
|
+
const records = Array.isArray(page) ? page : page.items || [];
|
|
13009
|
+
if (records.length === 0) break;
|
|
13010
|
+
for (const record of records) {
|
|
13011
|
+
await replicator.replicate(resourceName, "insert", record, record.id);
|
|
13012
|
+
}
|
|
13013
|
+
offset += pageSize;
|
|
13014
|
+
}
|
|
13015
|
+
}
|
|
13016
|
+
}
|
|
13017
|
+
this.emit("replicator.sync.completed", { replicatorId, stats: this.stats });
|
|
13018
|
+
}
|
|
13019
|
+
async cleanup() {
|
|
13020
|
+
const [ok, error] = await tryFn(async () => {
|
|
13021
|
+
if (this.replicators && this.replicators.length > 0) {
|
|
13022
|
+
const cleanupPromises = this.replicators.map(async (replicator) => {
|
|
13023
|
+
const [replicatorOk, replicatorError] = await tryFn(async () => {
|
|
13024
|
+
if (replicator && typeof replicator.cleanup === "function") {
|
|
13025
|
+
await replicator.cleanup();
|
|
13026
|
+
}
|
|
13027
|
+
});
|
|
13028
|
+
if (!replicatorOk) {
|
|
13029
|
+
if (this.config.verbose) {
|
|
13030
|
+
console.warn(`[ReplicatorPlugin] Failed to cleanup replicator ${replicator.name || replicator.id}: ${replicatorError.message}`);
|
|
13031
|
+
}
|
|
13032
|
+
this.emit("replicator_cleanup_error", {
|
|
13033
|
+
replicator: replicator.name || replicator.id || "unknown",
|
|
13034
|
+
driver: replicator.driver || "unknown",
|
|
13035
|
+
error: replicatorError.message
|
|
13036
|
+
});
|
|
13037
|
+
}
|
|
13038
|
+
});
|
|
13039
|
+
await Promise.allSettled(cleanupPromises);
|
|
13040
|
+
}
|
|
13041
|
+
if (this.database && this.database.resources) {
|
|
13042
|
+
for (const resourceName of this.eventListenersInstalled) {
|
|
13043
|
+
const resource = this.database.resources[resourceName];
|
|
13044
|
+
const handlers = this.eventHandlers.get(resourceName);
|
|
13045
|
+
if (resource && handlers) {
|
|
13046
|
+
resource.off("insert", handlers.insert);
|
|
13047
|
+
resource.off("update", handlers.update);
|
|
13048
|
+
resource.off("delete", handlers.delete);
|
|
13049
|
+
}
|
|
13050
|
+
}
|
|
13051
|
+
}
|
|
13052
|
+
this.replicators = [];
|
|
13053
|
+
this.database = null;
|
|
13054
|
+
this.eventListenersInstalled.clear();
|
|
13055
|
+
this.eventHandlers.clear();
|
|
13056
|
+
this.removeAllListeners();
|
|
13057
|
+
});
|
|
13058
|
+
if (!ok) {
|
|
13059
|
+
if (this.config.verbose) {
|
|
13060
|
+
console.warn(`[ReplicatorPlugin] Failed to cleanup plugin: ${error.message}`);
|
|
13061
|
+
}
|
|
13062
|
+
this.emit("replicator_plugin_cleanup_error", {
|
|
13063
|
+
error: error.message
|
|
13064
|
+
});
|
|
13065
|
+
}
|
|
13066
|
+
}
|
|
13067
|
+
}
|
|
13068
|
+
|
|
13069
|
+
class S3QueuePlugin extends Plugin {
|
|
13070
|
+
constructor(options = {}) {
|
|
13071
|
+
super(options);
|
|
13072
|
+
if (!options.resource) {
|
|
13073
|
+
throw new Error('S3QueuePlugin requires "resource" option');
|
|
13074
|
+
}
|
|
13075
|
+
this.config = {
|
|
13076
|
+
resource: options.resource,
|
|
13077
|
+
visibilityTimeout: options.visibilityTimeout || 3e4,
|
|
13078
|
+
// 30 seconds
|
|
13079
|
+
pollInterval: options.pollInterval || 1e3,
|
|
13080
|
+
// 1 second
|
|
13081
|
+
maxAttempts: options.maxAttempts || 3,
|
|
13082
|
+
concurrency: options.concurrency || 1,
|
|
13083
|
+
deadLetterResource: options.deadLetterResource || null,
|
|
13084
|
+
autoStart: options.autoStart !== false,
|
|
13085
|
+
onMessage: options.onMessage,
|
|
13086
|
+
onError: options.onError,
|
|
13087
|
+
onComplete: options.onComplete,
|
|
13088
|
+
verbose: options.verbose || false,
|
|
13089
|
+
...options
|
|
13090
|
+
};
|
|
13091
|
+
this.queueResource = null;
|
|
13092
|
+
this.targetResource = null;
|
|
13093
|
+
this.deadLetterResourceObj = null;
|
|
13094
|
+
this.workers = [];
|
|
13095
|
+
this.isRunning = false;
|
|
13096
|
+
this.workerId = `worker-${Date.now()}-${Math.random().toString(36).slice(2, 9)}`;
|
|
13097
|
+
this.processedCache = /* @__PURE__ */ new Map();
|
|
13098
|
+
this.cacheCleanupInterval = null;
|
|
13099
|
+
this.lockCleanupInterval = null;
|
|
13100
|
+
}
|
|
13101
|
+
async onSetup() {
|
|
13102
|
+
this.targetResource = this.database.resources[this.config.resource];
|
|
13103
|
+
if (!this.targetResource) {
|
|
13104
|
+
throw new Error(`S3QueuePlugin: resource '${this.config.resource}' not found`);
|
|
13105
|
+
}
|
|
13106
|
+
const queueName = `${this.config.resource}_queue`;
|
|
13107
|
+
const [ok, err] = await tryFn(
|
|
13108
|
+
() => this.database.createResource({
|
|
13109
|
+
name: queueName,
|
|
13110
|
+
attributes: {
|
|
13111
|
+
id: "string|required",
|
|
13112
|
+
originalId: "string|required",
|
|
13113
|
+
// ID do registro original
|
|
13114
|
+
status: "string|required",
|
|
13115
|
+
// pending/processing/completed/failed/dead
|
|
13116
|
+
visibleAt: "number|required",
|
|
13117
|
+
// Timestamp de visibilidade
|
|
13118
|
+
claimedBy: "string|optional",
|
|
13119
|
+
// Worker que claimed
|
|
13120
|
+
claimedAt: "number|optional",
|
|
13121
|
+
// Timestamp do claim
|
|
13122
|
+
attempts: "number|default:0",
|
|
13123
|
+
maxAttempts: "number|default:3",
|
|
13124
|
+
error: "string|optional",
|
|
13125
|
+
result: "json|optional",
|
|
13126
|
+
createdAt: "string|required",
|
|
13127
|
+
completedAt: "number|optional"
|
|
13128
|
+
},
|
|
13129
|
+
behavior: "body-overflow",
|
|
13130
|
+
timestamps: true,
|
|
13131
|
+
asyncPartitions: true,
|
|
13132
|
+
partitions: {
|
|
13133
|
+
byStatus: { fields: { status: "string" } },
|
|
13134
|
+
byDate: { fields: { createdAt: "string|maxlength:10" } }
|
|
13135
|
+
}
|
|
13136
|
+
})
|
|
13137
|
+
);
|
|
13138
|
+
if (!ok && !this.database.resources[queueName]) {
|
|
13139
|
+
throw new Error(`Failed to create queue resource: ${err?.message}`);
|
|
13140
|
+
}
|
|
13141
|
+
this.queueResource = this.database.resources[queueName];
|
|
13142
|
+
const lockName = `${this.config.resource}_locks`;
|
|
13143
|
+
const [okLock, errLock] = await tryFn(
|
|
13144
|
+
() => this.database.createResource({
|
|
13145
|
+
name: lockName,
|
|
13146
|
+
attributes: {
|
|
13147
|
+
id: "string|required",
|
|
13148
|
+
workerId: "string|required",
|
|
13149
|
+
timestamp: "number|required",
|
|
13150
|
+
ttl: "number|default:5000"
|
|
13151
|
+
},
|
|
13152
|
+
behavior: "body-overflow",
|
|
13153
|
+
timestamps: false
|
|
13154
|
+
})
|
|
13155
|
+
);
|
|
13156
|
+
if (okLock || this.database.resources[lockName]) {
|
|
13157
|
+
this.lockResource = this.database.resources[lockName];
|
|
13158
|
+
} else {
|
|
13159
|
+
this.lockResource = null;
|
|
13160
|
+
if (this.config.verbose) {
|
|
13161
|
+
console.log(`[S3QueuePlugin] Lock resource creation failed, locking disabled: ${errLock?.message}`);
|
|
13162
|
+
}
|
|
13163
|
+
}
|
|
13164
|
+
this.addHelperMethods();
|
|
13165
|
+
if (this.config.deadLetterResource) {
|
|
13166
|
+
await this.createDeadLetterResource();
|
|
13167
|
+
}
|
|
13168
|
+
if (this.config.verbose) {
|
|
13169
|
+
console.log(`[S3QueuePlugin] Setup completed for resource '${this.config.resource}'`);
|
|
13170
|
+
}
|
|
13171
|
+
}
|
|
13172
|
+
async onStart() {
|
|
13173
|
+
if (this.config.autoStart && this.config.onMessage) {
|
|
13174
|
+
await this.startProcessing();
|
|
13175
|
+
}
|
|
13176
|
+
}
|
|
13177
|
+
async onStop() {
|
|
13178
|
+
await this.stopProcessing();
|
|
13179
|
+
}
|
|
13180
|
+
addHelperMethods() {
|
|
13181
|
+
const plugin = this;
|
|
13182
|
+
const resource = this.targetResource;
|
|
13183
|
+
resource.enqueue = async function(data, options = {}) {
|
|
13184
|
+
const recordData = {
|
|
13185
|
+
id: data.id || idGenerator(),
|
|
13186
|
+
...data
|
|
13187
|
+
};
|
|
13188
|
+
const record = await resource.insert(recordData);
|
|
13189
|
+
const queueEntry = {
|
|
13190
|
+
id: idGenerator(),
|
|
13191
|
+
originalId: record.id,
|
|
13192
|
+
status: "pending",
|
|
13193
|
+
visibleAt: Date.now(),
|
|
13194
|
+
attempts: 0,
|
|
13195
|
+
maxAttempts: options.maxAttempts || plugin.config.maxAttempts,
|
|
13196
|
+
createdAt: (/* @__PURE__ */ new Date()).toISOString().slice(0, 10)
|
|
13197
|
+
};
|
|
13198
|
+
await plugin.queueResource.insert(queueEntry);
|
|
13199
|
+
plugin.emit("message.enqueued", { id: record.id, queueId: queueEntry.id });
|
|
13200
|
+
return record;
|
|
13201
|
+
};
|
|
13202
|
+
resource.queueStats = async function() {
|
|
13203
|
+
return await plugin.getStats();
|
|
13204
|
+
};
|
|
13205
|
+
resource.startProcessing = async function(handler, options = {}) {
|
|
13206
|
+
return await plugin.startProcessing(handler, options);
|
|
13207
|
+
};
|
|
13208
|
+
resource.stopProcessing = async function() {
|
|
13209
|
+
return await plugin.stopProcessing();
|
|
13210
|
+
};
|
|
13211
|
+
}
|
|
13212
|
+
async startProcessing(handler = null, options = {}) {
|
|
13213
|
+
if (this.isRunning) {
|
|
13214
|
+
if (this.config.verbose) {
|
|
13215
|
+
console.log("[S3QueuePlugin] Already running");
|
|
13216
|
+
}
|
|
13217
|
+
return;
|
|
13218
|
+
}
|
|
13219
|
+
const messageHandler = handler || this.config.onMessage;
|
|
13220
|
+
if (!messageHandler) {
|
|
13221
|
+
throw new Error("S3QueuePlugin: onMessage handler required");
|
|
13222
|
+
}
|
|
13223
|
+
this.isRunning = true;
|
|
13224
|
+
const concurrency = options.concurrency || this.config.concurrency;
|
|
13225
|
+
this.cacheCleanupInterval = setInterval(() => {
|
|
13226
|
+
const now = Date.now();
|
|
13227
|
+
const maxAge = 3e4;
|
|
13228
|
+
for (const [queueId, timestamp] of this.processedCache.entries()) {
|
|
13229
|
+
if (now - timestamp > maxAge) {
|
|
13230
|
+
this.processedCache.delete(queueId);
|
|
13231
|
+
}
|
|
13232
|
+
}
|
|
13233
|
+
}, 5e3);
|
|
13234
|
+
this.lockCleanupInterval = setInterval(() => {
|
|
13235
|
+
this.cleanupStaleLocks().catch((err) => {
|
|
13236
|
+
if (this.config.verbose) {
|
|
13237
|
+
console.log(`[lockCleanup] Error: ${err.message}`);
|
|
13238
|
+
}
|
|
13239
|
+
});
|
|
13240
|
+
}, 1e4);
|
|
13241
|
+
for (let i = 0; i < concurrency; i++) {
|
|
13242
|
+
const worker = this.createWorker(messageHandler, i);
|
|
13243
|
+
this.workers.push(worker);
|
|
13244
|
+
}
|
|
13245
|
+
if (this.config.verbose) {
|
|
13246
|
+
console.log(`[S3QueuePlugin] Started ${concurrency} workers`);
|
|
13247
|
+
}
|
|
13248
|
+
this.emit("workers.started", { concurrency, workerId: this.workerId });
|
|
13249
|
+
}
|
|
13250
|
+
async stopProcessing() {
|
|
13251
|
+
if (!this.isRunning) return;
|
|
13252
|
+
this.isRunning = false;
|
|
13253
|
+
if (this.cacheCleanupInterval) {
|
|
13254
|
+
clearInterval(this.cacheCleanupInterval);
|
|
13255
|
+
this.cacheCleanupInterval = null;
|
|
13256
|
+
}
|
|
13257
|
+
if (this.lockCleanupInterval) {
|
|
13258
|
+
clearInterval(this.lockCleanupInterval);
|
|
13259
|
+
this.lockCleanupInterval = null;
|
|
13260
|
+
}
|
|
13261
|
+
await Promise.all(this.workers);
|
|
13262
|
+
this.workers = [];
|
|
13263
|
+
this.processedCache.clear();
|
|
13264
|
+
if (this.config.verbose) {
|
|
13265
|
+
console.log("[S3QueuePlugin] Stopped all workers");
|
|
13266
|
+
}
|
|
13267
|
+
this.emit("workers.stopped", { workerId: this.workerId });
|
|
13268
|
+
}
|
|
13269
|
+
createWorker(handler, workerIndex) {
|
|
13270
|
+
return (async () => {
|
|
13271
|
+
while (this.isRunning) {
|
|
13272
|
+
try {
|
|
13273
|
+
const message = await this.claimMessage();
|
|
13274
|
+
if (message) {
|
|
13275
|
+
await this.processMessage(message, handler);
|
|
13276
|
+
} else {
|
|
13277
|
+
await new Promise((resolve) => setTimeout(resolve, this.config.pollInterval));
|
|
13278
|
+
}
|
|
13279
|
+
} catch (error) {
|
|
13280
|
+
if (this.config.verbose) {
|
|
13281
|
+
console.error(`[Worker ${workerIndex}] Error:`, error.message);
|
|
13282
|
+
}
|
|
13283
|
+
await new Promise((resolve) => setTimeout(resolve, 1e3));
|
|
13284
|
+
}
|
|
13285
|
+
}
|
|
13286
|
+
})();
|
|
13287
|
+
}
|
|
13288
|
+
async claimMessage() {
|
|
13289
|
+
const now = Date.now();
|
|
13290
|
+
const [ok, err, messages] = await tryFn(
|
|
13291
|
+
() => this.queueResource.query({
|
|
13292
|
+
status: "pending"
|
|
13293
|
+
})
|
|
13294
|
+
);
|
|
13295
|
+
if (!ok || !messages || messages.length === 0) {
|
|
13296
|
+
return null;
|
|
13297
|
+
}
|
|
13298
|
+
const available = messages.filter((m) => m.visibleAt <= now);
|
|
13299
|
+
if (available.length === 0) {
|
|
13300
|
+
return null;
|
|
13301
|
+
}
|
|
13302
|
+
for (const msg of available) {
|
|
13303
|
+
const claimed = await this.attemptClaim(msg);
|
|
13304
|
+
if (claimed) {
|
|
13305
|
+
return claimed;
|
|
13306
|
+
}
|
|
13307
|
+
}
|
|
13308
|
+
return null;
|
|
13309
|
+
}
|
|
13310
|
+
/**
|
|
13311
|
+
* Acquire a distributed lock using ETag-based conditional updates
|
|
13312
|
+
* This ensures only one worker can claim a message at a time
|
|
13313
|
+
*
|
|
13314
|
+
* Uses a two-step process:
|
|
13315
|
+
* 1. Create lock resource (similar to queue resource) if not exists
|
|
13316
|
+
* 2. Try to claim lock using ETag-based conditional update
|
|
13317
|
+
*/
|
|
13318
|
+
async acquireLock(messageId) {
|
|
13319
|
+
if (!this.lockResource) {
|
|
13320
|
+
return true;
|
|
13321
|
+
}
|
|
13322
|
+
const lockId = `lock-${messageId}`;
|
|
13323
|
+
const now = Date.now();
|
|
13324
|
+
try {
|
|
13325
|
+
const [okGet, errGet, existingLock] = await tryFn(
|
|
13326
|
+
() => this.lockResource.get(lockId)
|
|
13327
|
+
);
|
|
13328
|
+
if (existingLock) {
|
|
13329
|
+
const lockAge = now - existingLock.timestamp;
|
|
13330
|
+
if (lockAge < existingLock.ttl) {
|
|
13331
|
+
return false;
|
|
13332
|
+
}
|
|
13333
|
+
const [ok, err, result] = await tryFn(
|
|
13334
|
+
() => this.lockResource.updateConditional(lockId, {
|
|
13335
|
+
workerId: this.workerId,
|
|
13336
|
+
timestamp: now,
|
|
13337
|
+
ttl: 5e3
|
|
13338
|
+
}, {
|
|
13339
|
+
ifMatch: existingLock._etag
|
|
13340
|
+
})
|
|
13341
|
+
);
|
|
13342
|
+
return ok && result.success;
|
|
13343
|
+
}
|
|
13344
|
+
const [okCreate, errCreate] = await tryFn(
|
|
13345
|
+
() => this.lockResource.insert({
|
|
13346
|
+
id: lockId,
|
|
13347
|
+
workerId: this.workerId,
|
|
13348
|
+
timestamp: now,
|
|
13349
|
+
ttl: 5e3
|
|
13350
|
+
})
|
|
13351
|
+
);
|
|
13352
|
+
return okCreate;
|
|
13353
|
+
} catch (error) {
|
|
13354
|
+
if (this.config.verbose) {
|
|
13355
|
+
console.log(`[acquireLock] Error: ${error.message}`);
|
|
13356
|
+
}
|
|
13357
|
+
return false;
|
|
13358
|
+
}
|
|
13359
|
+
}
|
|
13360
|
+
/**
|
|
13361
|
+
* Release a distributed lock by deleting the lock record
|
|
13362
|
+
*/
|
|
13363
|
+
async releaseLock(messageId) {
|
|
13364
|
+
if (!this.lockResource) {
|
|
13365
|
+
return;
|
|
13366
|
+
}
|
|
13367
|
+
const lockId = `lock-${messageId}`;
|
|
13368
|
+
try {
|
|
13369
|
+
await this.lockResource.delete(lockId);
|
|
13370
|
+
} catch (error) {
|
|
13371
|
+
if (this.config.verbose) {
|
|
13372
|
+
console.log(`[releaseLock] Failed to release lock for ${messageId}: ${error.message}`);
|
|
13373
|
+
}
|
|
13374
|
+
}
|
|
13375
|
+
}
|
|
13376
|
+
/**
|
|
13377
|
+
* Clean up stale locks (older than TTL)
|
|
13378
|
+
* This prevents deadlocks if a worker crashes while holding a lock
|
|
13379
|
+
*/
|
|
13380
|
+
async cleanupStaleLocks() {
|
|
13381
|
+
if (!this.lockResource) {
|
|
13382
|
+
return;
|
|
13383
|
+
}
|
|
13384
|
+
const now = Date.now();
|
|
13385
|
+
try {
|
|
13386
|
+
const locks = await this.lockResource.list();
|
|
13387
|
+
for (const lock of locks) {
|
|
13388
|
+
const lockAge = now - lock.timestamp;
|
|
13389
|
+
if (lockAge > lock.ttl) {
|
|
13390
|
+
await this.lockResource.delete(lock.id);
|
|
13391
|
+
if (this.config.verbose) {
|
|
13392
|
+
console.log(`[cleanupStaleLocks] Removed expired lock: ${lock.id}`);
|
|
13393
|
+
}
|
|
13394
|
+
}
|
|
13395
|
+
}
|
|
13396
|
+
} catch (error) {
|
|
13397
|
+
if (this.config.verbose) {
|
|
13398
|
+
console.log(`[cleanupStaleLocks] Error during cleanup: ${error.message}`);
|
|
13399
|
+
}
|
|
13400
|
+
}
|
|
13401
|
+
}
|
|
13402
|
+
async attemptClaim(msg) {
|
|
13403
|
+
const now = Date.now();
|
|
13404
|
+
const lockAcquired = await this.acquireLock(msg.id);
|
|
13405
|
+
if (!lockAcquired) {
|
|
13406
|
+
return null;
|
|
13407
|
+
}
|
|
13408
|
+
if (this.processedCache.has(msg.id)) {
|
|
13409
|
+
await this.releaseLock(msg.id);
|
|
13410
|
+
if (this.config.verbose) {
|
|
13411
|
+
console.log(`[attemptClaim] Message ${msg.id} already processed (in cache)`);
|
|
13412
|
+
}
|
|
13413
|
+
return null;
|
|
13414
|
+
}
|
|
13415
|
+
this.processedCache.set(msg.id, Date.now());
|
|
13416
|
+
await this.releaseLock(msg.id);
|
|
13417
|
+
const [okGet, errGet, msgWithETag] = await tryFn(
|
|
13418
|
+
() => this.queueResource.get(msg.id)
|
|
13419
|
+
);
|
|
13420
|
+
if (!okGet || !msgWithETag) {
|
|
13421
|
+
this.processedCache.delete(msg.id);
|
|
13422
|
+
if (this.config.verbose) {
|
|
13423
|
+
console.log(`[attemptClaim] Message ${msg.id} not found or error: ${errGet?.message}`);
|
|
13424
|
+
}
|
|
13425
|
+
return null;
|
|
13426
|
+
}
|
|
13427
|
+
if (msgWithETag.status !== "pending" || msgWithETag.visibleAt > now) {
|
|
13428
|
+
this.processedCache.delete(msg.id);
|
|
13429
|
+
if (this.config.verbose) {
|
|
13430
|
+
console.log(`[attemptClaim] Message ${msg.id} not claimable: status=${msgWithETag.status}, visibleAt=${msgWithETag.visibleAt}, now=${now}`);
|
|
13431
|
+
}
|
|
13432
|
+
return null;
|
|
13433
|
+
}
|
|
13434
|
+
if (this.config.verbose) {
|
|
13435
|
+
console.log(`[attemptClaim] Attempting to claim ${msg.id} with ETag: ${msgWithETag._etag}`);
|
|
13436
|
+
}
|
|
13437
|
+
const [ok, err, result] = await tryFn(
|
|
13438
|
+
() => this.queueResource.updateConditional(msgWithETag.id, {
|
|
13439
|
+
status: "processing",
|
|
13440
|
+
claimedBy: this.workerId,
|
|
13441
|
+
claimedAt: now,
|
|
13442
|
+
visibleAt: now + this.config.visibilityTimeout,
|
|
13443
|
+
attempts: msgWithETag.attempts + 1
|
|
13444
|
+
}, {
|
|
13445
|
+
ifMatch: msgWithETag._etag
|
|
13446
|
+
// ← ATOMIC CLAIM using ETag!
|
|
13447
|
+
})
|
|
13448
|
+
);
|
|
13449
|
+
if (!ok || !result.success) {
|
|
13450
|
+
this.processedCache.delete(msg.id);
|
|
13451
|
+
if (this.config.verbose) {
|
|
13452
|
+
console.log(`[attemptClaim] Failed to claim ${msg.id}: ${err?.message || result.error}`);
|
|
13453
|
+
}
|
|
13454
|
+
return null;
|
|
11784
13455
|
}
|
|
11785
|
-
if (
|
|
11786
|
-
|
|
13456
|
+
if (this.config.verbose) {
|
|
13457
|
+
console.log(`[attemptClaim] Successfully claimed ${msg.id}`);
|
|
11787
13458
|
}
|
|
11788
|
-
|
|
11789
|
-
|
|
13459
|
+
const [okRecord, errRecord, record] = await tryFn(
|
|
13460
|
+
() => this.targetResource.get(msgWithETag.originalId)
|
|
13461
|
+
);
|
|
13462
|
+
if (!okRecord) {
|
|
13463
|
+
await this.failMessage(msgWithETag.id, "Original record not found");
|
|
13464
|
+
return null;
|
|
11790
13465
|
}
|
|
11791
|
-
|
|
11792
|
-
|
|
13466
|
+
return {
|
|
13467
|
+
queueId: msgWithETag.id,
|
|
13468
|
+
record,
|
|
13469
|
+
attempts: msgWithETag.attempts + 1,
|
|
13470
|
+
maxAttempts: msgWithETag.maxAttempts
|
|
13471
|
+
};
|
|
11793
13472
|
}
|
|
11794
|
-
async
|
|
11795
|
-
|
|
11796
|
-
|
|
13473
|
+
async processMessage(message, handler) {
|
|
13474
|
+
const startTime = Date.now();
|
|
13475
|
+
try {
|
|
13476
|
+
const result = await handler(message.record, {
|
|
13477
|
+
queueId: message.queueId,
|
|
13478
|
+
attempts: message.attempts,
|
|
13479
|
+
workerId: this.workerId
|
|
13480
|
+
});
|
|
13481
|
+
await this.completeMessage(message.queueId, result);
|
|
13482
|
+
const duration = Date.now() - startTime;
|
|
13483
|
+
this.emit("message.completed", {
|
|
13484
|
+
queueId: message.queueId,
|
|
13485
|
+
originalId: message.record.id,
|
|
13486
|
+
duration,
|
|
13487
|
+
attempts: message.attempts
|
|
13488
|
+
});
|
|
13489
|
+
if (this.config.onComplete) {
|
|
13490
|
+
await this.config.onComplete(message.record, result);
|
|
13491
|
+
}
|
|
13492
|
+
} catch (error) {
|
|
13493
|
+
const shouldRetry = message.attempts < message.maxAttempts;
|
|
13494
|
+
if (shouldRetry) {
|
|
13495
|
+
await this.retryMessage(message.queueId, message.attempts, error.message);
|
|
13496
|
+
this.emit("message.retry", {
|
|
13497
|
+
queueId: message.queueId,
|
|
13498
|
+
originalId: message.record.id,
|
|
13499
|
+
attempts: message.attempts,
|
|
13500
|
+
error: error.message
|
|
13501
|
+
});
|
|
13502
|
+
} else {
|
|
13503
|
+
await this.moveToDeadLetter(message.queueId, message.record, error.message);
|
|
13504
|
+
this.emit("message.dead", {
|
|
13505
|
+
queueId: message.queueId,
|
|
13506
|
+
originalId: message.record.id,
|
|
13507
|
+
error: error.message
|
|
13508
|
+
});
|
|
13509
|
+
}
|
|
13510
|
+
if (this.config.onError) {
|
|
13511
|
+
await this.config.onError(error, message.record);
|
|
13512
|
+
}
|
|
11797
13513
|
}
|
|
11798
|
-
|
|
11799
|
-
|
|
13514
|
+
}
|
|
13515
|
+
async completeMessage(queueId, result) {
|
|
13516
|
+
await this.queueResource.update(queueId, {
|
|
13517
|
+
status: "completed",
|
|
13518
|
+
completedAt: Date.now(),
|
|
13519
|
+
result
|
|
11800
13520
|
});
|
|
11801
|
-
|
|
11802
|
-
|
|
11803
|
-
|
|
11804
|
-
|
|
11805
|
-
|
|
11806
|
-
|
|
11807
|
-
|
|
11808
|
-
|
|
11809
|
-
|
|
13521
|
+
}
|
|
13522
|
+
async failMessage(queueId, error) {
|
|
13523
|
+
await this.queueResource.update(queueId, {
|
|
13524
|
+
status: "failed",
|
|
13525
|
+
error
|
|
13526
|
+
});
|
|
13527
|
+
}
|
|
13528
|
+
async retryMessage(queueId, attempts, error) {
|
|
13529
|
+
const backoff = Math.min(Math.pow(2, attempts) * 1e3, 3e4);
|
|
13530
|
+
await this.queueResource.update(queueId, {
|
|
13531
|
+
status: "pending",
|
|
13532
|
+
visibleAt: Date.now() + backoff,
|
|
13533
|
+
error
|
|
13534
|
+
});
|
|
13535
|
+
this.processedCache.delete(queueId);
|
|
13536
|
+
}
|
|
13537
|
+
async moveToDeadLetter(queueId, record, error) {
|
|
13538
|
+
if (this.config.deadLetterResource && this.deadLetterResourceObj) {
|
|
13539
|
+
const msg = await this.queueResource.get(queueId);
|
|
13540
|
+
await this.deadLetterResourceObj.insert({
|
|
13541
|
+
id: idGenerator(),
|
|
13542
|
+
originalId: record.id,
|
|
13543
|
+
queueId,
|
|
13544
|
+
data: record,
|
|
13545
|
+
error,
|
|
13546
|
+
attempts: msg.attempts,
|
|
13547
|
+
createdAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
11810
13548
|
});
|
|
11811
|
-
if (ok) {
|
|
11812
|
-
retried++;
|
|
11813
|
-
}
|
|
11814
13549
|
}
|
|
11815
|
-
|
|
13550
|
+
await this.queueResource.update(queueId, {
|
|
13551
|
+
status: "dead",
|
|
13552
|
+
error
|
|
13553
|
+
});
|
|
11816
13554
|
}
|
|
11817
|
-
async
|
|
11818
|
-
const
|
|
11819
|
-
|
|
11820
|
-
|
|
13555
|
+
async getStats() {
|
|
13556
|
+
const [ok, err, allMessages] = await tryFn(
|
|
13557
|
+
() => this.queueResource.list()
|
|
13558
|
+
);
|
|
13559
|
+
if (!ok) {
|
|
13560
|
+
if (this.config.verbose) {
|
|
13561
|
+
console.warn("[S3QueuePlugin] Failed to get stats:", err.message);
|
|
13562
|
+
}
|
|
13563
|
+
return null;
|
|
11821
13564
|
}
|
|
11822
|
-
|
|
11823
|
-
|
|
11824
|
-
|
|
11825
|
-
|
|
11826
|
-
|
|
11827
|
-
|
|
11828
|
-
|
|
11829
|
-
|
|
11830
|
-
|
|
11831
|
-
|
|
13565
|
+
const stats = {
|
|
13566
|
+
total: allMessages.length,
|
|
13567
|
+
pending: 0,
|
|
13568
|
+
processing: 0,
|
|
13569
|
+
completed: 0,
|
|
13570
|
+
failed: 0,
|
|
13571
|
+
dead: 0
|
|
13572
|
+
};
|
|
13573
|
+
for (const msg of allMessages) {
|
|
13574
|
+
if (stats[msg.status] !== void 0) {
|
|
13575
|
+
stats[msg.status]++;
|
|
11832
13576
|
}
|
|
11833
13577
|
}
|
|
11834
|
-
|
|
13578
|
+
return stats;
|
|
11835
13579
|
}
|
|
11836
|
-
async
|
|
11837
|
-
const [ok,
|
|
11838
|
-
|
|
11839
|
-
|
|
11840
|
-
|
|
11841
|
-
|
|
11842
|
-
|
|
11843
|
-
|
|
11844
|
-
|
|
11845
|
-
|
|
11846
|
-
|
|
11847
|
-
|
|
11848
|
-
|
|
11849
|
-
|
|
11850
|
-
|
|
11851
|
-
|
|
11852
|
-
|
|
11853
|
-
|
|
11854
|
-
|
|
11855
|
-
});
|
|
11856
|
-
await Promise.allSettled(cleanupPromises);
|
|
11857
|
-
}
|
|
11858
|
-
this.replicators = [];
|
|
11859
|
-
this.database = null;
|
|
11860
|
-
this.eventListenersInstalled.clear();
|
|
11861
|
-
this.removeAllListeners();
|
|
11862
|
-
});
|
|
11863
|
-
if (!ok) {
|
|
13580
|
+
async createDeadLetterResource() {
|
|
13581
|
+
const [ok, err] = await tryFn(
|
|
13582
|
+
() => this.database.createResource({
|
|
13583
|
+
name: this.config.deadLetterResource,
|
|
13584
|
+
attributes: {
|
|
13585
|
+
id: "string|required",
|
|
13586
|
+
originalId: "string|required",
|
|
13587
|
+
queueId: "string|required",
|
|
13588
|
+
data: "json|required",
|
|
13589
|
+
error: "string|required",
|
|
13590
|
+
attempts: "number|required",
|
|
13591
|
+
createdAt: "string|required"
|
|
13592
|
+
},
|
|
13593
|
+
behavior: "body-overflow",
|
|
13594
|
+
timestamps: true
|
|
13595
|
+
})
|
|
13596
|
+
);
|
|
13597
|
+
if (ok || this.database.resources[this.config.deadLetterResource]) {
|
|
13598
|
+
this.deadLetterResourceObj = this.database.resources[this.config.deadLetterResource];
|
|
11864
13599
|
if (this.config.verbose) {
|
|
11865
|
-
console.
|
|
13600
|
+
console.log(`[S3QueuePlugin] Dead letter queue created: ${this.config.deadLetterResource}`);
|
|
11866
13601
|
}
|
|
11867
|
-
this.emit("replicator_plugin_cleanup_error", {
|
|
11868
|
-
error: error.message
|
|
11869
|
-
});
|
|
11870
13602
|
}
|
|
11871
13603
|
}
|
|
11872
13604
|
}
|
|
@@ -11880,7 +13612,7 @@ class SchedulerPlugin extends Plugin {
|
|
|
11880
13612
|
defaultTimeout: options.defaultTimeout || 3e5,
|
|
11881
13613
|
// 5 minutes
|
|
11882
13614
|
defaultRetries: options.defaultRetries || 1,
|
|
11883
|
-
jobHistoryResource: options.jobHistoryResource || "
|
|
13615
|
+
jobHistoryResource: options.jobHistoryResource || "plg_job_executions",
|
|
11884
13616
|
persistJobs: options.persistJobs !== false,
|
|
11885
13617
|
verbose: options.verbose || false,
|
|
11886
13618
|
onJobStart: options.onJobStart || null,
|
|
@@ -11889,12 +13621,20 @@ class SchedulerPlugin extends Plugin {
|
|
|
11889
13621
|
...options
|
|
11890
13622
|
};
|
|
11891
13623
|
this.database = null;
|
|
13624
|
+
this.lockResource = null;
|
|
11892
13625
|
this.jobs = /* @__PURE__ */ new Map();
|
|
11893
13626
|
this.activeJobs = /* @__PURE__ */ new Map();
|
|
11894
13627
|
this.timers = /* @__PURE__ */ new Map();
|
|
11895
13628
|
this.statistics = /* @__PURE__ */ new Map();
|
|
11896
13629
|
this._validateConfiguration();
|
|
11897
13630
|
}
|
|
13631
|
+
/**
|
|
13632
|
+
* Helper to detect test environment
|
|
13633
|
+
* @private
|
|
13634
|
+
*/
|
|
13635
|
+
_isTestEnvironment() {
|
|
13636
|
+
return process.env.NODE_ENV === "test" || process.env.JEST_WORKER_ID !== void 0 || global.expect !== void 0;
|
|
13637
|
+
}
|
|
11898
13638
|
_validateConfiguration() {
|
|
11899
13639
|
if (Object.keys(this.config.jobs).length === 0) {
|
|
11900
13640
|
throw new Error("SchedulerPlugin: At least one job must be defined");
|
|
@@ -11921,6 +13661,7 @@ class SchedulerPlugin extends Plugin {
|
|
|
11921
13661
|
}
|
|
11922
13662
|
async setup(database) {
|
|
11923
13663
|
this.database = database;
|
|
13664
|
+
await this._createLockResource();
|
|
11924
13665
|
if (this.config.persistJobs) {
|
|
11925
13666
|
await this._createJobHistoryResource();
|
|
11926
13667
|
}
|
|
@@ -11949,6 +13690,25 @@ class SchedulerPlugin extends Plugin {
|
|
|
11949
13690
|
await this._startScheduling();
|
|
11950
13691
|
this.emit("initialized", { jobs: this.jobs.size });
|
|
11951
13692
|
}
|
|
13693
|
+
async _createLockResource() {
|
|
13694
|
+
const [ok, err, lockResource] = await tryFn(
|
|
13695
|
+
() => this.database.createResource({
|
|
13696
|
+
name: "plg_scheduler_job_locks",
|
|
13697
|
+
attributes: {
|
|
13698
|
+
id: "string|required",
|
|
13699
|
+
jobName: "string|required",
|
|
13700
|
+
lockedAt: "number|required",
|
|
13701
|
+
instanceId: "string|optional"
|
|
13702
|
+
},
|
|
13703
|
+
behavior: "body-only",
|
|
13704
|
+
timestamps: false
|
|
13705
|
+
})
|
|
13706
|
+
);
|
|
13707
|
+
if (!ok && !this.database.resources.plg_scheduler_job_locks) {
|
|
13708
|
+
throw new Error(`Failed to create lock resource: ${err?.message}`);
|
|
13709
|
+
}
|
|
13710
|
+
this.lockResource = ok ? lockResource : this.database.resources.plg_scheduler_job_locks;
|
|
13711
|
+
}
|
|
11952
13712
|
async _createJobHistoryResource() {
|
|
11953
13713
|
const [ok] = await tryFn(() => this.database.createResource({
|
|
11954
13714
|
name: this.config.jobHistoryResource,
|
|
@@ -12042,18 +13802,37 @@ class SchedulerPlugin extends Plugin {
|
|
|
12042
13802
|
next.setHours(next.getHours() + 1);
|
|
12043
13803
|
}
|
|
12044
13804
|
}
|
|
12045
|
-
|
|
12046
|
-
if (isTestEnvironment) {
|
|
13805
|
+
if (this._isTestEnvironment()) {
|
|
12047
13806
|
next.setTime(next.getTime() + 1e3);
|
|
12048
13807
|
}
|
|
12049
13808
|
return next;
|
|
12050
13809
|
}
|
|
12051
13810
|
async _executeJob(jobName) {
|
|
12052
13811
|
const job = this.jobs.get(jobName);
|
|
12053
|
-
if (!job
|
|
13812
|
+
if (!job) {
|
|
13813
|
+
return;
|
|
13814
|
+
}
|
|
13815
|
+
if (this.activeJobs.has(jobName)) {
|
|
13816
|
+
return;
|
|
13817
|
+
}
|
|
13818
|
+
this.activeJobs.set(jobName, "acquiring-lock");
|
|
13819
|
+
const lockId = `lock-${jobName}`;
|
|
13820
|
+
const [lockAcquired, lockErr] = await tryFn(
|
|
13821
|
+
() => this.lockResource.insert({
|
|
13822
|
+
id: lockId,
|
|
13823
|
+
jobName,
|
|
13824
|
+
lockedAt: Date.now(),
|
|
13825
|
+
instanceId: process.pid ? String(process.pid) : "unknown"
|
|
13826
|
+
})
|
|
13827
|
+
);
|
|
13828
|
+
if (!lockAcquired) {
|
|
13829
|
+
if (this.config.verbose) {
|
|
13830
|
+
console.log(`[SchedulerPlugin] Job '${jobName}' already running on another instance`);
|
|
13831
|
+
}
|
|
13832
|
+
this.activeJobs.delete(jobName);
|
|
12054
13833
|
return;
|
|
12055
13834
|
}
|
|
12056
|
-
const executionId = `${jobName}_${
|
|
13835
|
+
const executionId = `${jobName}_${idGenerator()}`;
|
|
12057
13836
|
const startTime = Date.now();
|
|
12058
13837
|
const context = {
|
|
12059
13838
|
jobName,
|
|
@@ -12062,91 +13841,95 @@ class SchedulerPlugin extends Plugin {
|
|
|
12062
13841
|
database: this.database
|
|
12063
13842
|
};
|
|
12064
13843
|
this.activeJobs.set(jobName, executionId);
|
|
12065
|
-
|
|
12066
|
-
|
|
12067
|
-
|
|
12068
|
-
|
|
12069
|
-
|
|
12070
|
-
|
|
12071
|
-
|
|
12072
|
-
|
|
12073
|
-
|
|
12074
|
-
|
|
12075
|
-
|
|
12076
|
-
const actualTimeout = isTestEnvironment ? Math.min(job.timeout, 1e3) : job.timeout;
|
|
12077
|
-
let timeoutId;
|
|
12078
|
-
const timeoutPromise = new Promise((_, reject) => {
|
|
12079
|
-
timeoutId = setTimeout(() => reject(new Error("Job execution timeout")), actualTimeout);
|
|
12080
|
-
});
|
|
12081
|
-
const jobPromise = job.action(this.database, context, this);
|
|
13844
|
+
try {
|
|
13845
|
+
if (this.config.onJobStart) {
|
|
13846
|
+
await this._executeHook(this.config.onJobStart, jobName, context);
|
|
13847
|
+
}
|
|
13848
|
+
this.emit("job_start", { jobName, executionId, startTime });
|
|
13849
|
+
let attempt = 0;
|
|
13850
|
+
let lastError = null;
|
|
13851
|
+
let result = null;
|
|
13852
|
+
let status = "success";
|
|
13853
|
+
const isTestEnvironment = this._isTestEnvironment();
|
|
13854
|
+
while (attempt <= job.retries) {
|
|
12082
13855
|
try {
|
|
12083
|
-
|
|
12084
|
-
|
|
12085
|
-
|
|
12086
|
-
|
|
12087
|
-
|
|
12088
|
-
|
|
12089
|
-
|
|
12090
|
-
|
|
12091
|
-
|
|
12092
|
-
|
|
12093
|
-
|
|
12094
|
-
|
|
12095
|
-
|
|
12096
|
-
|
|
13856
|
+
const actualTimeout = isTestEnvironment ? Math.min(job.timeout, 1e3) : job.timeout;
|
|
13857
|
+
let timeoutId;
|
|
13858
|
+
const timeoutPromise = new Promise((_, reject) => {
|
|
13859
|
+
timeoutId = setTimeout(() => reject(new Error("Job execution timeout")), actualTimeout);
|
|
13860
|
+
});
|
|
13861
|
+
const jobPromise = job.action(this.database, context, this);
|
|
13862
|
+
try {
|
|
13863
|
+
result = await Promise.race([jobPromise, timeoutPromise]);
|
|
13864
|
+
clearTimeout(timeoutId);
|
|
13865
|
+
} catch (raceError) {
|
|
13866
|
+
clearTimeout(timeoutId);
|
|
13867
|
+
throw raceError;
|
|
13868
|
+
}
|
|
13869
|
+
status = "success";
|
|
13870
|
+
break;
|
|
13871
|
+
} catch (error) {
|
|
13872
|
+
lastError = error;
|
|
13873
|
+
attempt++;
|
|
13874
|
+
if (attempt <= job.retries) {
|
|
13875
|
+
if (this.config.verbose) {
|
|
13876
|
+
console.warn(`[SchedulerPlugin] Job '${jobName}' failed (attempt ${attempt + 1}):`, error.message);
|
|
13877
|
+
}
|
|
13878
|
+
const baseDelay = Math.min(Math.pow(2, attempt) * 1e3, 5e3);
|
|
13879
|
+
const delay = isTestEnvironment ? 1 : baseDelay;
|
|
13880
|
+
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
12097
13881
|
}
|
|
12098
|
-
const baseDelay = Math.min(Math.pow(2, attempt) * 1e3, 5e3);
|
|
12099
|
-
const delay = isTestEnvironment ? 1 : baseDelay;
|
|
12100
|
-
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
12101
13882
|
}
|
|
12102
13883
|
}
|
|
12103
|
-
|
|
12104
|
-
|
|
12105
|
-
|
|
12106
|
-
|
|
12107
|
-
|
|
12108
|
-
|
|
12109
|
-
|
|
12110
|
-
|
|
12111
|
-
|
|
12112
|
-
|
|
12113
|
-
|
|
12114
|
-
|
|
12115
|
-
|
|
12116
|
-
|
|
12117
|
-
|
|
12118
|
-
|
|
12119
|
-
|
|
12120
|
-
|
|
12121
|
-
|
|
12122
|
-
|
|
12123
|
-
|
|
12124
|
-
|
|
12125
|
-
|
|
12126
|
-
|
|
12127
|
-
|
|
12128
|
-
|
|
12129
|
-
|
|
12130
|
-
|
|
12131
|
-
|
|
12132
|
-
|
|
12133
|
-
|
|
12134
|
-
|
|
12135
|
-
|
|
12136
|
-
|
|
12137
|
-
|
|
12138
|
-
|
|
12139
|
-
|
|
12140
|
-
|
|
12141
|
-
|
|
12142
|
-
|
|
12143
|
-
|
|
12144
|
-
|
|
12145
|
-
|
|
12146
|
-
|
|
12147
|
-
|
|
12148
|
-
|
|
12149
|
-
|
|
13884
|
+
const endTime = Date.now();
|
|
13885
|
+
const duration = Math.max(1, endTime - startTime);
|
|
13886
|
+
if (lastError && attempt > job.retries) {
|
|
13887
|
+
status = lastError.message.includes("timeout") ? "timeout" : "error";
|
|
13888
|
+
}
|
|
13889
|
+
job.lastRun = new Date(endTime);
|
|
13890
|
+
job.runCount++;
|
|
13891
|
+
if (status === "success") {
|
|
13892
|
+
job.successCount++;
|
|
13893
|
+
} else {
|
|
13894
|
+
job.errorCount++;
|
|
13895
|
+
}
|
|
13896
|
+
const stats = this.statistics.get(jobName);
|
|
13897
|
+
stats.totalRuns++;
|
|
13898
|
+
stats.lastRun = new Date(endTime);
|
|
13899
|
+
if (status === "success") {
|
|
13900
|
+
stats.totalSuccesses++;
|
|
13901
|
+
stats.lastSuccess = new Date(endTime);
|
|
13902
|
+
} else {
|
|
13903
|
+
stats.totalErrors++;
|
|
13904
|
+
stats.lastError = { time: new Date(endTime), message: lastError?.message };
|
|
13905
|
+
}
|
|
13906
|
+
stats.avgDuration = (stats.avgDuration * (stats.totalRuns - 1) + duration) / stats.totalRuns;
|
|
13907
|
+
if (this.config.persistJobs) {
|
|
13908
|
+
await this._persistJobExecution(jobName, executionId, startTime, endTime, duration, status, result, lastError, attempt);
|
|
13909
|
+
}
|
|
13910
|
+
if (status === "success" && this.config.onJobComplete) {
|
|
13911
|
+
await this._executeHook(this.config.onJobComplete, jobName, result, duration);
|
|
13912
|
+
} else if (status !== "success" && this.config.onJobError) {
|
|
13913
|
+
await this._executeHook(this.config.onJobError, jobName, lastError, attempt);
|
|
13914
|
+
}
|
|
13915
|
+
this.emit("job_complete", {
|
|
13916
|
+
jobName,
|
|
13917
|
+
executionId,
|
|
13918
|
+
status,
|
|
13919
|
+
duration,
|
|
13920
|
+
result,
|
|
13921
|
+
error: lastError?.message,
|
|
13922
|
+
retryCount: attempt
|
|
13923
|
+
});
|
|
13924
|
+
this.activeJobs.delete(jobName);
|
|
13925
|
+
if (job.enabled) {
|
|
13926
|
+
this._scheduleNextExecution(jobName);
|
|
13927
|
+
}
|
|
13928
|
+
if (lastError && status !== "success") {
|
|
13929
|
+
throw lastError;
|
|
13930
|
+
}
|
|
13931
|
+
} finally {
|
|
13932
|
+
await tryFn(() => this.lockResource.delete(lockId));
|
|
12150
13933
|
}
|
|
12151
13934
|
}
|
|
12152
13935
|
async _persistJobExecution(jobName, executionId, startTime, endTime, duration, status, result, error, retryCount) {
|
|
@@ -12178,6 +13961,7 @@ class SchedulerPlugin extends Plugin {
|
|
|
12178
13961
|
}
|
|
12179
13962
|
/**
|
|
12180
13963
|
* Manually trigger a job execution
|
|
13964
|
+
* Note: Race conditions are prevented by distributed locking in _executeJob()
|
|
12181
13965
|
*/
|
|
12182
13966
|
async runJob(jobName, context = {}) {
|
|
12183
13967
|
const job = this.jobs.get(jobName);
|
|
@@ -12263,12 +14047,15 @@ class SchedulerPlugin extends Plugin {
|
|
|
12263
14047
|
return [];
|
|
12264
14048
|
}
|
|
12265
14049
|
const { limit = 50, status = null } = options;
|
|
12266
|
-
const
|
|
12267
|
-
|
|
12268
|
-
|
|
12269
|
-
|
|
12270
|
-
|
|
12271
|
-
|
|
14050
|
+
const queryParams = {
|
|
14051
|
+
jobName
|
|
14052
|
+
// Uses byJob partition for efficient lookup
|
|
14053
|
+
};
|
|
14054
|
+
if (status) {
|
|
14055
|
+
queryParams.status = status;
|
|
14056
|
+
}
|
|
14057
|
+
const [ok, err, history] = await tryFn(
|
|
14058
|
+
() => this.database.resource(this.config.jobHistoryResource).query(queryParams)
|
|
12272
14059
|
);
|
|
12273
14060
|
if (!ok) {
|
|
12274
14061
|
if (this.config.verbose) {
|
|
@@ -12276,11 +14063,7 @@ class SchedulerPlugin extends Plugin {
|
|
|
12276
14063
|
}
|
|
12277
14064
|
return [];
|
|
12278
14065
|
}
|
|
12279
|
-
let filtered =
|
|
12280
|
-
if (status) {
|
|
12281
|
-
filtered = filtered.filter((h) => h.status === status);
|
|
12282
|
-
}
|
|
12283
|
-
filtered = filtered.sort((a, b) => b.startTime - a.startTime).slice(0, limit);
|
|
14066
|
+
let filtered = history.sort((a, b) => b.startTime - a.startTime).slice(0, limit);
|
|
12284
14067
|
return filtered.map((h) => {
|
|
12285
14068
|
let result = null;
|
|
12286
14069
|
if (h.result) {
|
|
@@ -12375,8 +14158,7 @@ class SchedulerPlugin extends Plugin {
|
|
|
12375
14158
|
clearTimeout(timer);
|
|
12376
14159
|
}
|
|
12377
14160
|
this.timers.clear();
|
|
12378
|
-
|
|
12379
|
-
if (!isTestEnvironment && this.activeJobs.size > 0) {
|
|
14161
|
+
if (!this._isTestEnvironment() && this.activeJobs.size > 0) {
|
|
12380
14162
|
if (this.config.verbose) {
|
|
12381
14163
|
console.log(`[SchedulerPlugin] Waiting for ${this.activeJobs.size} active jobs to complete...`);
|
|
12382
14164
|
}
|
|
@@ -12389,7 +14171,7 @@ class SchedulerPlugin extends Plugin {
|
|
|
12389
14171
|
console.warn(`[SchedulerPlugin] ${this.activeJobs.size} jobs still running after timeout`);
|
|
12390
14172
|
}
|
|
12391
14173
|
}
|
|
12392
|
-
if (
|
|
14174
|
+
if (this._isTestEnvironment()) {
|
|
12393
14175
|
this.activeJobs.clear();
|
|
12394
14176
|
}
|
|
12395
14177
|
}
|
|
@@ -12410,14 +14192,14 @@ class StateMachinePlugin extends Plugin {
|
|
|
12410
14192
|
actions: options.actions || {},
|
|
12411
14193
|
guards: options.guards || {},
|
|
12412
14194
|
persistTransitions: options.persistTransitions !== false,
|
|
12413
|
-
transitionLogResource: options.transitionLogResource || "
|
|
12414
|
-
stateResource: options.stateResource || "
|
|
12415
|
-
|
|
12416
|
-
|
|
14195
|
+
transitionLogResource: options.transitionLogResource || "plg_state_transitions",
|
|
14196
|
+
stateResource: options.stateResource || "plg_entity_states",
|
|
14197
|
+
retryAttempts: options.retryAttempts || 3,
|
|
14198
|
+
retryDelay: options.retryDelay || 100,
|
|
14199
|
+
verbose: options.verbose || false
|
|
12417
14200
|
};
|
|
12418
14201
|
this.database = null;
|
|
12419
14202
|
this.machines = /* @__PURE__ */ new Map();
|
|
12420
|
-
this.stateStorage = /* @__PURE__ */ new Map();
|
|
12421
14203
|
this._validateConfiguration();
|
|
12422
14204
|
}
|
|
12423
14205
|
_validateConfiguration() {
|
|
@@ -12558,43 +14340,55 @@ class StateMachinePlugin extends Plugin {
|
|
|
12558
14340
|
machine.currentStates.set(entityId, toState);
|
|
12559
14341
|
if (this.config.persistTransitions) {
|
|
12560
14342
|
const transitionId = `${machineId}_${entityId}_${timestamp}`;
|
|
12561
|
-
|
|
12562
|
-
|
|
12563
|
-
|
|
12564
|
-
|
|
12565
|
-
|
|
12566
|
-
|
|
12567
|
-
|
|
12568
|
-
|
|
12569
|
-
|
|
12570
|
-
|
|
12571
|
-
|
|
12572
|
-
|
|
12573
|
-
|
|
12574
|
-
|
|
14343
|
+
let logOk = false;
|
|
14344
|
+
let lastLogErr;
|
|
14345
|
+
for (let attempt = 0; attempt < this.config.retryAttempts; attempt++) {
|
|
14346
|
+
const [ok, err] = await tryFn(
|
|
14347
|
+
() => this.database.resource(this.config.transitionLogResource).insert({
|
|
14348
|
+
id: transitionId,
|
|
14349
|
+
machineId,
|
|
14350
|
+
entityId,
|
|
14351
|
+
fromState,
|
|
14352
|
+
toState,
|
|
14353
|
+
event,
|
|
14354
|
+
context,
|
|
14355
|
+
timestamp,
|
|
14356
|
+
createdAt: now.slice(0, 10)
|
|
14357
|
+
// YYYY-MM-DD for partitioning
|
|
14358
|
+
})
|
|
14359
|
+
);
|
|
14360
|
+
if (ok) {
|
|
14361
|
+
logOk = true;
|
|
14362
|
+
break;
|
|
14363
|
+
}
|
|
14364
|
+
lastLogErr = err;
|
|
14365
|
+
if (attempt < this.config.retryAttempts - 1) {
|
|
14366
|
+
const delay = this.config.retryDelay * Math.pow(2, attempt);
|
|
14367
|
+
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
14368
|
+
}
|
|
14369
|
+
}
|
|
12575
14370
|
if (!logOk && this.config.verbose) {
|
|
12576
|
-
console.warn(`[StateMachinePlugin] Failed to log transition:`,
|
|
14371
|
+
console.warn(`[StateMachinePlugin] Failed to log transition after ${this.config.retryAttempts} attempts:`, lastLogErr.message);
|
|
12577
14372
|
}
|
|
12578
14373
|
const stateId = `${machineId}_${entityId}`;
|
|
12579
|
-
const
|
|
12580
|
-
|
|
12581
|
-
|
|
12582
|
-
|
|
12583
|
-
|
|
12584
|
-
|
|
12585
|
-
|
|
12586
|
-
|
|
12587
|
-
|
|
12588
|
-
|
|
12589
|
-
|
|
12590
|
-
|
|
12591
|
-
|
|
12592
|
-
|
|
12593
|
-
|
|
14374
|
+
const stateData = {
|
|
14375
|
+
machineId,
|
|
14376
|
+
entityId,
|
|
14377
|
+
currentState: toState,
|
|
14378
|
+
context,
|
|
14379
|
+
lastTransition: transitionId,
|
|
14380
|
+
updatedAt: now
|
|
14381
|
+
};
|
|
14382
|
+
const [updateOk] = await tryFn(
|
|
14383
|
+
() => this.database.resource(this.config.stateResource).update(stateId, stateData)
|
|
14384
|
+
);
|
|
14385
|
+
if (!updateOk) {
|
|
14386
|
+
const [insertOk, insertErr] = await tryFn(
|
|
14387
|
+
() => this.database.resource(this.config.stateResource).insert({ id: stateId, ...stateData })
|
|
14388
|
+
);
|
|
14389
|
+
if (!insertOk && this.config.verbose) {
|
|
14390
|
+
console.warn(`[StateMachinePlugin] Failed to upsert state:`, insertErr.message);
|
|
12594
14391
|
}
|
|
12595
|
-
});
|
|
12596
|
-
if (!stateOk && this.config.verbose) {
|
|
12597
|
-
console.warn(`[StateMachinePlugin] Failed to update state:`, stateErr.message);
|
|
12598
14392
|
}
|
|
12599
14393
|
}
|
|
12600
14394
|
}
|
|
@@ -12625,8 +14419,9 @@ class StateMachinePlugin extends Plugin {
|
|
|
12625
14419
|
}
|
|
12626
14420
|
/**
|
|
12627
14421
|
* Get valid events for current state
|
|
14422
|
+
* Can accept either a state name (sync) or entityId (async to fetch latest state)
|
|
12628
14423
|
*/
|
|
12629
|
-
getValidEvents(machineId, stateOrEntityId) {
|
|
14424
|
+
async getValidEvents(machineId, stateOrEntityId) {
|
|
12630
14425
|
const machine = this.machines.get(machineId);
|
|
12631
14426
|
if (!machine) {
|
|
12632
14427
|
throw new Error(`State machine '${machineId}' not found`);
|
|
@@ -12635,7 +14430,7 @@ class StateMachinePlugin extends Plugin {
|
|
|
12635
14430
|
if (machine.config.states[stateOrEntityId]) {
|
|
12636
14431
|
state = stateOrEntityId;
|
|
12637
14432
|
} else {
|
|
12638
|
-
state =
|
|
14433
|
+
state = await this.getState(machineId, stateOrEntityId);
|
|
12639
14434
|
}
|
|
12640
14435
|
const stateConfig = machine.config.states[state];
|
|
12641
14436
|
return stateConfig && stateConfig.on ? Object.keys(stateConfig.on) : [];
|
|
@@ -12649,9 +14444,10 @@ class StateMachinePlugin extends Plugin {
|
|
|
12649
14444
|
}
|
|
12650
14445
|
const { limit = 50, offset = 0 } = options;
|
|
12651
14446
|
const [ok, err, transitions] = await tryFn(
|
|
12652
|
-
() => this.database.resource(this.config.transitionLogResource).
|
|
12653
|
-
|
|
12654
|
-
|
|
14447
|
+
() => this.database.resource(this.config.transitionLogResource).query({
|
|
14448
|
+
machineId,
|
|
14449
|
+
entityId
|
|
14450
|
+
}, {
|
|
12655
14451
|
limit,
|
|
12656
14452
|
offset
|
|
12657
14453
|
})
|
|
@@ -12662,8 +14458,8 @@ class StateMachinePlugin extends Plugin {
|
|
|
12662
14458
|
}
|
|
12663
14459
|
return [];
|
|
12664
14460
|
}
|
|
12665
|
-
const
|
|
12666
|
-
return
|
|
14461
|
+
const sorted = (transitions || []).sort((a, b) => b.timestamp - a.timestamp);
|
|
14462
|
+
return sorted.map((t) => ({
|
|
12667
14463
|
from: t.fromState,
|
|
12668
14464
|
to: t.toState,
|
|
12669
14465
|
event: t.event,
|
|
@@ -12684,15 +14480,20 @@ class StateMachinePlugin extends Plugin {
|
|
|
12684
14480
|
if (this.config.persistTransitions) {
|
|
12685
14481
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
12686
14482
|
const stateId = `${machineId}_${entityId}`;
|
|
12687
|
-
await
|
|
12688
|
-
|
|
12689
|
-
|
|
12690
|
-
|
|
12691
|
-
|
|
12692
|
-
|
|
12693
|
-
|
|
12694
|
-
|
|
12695
|
-
|
|
14483
|
+
const [ok, err] = await tryFn(
|
|
14484
|
+
() => this.database.resource(this.config.stateResource).insert({
|
|
14485
|
+
id: stateId,
|
|
14486
|
+
machineId,
|
|
14487
|
+
entityId,
|
|
14488
|
+
currentState: initialState,
|
|
14489
|
+
context,
|
|
14490
|
+
lastTransition: null,
|
|
14491
|
+
updatedAt: now
|
|
14492
|
+
})
|
|
14493
|
+
);
|
|
14494
|
+
if (!ok && err && !err.message?.includes("already exists")) {
|
|
14495
|
+
throw new Error(`Failed to initialize entity state: ${err.message}`);
|
|
14496
|
+
}
|
|
12696
14497
|
}
|
|
12697
14498
|
const initialStateConfig = machine.config.states[initialState];
|
|
12698
14499
|
if (initialStateConfig && initialStateConfig.entry) {
|
|
@@ -12757,7 +14558,6 @@ class StateMachinePlugin extends Plugin {
|
|
|
12757
14558
|
}
|
|
12758
14559
|
async stop() {
|
|
12759
14560
|
this.machines.clear();
|
|
12760
|
-
this.stateStorage.clear();
|
|
12761
14561
|
}
|
|
12762
14562
|
async cleanup() {
|
|
12763
14563
|
await this.stop();
|
|
@@ -12765,5 +14565,5 @@ class StateMachinePlugin extends Plugin {
|
|
|
12765
14565
|
}
|
|
12766
14566
|
}
|
|
12767
14567
|
|
|
12768
|
-
export { AVAILABLE_BEHAVIORS, AuditPlugin, AuthenticationError, BackupPlugin, BaseError, CachePlugin, Client, ConnectionString, ConnectionStringError, CostsPlugin, CryptoError, DEFAULT_BEHAVIOR, Database, DatabaseError, EncryptionError, ErrorMap, FullTextPlugin, InvalidResourceItem, MetricsPlugin, MissingMetadata, NoSuchBucket, NoSuchKey, NotFound, PartitionError, PermissionError, Plugin, PluginObject, ReplicatorPlugin, Resource, ResourceError, ResourceIdsPageReader, ResourceIdsReader, ResourceNotFound, ResourceReader, ResourceWriter, Database as S3db, S3dbError, SchedulerPlugin, Schema, SchemaError, StateMachinePlugin, UnknownError, ValidationError, Validator, behaviors, calculateAttributeNamesSize, calculateAttributeSizes, calculateEffectiveLimit, calculateSystemOverhead, calculateTotalSize, calculateUTF8Bytes, clearUTF8Cache, clearUTF8Memo, clearUTF8Memory, decode, decodeDecimal, decrypt, S3db as default, encode, encodeDecimal, encrypt, getBehavior, getSizeBreakdown, idGenerator, mapAwsError, md5, passwordGenerator, sha256, streamToString, transformValue, tryFn, tryFnSync };
|
|
14568
|
+
export { AVAILABLE_BEHAVIORS, AuditPlugin, AuthenticationError, BackupPlugin, BaseError, CachePlugin, Client, ConnectionString, ConnectionStringError, CostsPlugin, CryptoError, DEFAULT_BEHAVIOR, Database, DatabaseError, EncryptionError, ErrorMap, EventualConsistencyPlugin, FullTextPlugin, InvalidResourceItem, MetricsPlugin, MissingMetadata, NoSuchBucket, NoSuchKey, NotFound, PartitionError, PermissionError, Plugin, PluginObject, ReplicatorPlugin, Resource, ResourceError, ResourceIdsPageReader, ResourceIdsReader, ResourceNotFound, ResourceReader, ResourceWriter, S3QueuePlugin, Database as S3db, S3dbError, SchedulerPlugin, Schema, SchemaError, StateMachinePlugin, UnknownError, ValidationError, Validator, behaviors, calculateAttributeNamesSize, calculateAttributeSizes, calculateEffectiveLimit, calculateSystemOverhead, calculateTotalSize, calculateUTF8Bytes, clearUTF8Cache, clearUTF8Memo, clearUTF8Memory, decode, decodeDecimal, decrypt, S3db as default, encode, encodeDecimal, encrypt, getBehavior, getSizeBreakdown, idGenerator, mapAwsError, md5, passwordGenerator, sha256, streamToString, transformValue, tryFn, tryFnSync };
|
|
12769
14569
|
//# sourceMappingURL=s3db.es.js.map
|