s3db.js 9.3.0 → 10.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/s3db.cjs.js CHANGED
@@ -10,11 +10,12 @@ var promises$1 = require('stream/promises');
10
10
  var path = require('path');
11
11
  var crypto = require('crypto');
12
12
  var zlib = require('node:zlib');
13
+ var os = require('os');
14
+ var jsonStableStringify = require('json-stable-stringify');
13
15
  var stream = require('stream');
14
16
  var promisePool = require('@supercharge/promise-pool');
15
17
  var web = require('node:stream/web');
16
18
  var lodashEs = require('lodash-es');
17
- var jsonStableStringify = require('json-stable-stringify');
18
19
  var http = require('http');
19
20
  var https = require('https');
20
21
  var nodeHttpHandler = require('@smithy/node-http-handler');
@@ -423,8 +424,14 @@ function mapAwsError(err, context = {}) {
423
424
  suggestion = "Check if the object metadata is present and valid.";
424
425
  return new MissingMetadata({ ...context, original: err, metadata, commandName, commandInput, suggestion });
425
426
  }
426
- suggestion = "Check the error details and AWS documentation.";
427
- return new UnknownError("Unknown error", { ...context, original: err, metadata, commandName, commandInput, suggestion });
427
+ const errorDetails = [
428
+ `Unknown error: ${err.message || err.toString()}`,
429
+ err.code && `Code: ${err.code}`,
430
+ err.statusCode && `Status: ${err.statusCode}`,
431
+ err.stack && `Stack: ${err.stack.split("\n")[0]}`
432
+ ].filter(Boolean).join(" | ");
433
+ suggestion = `Check the error details and AWS documentation. Original error: ${err.message || err.toString()}`;
434
+ return new UnknownError(errorDetails, { ...context, original: err, metadata, commandName, commandInput, suggestion });
428
435
  }
429
436
  class ConnectionStringError extends S3dbError {
430
437
  constructor(message, details = {}) {
@@ -836,7 +843,7 @@ class AuditPlugin extends Plugin {
836
843
  }
837
844
  async onSetup() {
838
845
  const [ok, err, auditResource] = await tryFn(() => this.database.createResource({
839
- name: "audits",
846
+ name: "plg_audits",
840
847
  attributes: {
841
848
  id: "string|required",
842
849
  resourceName: "string|required",
@@ -852,15 +859,15 @@ class AuditPlugin extends Plugin {
852
859
  },
853
860
  behavior: "body-overflow"
854
861
  }));
855
- this.auditResource = ok ? auditResource : this.database.resources.audits || null;
862
+ this.auditResource = ok ? auditResource : this.database.resources.plg_audits || null;
856
863
  if (!ok && !this.auditResource) return;
857
864
  this.database.addHook("afterCreateResource", (context) => {
858
- if (context.resource.name !== "audits") {
865
+ if (context.resource.name !== "plg_audits") {
859
866
  this.setupResourceAuditing(context.resource);
860
867
  }
861
868
  });
862
869
  for (const resource of Object.values(this.database.resources)) {
863
- if (resource.name !== "audits") {
870
+ if (resource.name !== "plg_audits") {
864
871
  this.setupResourceAuditing(resource);
865
872
  }
866
873
  }
@@ -1880,11 +1887,10 @@ function validateBackupConfig(driver, config = {}) {
1880
1887
  class BackupPlugin extends Plugin {
1881
1888
  constructor(options = {}) {
1882
1889
  super();
1883
- this.driverName = options.driver || "filesystem";
1884
- this.driverConfig = options.config || {};
1885
1890
  this.config = {
1886
- // Legacy destinations support (will be converted to multi driver)
1887
- destinations: options.destinations || null,
1891
+ // Driver configuration
1892
+ driver: options.driver || "filesystem",
1893
+ driverConfig: options.config || {},
1888
1894
  // Scheduling configuration
1889
1895
  schedule: options.schedule || {},
1890
1896
  // Retention policy (Grandfather-Father-Son)
@@ -1902,8 +1908,8 @@ class BackupPlugin extends Plugin {
1902
1908
  parallelism: options.parallelism || 4,
1903
1909
  include: options.include || null,
1904
1910
  exclude: options.exclude || [],
1905
- backupMetadataResource: options.backupMetadataResource || "backup_metadata",
1906
- tempDir: options.tempDir || "./tmp/backups",
1911
+ backupMetadataResource: options.backupMetadataResource || "plg_backup_metadata",
1912
+ tempDir: options.tempDir || path.join(os.tmpdir(), "s3db", "backups"),
1907
1913
  verbose: options.verbose || false,
1908
1914
  // Hooks
1909
1915
  onBackupStart: options.onBackupStart || null,
@@ -1915,32 +1921,9 @@ class BackupPlugin extends Plugin {
1915
1921
  };
1916
1922
  this.driver = null;
1917
1923
  this.activeBackups = /* @__PURE__ */ new Set();
1918
- this._handleLegacyDestinations();
1919
- validateBackupConfig(this.driverName, this.driverConfig);
1924
+ validateBackupConfig(this.config.driver, this.config.driverConfig);
1920
1925
  this._validateConfiguration();
1921
1926
  }
1922
- /**
1923
- * Convert legacy destinations format to multi driver format
1924
- */
1925
- _handleLegacyDestinations() {
1926
- if (this.config.destinations && Array.isArray(this.config.destinations)) {
1927
- this.driverName = "multi";
1928
- this.driverConfig = {
1929
- strategy: "all",
1930
- destinations: this.config.destinations.map((dest) => {
1931
- const { type, ...config } = dest;
1932
- return {
1933
- driver: type,
1934
- config
1935
- };
1936
- })
1937
- };
1938
- this.config.destinations = null;
1939
- if (this.config.verbose) {
1940
- console.log("[BackupPlugin] Converted legacy destinations format to multi driver");
1941
- }
1942
- }
1943
- }
1944
1927
  _validateConfiguration() {
1945
1928
  if (this.config.encryption && (!this.config.encryption.key || !this.config.encryption.algorithm)) {
1946
1929
  throw new Error("BackupPlugin: Encryption requires both key and algorithm");
@@ -1950,7 +1933,7 @@ class BackupPlugin extends Plugin {
1950
1933
  }
1951
1934
  }
1952
1935
  async onSetup() {
1953
- this.driver = createBackupDriver(this.driverName, this.driverConfig);
1936
+ this.driver = createBackupDriver(this.config.driver, this.config.driverConfig);
1954
1937
  await this.driver.setup(this.database);
1955
1938
  await promises.mkdir(this.config.tempDir, { recursive: true });
1956
1939
  await this._createBackupMetadataResource();
@@ -1998,6 +1981,9 @@ class BackupPlugin extends Plugin {
1998
1981
  async backup(type = "full", options = {}) {
1999
1982
  const backupId = this._generateBackupId(type);
2000
1983
  const startTime = Date.now();
1984
+ if (this.activeBackups.has(backupId)) {
1985
+ throw new Error(`Backup '${backupId}' is already in progress`);
1986
+ }
2001
1987
  try {
2002
1988
  this.activeBackups.add(backupId);
2003
1989
  if (this.config.onBackupStart) {
@@ -2013,16 +1999,9 @@ class BackupPlugin extends Plugin {
2013
1999
  if (exportedFiles.length === 0) {
2014
2000
  throw new Error("No resources were exported for backup");
2015
2001
  }
2016
- let finalPath;
2017
- let totalSize = 0;
2018
- if (this.config.compression !== "none") {
2019
- finalPath = path.join(tempBackupDir, `${backupId}.tar.gz`);
2020
- totalSize = await this._createCompressedArchive(exportedFiles, finalPath);
2021
- } else {
2022
- finalPath = exportedFiles[0];
2023
- const [statOk, , stats] = await tryFn(() => promises.stat(finalPath));
2024
- totalSize = statOk ? stats.size : 0;
2025
- }
2002
+ const archiveExtension = this.config.compression !== "none" ? ".tar.gz" : ".json";
2003
+ const finalPath = path.join(tempBackupDir, `${backupId}${archiveExtension}`);
2004
+ const totalSize = await this._createArchive(exportedFiles, finalPath, this.config.compression);
2026
2005
  const checksum = await this._generateChecksum(finalPath);
2027
2006
  const uploadResult = await this.driver.upload(finalPath, backupId, manifest);
2028
2007
  if (this.config.verification) {
@@ -2131,15 +2110,35 @@ class BackupPlugin extends Plugin {
2131
2110
  for (const resourceName of resourceNames) {
2132
2111
  const resource = this.database.resources[resourceName];
2133
2112
  if (!resource) {
2134
- console.warn(`[BackupPlugin] Resource '${resourceName}' not found, skipping`);
2113
+ if (this.config.verbose) {
2114
+ console.warn(`[BackupPlugin] Resource '${resourceName}' not found, skipping`);
2115
+ }
2135
2116
  continue;
2136
2117
  }
2137
2118
  const exportPath = path.join(tempDir, `${resourceName}.json`);
2138
2119
  let records;
2139
2120
  if (type === "incremental") {
2140
- const yesterday = new Date(Date.now() - 24 * 60 * 60 * 1e3);
2121
+ const [lastBackupOk, , lastBackups] = await tryFn(
2122
+ () => this.database.resource(this.config.backupMetadataResource).list({
2123
+ filter: {
2124
+ status: "completed",
2125
+ type: { $in: ["full", "incremental"] }
2126
+ },
2127
+ sort: { timestamp: -1 },
2128
+ limit: 1
2129
+ })
2130
+ );
2131
+ let sinceTimestamp;
2132
+ if (lastBackupOk && lastBackups && lastBackups.length > 0) {
2133
+ sinceTimestamp = new Date(lastBackups[0].timestamp);
2134
+ } else {
2135
+ sinceTimestamp = new Date(Date.now() - 24 * 60 * 60 * 1e3);
2136
+ }
2137
+ if (this.config.verbose) {
2138
+ console.log(`[BackupPlugin] Incremental backup for '${resourceName}' since ${sinceTimestamp.toISOString()}`);
2139
+ }
2141
2140
  records = await resource.list({
2142
- filter: { updatedAt: { ">": yesterday.toISOString() } }
2141
+ filter: { updatedAt: { ">": sinceTimestamp.toISOString() } }
2143
2142
  });
2144
2143
  } else {
2145
2144
  records = await resource.list();
@@ -2159,29 +2158,57 @@ class BackupPlugin extends Plugin {
2159
2158
  }
2160
2159
  return exportedFiles;
2161
2160
  }
2162
- async _createCompressedArchive(files, targetPath) {
2163
- const output = fs.createWriteStream(targetPath);
2164
- const gzip = zlib.createGzip({ level: 6 });
2161
+ async _createArchive(files, targetPath, compressionType) {
2162
+ const archive = {
2163
+ version: "1.0",
2164
+ created: (/* @__PURE__ */ new Date()).toISOString(),
2165
+ files: []
2166
+ };
2165
2167
  let totalSize = 0;
2166
- await promises$1.pipeline(
2167
- async function* () {
2168
- for (const filePath of files) {
2169
- const content = await promises.readFile(filePath);
2170
- totalSize += content.length;
2171
- yield content;
2168
+ for (const filePath of files) {
2169
+ const [readOk, readErr, content] = await tryFn(() => promises.readFile(filePath, "utf8"));
2170
+ if (!readOk) {
2171
+ if (this.config.verbose) {
2172
+ console.warn(`[BackupPlugin] Failed to read ${filePath}: ${readErr?.message}`);
2172
2173
  }
2173
- },
2174
- gzip,
2175
- output
2176
- );
2174
+ continue;
2175
+ }
2176
+ const fileName = path.basename(filePath);
2177
+ totalSize += content.length;
2178
+ archive.files.push({
2179
+ name: fileName,
2180
+ size: content.length,
2181
+ content
2182
+ });
2183
+ }
2184
+ const archiveJson = JSON.stringify(archive);
2185
+ if (compressionType === "none") {
2186
+ await promises.writeFile(targetPath, archiveJson, "utf8");
2187
+ } else {
2188
+ const output = fs.createWriteStream(targetPath);
2189
+ const gzip = zlib.createGzip({ level: 6 });
2190
+ await promises$1.pipeline(
2191
+ async function* () {
2192
+ yield Buffer.from(archiveJson, "utf8");
2193
+ },
2194
+ gzip,
2195
+ output
2196
+ );
2197
+ }
2177
2198
  const [statOk, , stats] = await tryFn(() => promises.stat(targetPath));
2178
2199
  return statOk ? stats.size : totalSize;
2179
2200
  }
2180
2201
  async _generateChecksum(filePath) {
2181
- const hash = crypto.createHash("sha256");
2182
- const stream = fs.createReadStream(filePath);
2183
- await promises$1.pipeline(stream, hash);
2184
- return hash.digest("hex");
2202
+ const [ok, err, result] = await tryFn(async () => {
2203
+ const hash = crypto.createHash("sha256");
2204
+ const stream = fs.createReadStream(filePath);
2205
+ await promises$1.pipeline(stream, hash);
2206
+ return hash.digest("hex");
2207
+ });
2208
+ if (!ok) {
2209
+ throw new Error(`Failed to generate checksum for ${filePath}: ${err?.message}`);
2210
+ }
2211
+ return result;
2185
2212
  }
2186
2213
  async _cleanupTempFiles(tempDir) {
2187
2214
  const [ok] = await tryFn(
@@ -2243,7 +2270,109 @@ class BackupPlugin extends Plugin {
2243
2270
  }
2244
2271
  async _restoreFromBackup(backupPath, options) {
2245
2272
  const restoredResources = [];
2246
- return restoredResources;
2273
+ try {
2274
+ let archiveData = "";
2275
+ if (this.config.compression !== "none") {
2276
+ const input = fs.createReadStream(backupPath);
2277
+ const gunzip = zlib.createGunzip();
2278
+ const chunks = [];
2279
+ await new Promise((resolve, reject) => {
2280
+ input.pipe(gunzip).on("data", (chunk) => chunks.push(chunk)).on("end", resolve).on("error", reject);
2281
+ });
2282
+ archiveData = Buffer.concat(chunks).toString("utf8");
2283
+ } else {
2284
+ archiveData = await promises.readFile(backupPath, "utf8");
2285
+ }
2286
+ let archive;
2287
+ try {
2288
+ archive = JSON.parse(archiveData);
2289
+ } catch (parseError) {
2290
+ throw new Error(`Failed to parse backup archive: ${parseError.message}`);
2291
+ }
2292
+ if (!archive || typeof archive !== "object") {
2293
+ throw new Error("Invalid backup archive: not a valid JSON object");
2294
+ }
2295
+ if (!archive.version || !archive.files) {
2296
+ throw new Error("Invalid backup archive format: missing version or files array");
2297
+ }
2298
+ if (this.config.verbose) {
2299
+ console.log(`[BackupPlugin] Restoring ${archive.files.length} files from backup`);
2300
+ }
2301
+ for (const file of archive.files) {
2302
+ try {
2303
+ const resourceData = JSON.parse(file.content);
2304
+ if (!resourceData.resourceName || !resourceData.definition) {
2305
+ if (this.config.verbose) {
2306
+ console.warn(`[BackupPlugin] Skipping invalid file: ${file.name}`);
2307
+ }
2308
+ continue;
2309
+ }
2310
+ const resourceName = resourceData.resourceName;
2311
+ if (options.resources && !options.resources.includes(resourceName)) {
2312
+ continue;
2313
+ }
2314
+ let resource = this.database.resources[resourceName];
2315
+ if (!resource) {
2316
+ if (this.config.verbose) {
2317
+ console.log(`[BackupPlugin] Creating resource '${resourceName}'`);
2318
+ }
2319
+ const [createOk, createErr] = await tryFn(
2320
+ () => this.database.createResource(resourceData.definition)
2321
+ );
2322
+ if (!createOk) {
2323
+ if (this.config.verbose) {
2324
+ console.warn(`[BackupPlugin] Failed to create resource '${resourceName}': ${createErr?.message}`);
2325
+ }
2326
+ continue;
2327
+ }
2328
+ resource = this.database.resources[resourceName];
2329
+ }
2330
+ if (resourceData.records && Array.isArray(resourceData.records)) {
2331
+ const mode = options.mode || "merge";
2332
+ if (mode === "replace") {
2333
+ const ids = await resource.listIds();
2334
+ for (const id of ids) {
2335
+ await resource.delete(id);
2336
+ }
2337
+ }
2338
+ let insertedCount = 0;
2339
+ for (const record of resourceData.records) {
2340
+ const [insertOk] = await tryFn(async () => {
2341
+ if (mode === "skip") {
2342
+ const existing = await resource.get(record.id);
2343
+ if (existing) {
2344
+ return false;
2345
+ }
2346
+ }
2347
+ await resource.insert(record);
2348
+ return true;
2349
+ });
2350
+ if (insertOk) {
2351
+ insertedCount++;
2352
+ }
2353
+ }
2354
+ restoredResources.push({
2355
+ name: resourceName,
2356
+ recordsRestored: insertedCount,
2357
+ totalRecords: resourceData.records.length
2358
+ });
2359
+ if (this.config.verbose) {
2360
+ console.log(`[BackupPlugin] Restored ${insertedCount}/${resourceData.records.length} records to '${resourceName}'`);
2361
+ }
2362
+ }
2363
+ } catch (fileError) {
2364
+ if (this.config.verbose) {
2365
+ console.warn(`[BackupPlugin] Error processing file ${file.name}: ${fileError.message}`);
2366
+ }
2367
+ }
2368
+ }
2369
+ return restoredResources;
2370
+ } catch (error) {
2371
+ if (this.config.verbose) {
2372
+ console.error(`[BackupPlugin] Error restoring backup: ${error.message}`);
2373
+ }
2374
+ throw new Error(`Failed to restore backup: ${error.message}`);
2375
+ }
2247
2376
  }
2248
2377
  /**
2249
2378
  * List available backups
@@ -2287,6 +2416,90 @@ class BackupPlugin extends Plugin {
2287
2416
  return ok ? backup : null;
2288
2417
  }
2289
2418
  async _cleanupOldBackups() {
2419
+ try {
2420
+ const [listOk, , allBackups] = await tryFn(
2421
+ () => this.database.resource(this.config.backupMetadataResource).list({
2422
+ filter: { status: "completed" },
2423
+ sort: { timestamp: -1 }
2424
+ })
2425
+ );
2426
+ if (!listOk || !allBackups || allBackups.length === 0) {
2427
+ return;
2428
+ }
2429
+ const now = Date.now();
2430
+ const msPerDay = 24 * 60 * 60 * 1e3;
2431
+ const msPerWeek = 7 * msPerDay;
2432
+ const msPerMonth = 30 * msPerDay;
2433
+ const msPerYear = 365 * msPerDay;
2434
+ const categorized = {
2435
+ daily: [],
2436
+ weekly: [],
2437
+ monthly: [],
2438
+ yearly: []
2439
+ };
2440
+ for (const backup of allBackups) {
2441
+ const age = now - backup.timestamp;
2442
+ if (age <= msPerDay * this.config.retention.daily) {
2443
+ categorized.daily.push(backup);
2444
+ } else if (age <= msPerWeek * this.config.retention.weekly) {
2445
+ categorized.weekly.push(backup);
2446
+ } else if (age <= msPerMonth * this.config.retention.monthly) {
2447
+ categorized.monthly.push(backup);
2448
+ } else if (age <= msPerYear * this.config.retention.yearly) {
2449
+ categorized.yearly.push(backup);
2450
+ }
2451
+ }
2452
+ const toKeep = /* @__PURE__ */ new Set();
2453
+ categorized.daily.forEach((b) => toKeep.add(b.id));
2454
+ const weeklyByWeek = /* @__PURE__ */ new Map();
2455
+ for (const backup of categorized.weekly) {
2456
+ const weekNum = Math.floor((now - backup.timestamp) / msPerWeek);
2457
+ if (!weeklyByWeek.has(weekNum)) {
2458
+ weeklyByWeek.set(weekNum, backup);
2459
+ toKeep.add(backup.id);
2460
+ }
2461
+ }
2462
+ const monthlyByMonth = /* @__PURE__ */ new Map();
2463
+ for (const backup of categorized.monthly) {
2464
+ const monthNum = Math.floor((now - backup.timestamp) / msPerMonth);
2465
+ if (!monthlyByMonth.has(monthNum)) {
2466
+ monthlyByMonth.set(monthNum, backup);
2467
+ toKeep.add(backup.id);
2468
+ }
2469
+ }
2470
+ const yearlyByYear = /* @__PURE__ */ new Map();
2471
+ for (const backup of categorized.yearly) {
2472
+ const yearNum = Math.floor((now - backup.timestamp) / msPerYear);
2473
+ if (!yearlyByYear.has(yearNum)) {
2474
+ yearlyByYear.set(yearNum, backup);
2475
+ toKeep.add(backup.id);
2476
+ }
2477
+ }
2478
+ const backupsToDelete = allBackups.filter((b) => !toKeep.has(b.id));
2479
+ if (backupsToDelete.length === 0) {
2480
+ return;
2481
+ }
2482
+ if (this.config.verbose) {
2483
+ console.log(`[BackupPlugin] Cleaning up ${backupsToDelete.length} old backups (keeping ${toKeep.size})`);
2484
+ }
2485
+ for (const backup of backupsToDelete) {
2486
+ try {
2487
+ await this.driver.delete(backup.id, backup.driverInfo);
2488
+ await this.database.resource(this.config.backupMetadataResource).delete(backup.id);
2489
+ if (this.config.verbose) {
2490
+ console.log(`[BackupPlugin] Deleted old backup: ${backup.id}`);
2491
+ }
2492
+ } catch (deleteError) {
2493
+ if (this.config.verbose) {
2494
+ console.warn(`[BackupPlugin] Failed to delete backup ${backup.id}: ${deleteError.message}`);
2495
+ }
2496
+ }
2497
+ }
2498
+ } catch (error) {
2499
+ if (this.config.verbose) {
2500
+ console.warn(`[BackupPlugin] Error during cleanup: ${error.message}`);
2501
+ }
2502
+ }
2290
2503
  }
2291
2504
  async _executeHook(hook, ...args) {
2292
2505
  if (typeof hook === "function") {
@@ -3576,81 +3789,58 @@ class PartitionAwareFilesystemCache extends FilesystemCache {
3576
3789
  class CachePlugin extends Plugin {
3577
3790
  constructor(options = {}) {
3578
3791
  super(options);
3579
- this.driverName = options.driver || "s3";
3580
- this.ttl = options.ttl;
3581
- this.maxSize = options.maxSize;
3582
- this.config = options.config || {};
3583
- this.includePartitions = options.includePartitions !== false;
3584
- this.partitionStrategy = options.partitionStrategy || "hierarchical";
3585
- this.partitionAware = options.partitionAware !== false;
3586
- this.trackUsage = options.trackUsage !== false;
3587
- this.preloadRelated = options.preloadRelated !== false;
3588
- this.legacyConfig = {
3589
- memoryOptions: options.memoryOptions,
3590
- filesystemOptions: options.filesystemOptions,
3591
- s3Options: options.s3Options,
3592
- driver: options.driver
3792
+ this.config = {
3793
+ // Driver configuration
3794
+ driver: options.driver || "s3",
3795
+ config: {
3796
+ ttl: options.ttl,
3797
+ maxSize: options.maxSize,
3798
+ ...options.config
3799
+ // Driver-specific config (can override ttl/maxSize)
3800
+ },
3801
+ // Resource filtering
3802
+ include: options.include || null,
3803
+ // Array of resource names to cache (null = all)
3804
+ exclude: options.exclude || [],
3805
+ // Array of resource names to exclude
3806
+ // Partition settings
3807
+ includePartitions: options.includePartitions !== false,
3808
+ partitionStrategy: options.partitionStrategy || "hierarchical",
3809
+ partitionAware: options.partitionAware !== false,
3810
+ trackUsage: options.trackUsage !== false,
3811
+ preloadRelated: options.preloadRelated !== false,
3812
+ // Retry configuration
3813
+ retryAttempts: options.retryAttempts || 3,
3814
+ retryDelay: options.retryDelay || 100,
3815
+ // ms
3816
+ // Logging
3817
+ verbose: options.verbose || false
3593
3818
  };
3594
3819
  }
3595
3820
  async setup(database) {
3596
3821
  await super.setup(database);
3597
3822
  }
3598
3823
  async onSetup() {
3599
- if (this.driverName && typeof this.driverName === "object") {
3600
- this.driver = this.driverName;
3601
- } else if (this.driverName === "memory") {
3602
- const driverConfig = {
3603
- ...this.legacyConfig.memoryOptions,
3604
- // Legacy support (lowest priority)
3605
- ...this.config
3606
- // New config format (medium priority)
3607
- };
3608
- if (this.ttl !== void 0) {
3609
- driverConfig.ttl = this.ttl;
3610
- }
3611
- if (this.maxSize !== void 0) {
3612
- driverConfig.maxSize = this.maxSize;
3613
- }
3614
- this.driver = new MemoryCache(driverConfig);
3615
- } else if (this.driverName === "filesystem") {
3616
- const driverConfig = {
3617
- ...this.legacyConfig.filesystemOptions,
3618
- // Legacy support (lowest priority)
3619
- ...this.config
3620
- // New config format (medium priority)
3621
- };
3622
- if (this.ttl !== void 0) {
3623
- driverConfig.ttl = this.ttl;
3624
- }
3625
- if (this.maxSize !== void 0) {
3626
- driverConfig.maxSize = this.maxSize;
3627
- }
3628
- if (this.partitionAware) {
3824
+ if (this.config.driver && typeof this.config.driver === "object") {
3825
+ this.driver = this.config.driver;
3826
+ } else if (this.config.driver === "memory") {
3827
+ this.driver = new MemoryCache(this.config.config);
3828
+ } else if (this.config.driver === "filesystem") {
3829
+ if (this.config.partitionAware) {
3629
3830
  this.driver = new PartitionAwareFilesystemCache({
3630
- partitionStrategy: this.partitionStrategy,
3631
- trackUsage: this.trackUsage,
3632
- preloadRelated: this.preloadRelated,
3633
- ...driverConfig
3831
+ partitionStrategy: this.config.partitionStrategy,
3832
+ trackUsage: this.config.trackUsage,
3833
+ preloadRelated: this.config.preloadRelated,
3834
+ ...this.config.config
3634
3835
  });
3635
3836
  } else {
3636
- this.driver = new FilesystemCache(driverConfig);
3837
+ this.driver = new FilesystemCache(this.config.config);
3637
3838
  }
3638
3839
  } else {
3639
- const driverConfig = {
3840
+ this.driver = new S3Cache({
3640
3841
  client: this.database.client,
3641
- // Required for S3Cache
3642
- ...this.legacyConfig.s3Options,
3643
- // Legacy support (lowest priority)
3644
- ...this.config
3645
- // New config format (medium priority)
3646
- };
3647
- if (this.ttl !== void 0) {
3648
- driverConfig.ttl = this.ttl;
3649
- }
3650
- if (this.maxSize !== void 0) {
3651
- driverConfig.maxSize = this.maxSize;
3652
- }
3653
- this.driver = new S3Cache(driverConfig);
3842
+ ...this.config.config
3843
+ });
3654
3844
  }
3655
3845
  this.installDatabaseHooks();
3656
3846
  this.installResourceHooks();
@@ -3660,7 +3850,9 @@ class CachePlugin extends Plugin {
3660
3850
  */
3661
3851
  installDatabaseHooks() {
3662
3852
  this.database.addHook("afterCreateResource", async ({ resource }) => {
3663
- this.installResourceHooksForResource(resource);
3853
+ if (this.shouldCacheResource(resource.name)) {
3854
+ this.installResourceHooksForResource(resource);
3855
+ }
3664
3856
  });
3665
3857
  }
3666
3858
  async onStart() {
@@ -3670,9 +3862,24 @@ class CachePlugin extends Plugin {
3670
3862
  // Remove the old installDatabaseProxy method
3671
3863
  installResourceHooks() {
3672
3864
  for (const resource of Object.values(this.database.resources)) {
3865
+ if (!this.shouldCacheResource(resource.name)) {
3866
+ continue;
3867
+ }
3673
3868
  this.installResourceHooksForResource(resource);
3674
3869
  }
3675
3870
  }
3871
+ shouldCacheResource(resourceName) {
3872
+ if (resourceName.startsWith("plg_") && !this.config.include) {
3873
+ return false;
3874
+ }
3875
+ if (this.config.exclude.includes(resourceName)) {
3876
+ return false;
3877
+ }
3878
+ if (this.config.include && !this.config.include.includes(resourceName)) {
3879
+ return false;
3880
+ }
3881
+ return true;
3882
+ }
3676
3883
  installResourceHooksForResource(resource) {
3677
3884
  if (!this.driver) return;
3678
3885
  Object.defineProperty(resource, "cache", {
@@ -3821,37 +4028,74 @@ class CachePlugin extends Plugin {
3821
4028
  if (data && data.id) {
3822
4029
  const itemSpecificMethods = ["get", "exists", "content", "hasContent"];
3823
4030
  for (const method of itemSpecificMethods) {
3824
- try {
3825
- const specificKey = await this.generateCacheKey(resource, method, { id: data.id });
3826
- await resource.cache.clear(specificKey.replace(".json.gz", ""));
3827
- } catch (error) {
4031
+ const specificKey = await this.generateCacheKey(resource, method, { id: data.id });
4032
+ const [ok2, err2] = await this.clearCacheWithRetry(resource.cache, specificKey);
4033
+ if (!ok2) {
4034
+ this.emit("cache_clear_error", {
4035
+ resource: resource.name,
4036
+ method,
4037
+ id: data.id,
4038
+ error: err2.message
4039
+ });
4040
+ if (this.config.verbose) {
4041
+ console.warn(`[CachePlugin] Failed to clear ${method} cache for ${resource.name}:${data.id}:`, err2.message);
4042
+ }
3828
4043
  }
3829
4044
  }
3830
4045
  if (this.config.includePartitions === true && resource.config?.partitions && Object.keys(resource.config.partitions).length > 0) {
3831
4046
  const partitionValues = this.getPartitionValues(data, resource);
3832
4047
  for (const [partitionName, values] of Object.entries(partitionValues)) {
3833
4048
  if (values && Object.keys(values).length > 0 && Object.values(values).some((v) => v !== null && v !== void 0)) {
3834
- try {
3835
- const partitionKeyPrefix = path.join(keyPrefix, `partition=${partitionName}`);
3836
- await resource.cache.clear(partitionKeyPrefix);
3837
- } catch (error) {
4049
+ const partitionKeyPrefix = path.join(keyPrefix, `partition=${partitionName}`);
4050
+ const [ok2, err2] = await this.clearCacheWithRetry(resource.cache, partitionKeyPrefix);
4051
+ if (!ok2) {
4052
+ this.emit("cache_clear_error", {
4053
+ resource: resource.name,
4054
+ partition: partitionName,
4055
+ error: err2.message
4056
+ });
4057
+ if (this.config.verbose) {
4058
+ console.warn(`[CachePlugin] Failed to clear partition cache for ${resource.name}/${partitionName}:`, err2.message);
4059
+ }
3838
4060
  }
3839
4061
  }
3840
4062
  }
3841
4063
  }
3842
4064
  }
3843
- try {
3844
- await resource.cache.clear(keyPrefix);
3845
- } catch (error) {
4065
+ const [ok, err] = await this.clearCacheWithRetry(resource.cache, keyPrefix);
4066
+ if (!ok) {
4067
+ this.emit("cache_clear_error", {
4068
+ resource: resource.name,
4069
+ type: "broad",
4070
+ error: err.message
4071
+ });
4072
+ if (this.config.verbose) {
4073
+ console.warn(`[CachePlugin] Failed to clear broad cache for ${resource.name}, trying specific methods:`, err.message);
4074
+ }
3846
4075
  const aggregateMethods = ["count", "list", "listIds", "getAll", "page", "query"];
3847
4076
  for (const method of aggregateMethods) {
3848
- try {
3849
- await resource.cache.clear(`${keyPrefix}/action=${method}`);
3850
- await resource.cache.clear(`resource=${resource.name}/action=${method}`);
3851
- } catch (methodError) {
3852
- }
4077
+ await this.clearCacheWithRetry(resource.cache, `${keyPrefix}/action=${method}`);
4078
+ await this.clearCacheWithRetry(resource.cache, `resource=${resource.name}/action=${method}`);
4079
+ }
4080
+ }
4081
+ }
4082
+ async clearCacheWithRetry(cache, key) {
4083
+ let lastError;
4084
+ for (let attempt = 0; attempt < this.config.retryAttempts; attempt++) {
4085
+ const [ok, err] = await tryFn(() => cache.clear(key));
4086
+ if (ok) {
4087
+ return [true, null];
4088
+ }
4089
+ lastError = err;
4090
+ if (err.name === "NoSuchKey" || err.code === "NoSuchKey") {
4091
+ return [true, null];
4092
+ }
4093
+ if (attempt < this.config.retryAttempts - 1) {
4094
+ const delay = this.config.retryDelay * Math.pow(2, attempt);
4095
+ await new Promise((resolve) => setTimeout(resolve, delay));
3853
4096
  }
3854
4097
  }
4098
+ return [false, lastError];
3855
4099
  }
3856
4100
  async generateCacheKey(resource, action, params = {}, partition = null, partitionValues = null) {
3857
4101
  const keyParts = [
@@ -3867,14 +4111,14 @@ class CachePlugin extends Plugin {
3867
4111
  }
3868
4112
  }
3869
4113
  if (Object.keys(params).length > 0) {
3870
- const paramsHash = await this.hashParams(params);
4114
+ const paramsHash = this.hashParams(params);
3871
4115
  keyParts.push(paramsHash);
3872
4116
  }
3873
4117
  return path.join(...keyParts) + ".json.gz";
3874
4118
  }
3875
- async hashParams(params) {
3876
- const sortedParams = Object.keys(params).sort().map((key) => `${key}:${JSON.stringify(params[key])}`).join("|") || "empty";
3877
- return await sha256(sortedParams);
4119
+ hashParams(params) {
4120
+ const serialized = jsonStableStringify(params) || "empty";
4121
+ return crypto.createHash("md5").update(serialized).digest("hex").substring(0, 16);
3878
4122
  }
3879
4123
  // Utility methods
3880
4124
  async getCacheStats() {
@@ -3899,50 +4143,48 @@ class CachePlugin extends Plugin {
3899
4143
  if (!resource) {
3900
4144
  throw new Error(`Resource '${resourceName}' not found`);
3901
4145
  }
3902
- const { includePartitions = true } = options;
4146
+ const { includePartitions = true, sampleSize = 100 } = options;
3903
4147
  if (this.driver instanceof PartitionAwareFilesystemCache && resource.warmPartitionCache) {
3904
4148
  const partitionNames = resource.config.partitions ? Object.keys(resource.config.partitions) : [];
3905
4149
  return await resource.warmPartitionCache(partitionNames, options);
3906
4150
  }
3907
- await resource.getAll();
3908
- if (includePartitions && resource.config.partitions) {
4151
+ let offset = 0;
4152
+ const pageSize = 100;
4153
+ const sampledRecords = [];
4154
+ while (sampledRecords.length < sampleSize) {
4155
+ const [ok, err, pageResult] = await tryFn(() => resource.page({ offset, size: pageSize }));
4156
+ if (!ok || !pageResult) {
4157
+ break;
4158
+ }
4159
+ const pageItems = Array.isArray(pageResult) ? pageResult : pageResult.items || [];
4160
+ if (pageItems.length === 0) {
4161
+ break;
4162
+ }
4163
+ sampledRecords.push(...pageItems);
4164
+ offset += pageSize;
4165
+ }
4166
+ if (includePartitions && resource.config.partitions && sampledRecords.length > 0) {
3909
4167
  for (const [partitionName, partitionDef] of Object.entries(resource.config.partitions)) {
3910
4168
  if (partitionDef.fields) {
3911
- const allRecords = await resource.getAll();
3912
- const recordsArray = Array.isArray(allRecords) ? allRecords : [];
3913
- const partitionValues = /* @__PURE__ */ new Set();
3914
- for (const record of recordsArray.slice(0, 10)) {
4169
+ const partitionValuesSet = /* @__PURE__ */ new Set();
4170
+ for (const record of sampledRecords) {
3915
4171
  const values = this.getPartitionValues(record, resource);
3916
4172
  if (values[partitionName]) {
3917
- partitionValues.add(JSON.stringify(values[partitionName]));
4173
+ partitionValuesSet.add(JSON.stringify(values[partitionName]));
3918
4174
  }
3919
4175
  }
3920
- for (const partitionValueStr of partitionValues) {
3921
- const partitionValues2 = JSON.parse(partitionValueStr);
3922
- await resource.list({ partition: partitionName, partitionValues: partitionValues2 });
4176
+ for (const partitionValueStr of partitionValuesSet) {
4177
+ const partitionValues = JSON.parse(partitionValueStr);
4178
+ await tryFn(() => resource.list({ partition: partitionName, partitionValues }));
3923
4179
  }
3924
4180
  }
3925
4181
  }
3926
4182
  }
3927
- }
3928
- // Partition-specific methods
3929
- async getPartitionCacheStats(resourceName, partition = null) {
3930
- if (!(this.driver instanceof PartitionAwareFilesystemCache)) {
3931
- throw new Error("Partition cache statistics are only available with PartitionAwareFilesystemCache");
3932
- }
3933
- return await this.driver.getPartitionStats(resourceName, partition);
3934
- }
3935
- async getCacheRecommendations(resourceName) {
3936
- if (!(this.driver instanceof PartitionAwareFilesystemCache)) {
3937
- throw new Error("Cache recommendations are only available with PartitionAwareFilesystemCache");
3938
- }
3939
- return await this.driver.getCacheRecommendations(resourceName);
3940
- }
3941
- async clearPartitionCache(resourceName, partition, partitionValues = {}) {
3942
- if (!(this.driver instanceof PartitionAwareFilesystemCache)) {
3943
- throw new Error("Partition cache clearing is only available with PartitionAwareFilesystemCache");
3944
- }
3945
- return await this.driver.clearPartition(resourceName, partition, partitionValues);
4183
+ return {
4184
+ resourceName,
4185
+ recordsSampled: sampledRecords.length,
4186
+ partitionsWarmed: includePartitions && resource.config.partitions ? Object.keys(resource.config.partitions).length : 0
4187
+ };
3946
4188
  }
3947
4189
  async analyzeCacheUsage() {
3948
4190
  if (!(this.driver instanceof PartitionAwareFilesystemCache)) {
@@ -3959,6 +4201,9 @@ class CachePlugin extends Plugin {
3959
4201
  }
3960
4202
  };
3961
4203
  for (const [resourceName, resource] of Object.entries(this.database.resources)) {
4204
+ if (!this.shouldCacheResource(resourceName)) {
4205
+ continue;
4206
+ }
3962
4207
  try {
3963
4208
  analysis.resourceStats[resourceName] = await this.driver.getPartitionStats(resourceName);
3964
4209
  analysis.recommendations[resourceName] = await this.driver.getCacheRecommendations(resourceName);
@@ -4050,62 +4295,807 @@ const CostsPlugin = {
4050
4295
  }
4051
4296
  };
4052
4297
 
4053
- class FullTextPlugin extends Plugin {
4298
+ class EventualConsistencyPlugin extends Plugin {
4054
4299
  constructor(options = {}) {
4055
- super();
4056
- this.indexResource = null;
4300
+ super(options);
4301
+ if (!options.resource) {
4302
+ throw new Error("EventualConsistencyPlugin requires 'resource' option");
4303
+ }
4304
+ if (!options.field) {
4305
+ throw new Error("EventualConsistencyPlugin requires 'field' option");
4306
+ }
4307
+ const detectedTimezone = this._detectTimezone();
4057
4308
  this.config = {
4058
- minWordLength: options.minWordLength || 3,
4059
- maxResults: options.maxResults || 100,
4060
- ...options
4309
+ resource: options.resource,
4310
+ field: options.field,
4311
+ cohort: {
4312
+ timezone: options.cohort?.timezone || detectedTimezone
4313
+ },
4314
+ reducer: options.reducer || ((transactions) => {
4315
+ let baseValue = 0;
4316
+ for (const t of transactions) {
4317
+ if (t.operation === "set") {
4318
+ baseValue = t.value;
4319
+ } else if (t.operation === "add") {
4320
+ baseValue += t.value;
4321
+ } else if (t.operation === "sub") {
4322
+ baseValue -= t.value;
4323
+ }
4324
+ }
4325
+ return baseValue;
4326
+ }),
4327
+ consolidationInterval: options.consolidationInterval ?? 300,
4328
+ // 5 minutes (in seconds)
4329
+ consolidationConcurrency: options.consolidationConcurrency || 5,
4330
+ consolidationWindow: options.consolidationWindow || 24,
4331
+ // Hours to look back for pending transactions (watermark)
4332
+ autoConsolidate: options.autoConsolidate !== false,
4333
+ lateArrivalStrategy: options.lateArrivalStrategy || "warn",
4334
+ // 'ignore', 'warn', 'process'
4335
+ batchTransactions: options.batchTransactions || false,
4336
+ // CAUTION: Not safe in distributed environments! Loses data on container crash
4337
+ batchSize: options.batchSize || 100,
4338
+ mode: options.mode || "async",
4339
+ // 'async' or 'sync'
4340
+ lockTimeout: options.lockTimeout || 300,
4341
+ // 5 minutes (in seconds, configurable)
4342
+ transactionRetention: options.transactionRetention || 30,
4343
+ // Days to keep applied transactions
4344
+ gcInterval: options.gcInterval || 86400,
4345
+ // 24 hours (in seconds)
4346
+ verbose: options.verbose || false
4061
4347
  };
4062
- this.indexes = /* @__PURE__ */ new Map();
4063
- }
4064
- async setup(database) {
4065
- this.database = database;
4066
- const [ok, err, indexResource] = await tryFn(() => database.createResource({
4067
- name: "fulltext_indexes",
4068
- attributes: {
4069
- id: "string|required",
4070
- resourceName: "string|required",
4071
- fieldName: "string|required",
4072
- word: "string|required",
4073
- recordIds: "json|required",
4074
- // Array of record IDs containing this word
4075
- count: "number|required",
4076
- lastUpdated: "string|required"
4077
- }
4078
- }));
4079
- this.indexResource = ok ? indexResource : database.resources.fulltext_indexes;
4080
- await this.loadIndexes();
4081
- this.installDatabaseHooks();
4082
- this.installIndexingHooks();
4083
- }
4084
- async start() {
4348
+ this.transactionResource = null;
4349
+ this.targetResource = null;
4350
+ this.consolidationTimer = null;
4351
+ this.gcTimer = null;
4352
+ this.pendingTransactions = /* @__PURE__ */ new Map();
4353
+ if (this.config.batchTransactions && !this.config.verbose) {
4354
+ console.warn(
4355
+ `[EventualConsistency] WARNING: batchTransactions is enabled. This stores transactions in memory and will lose data if container crashes. Not recommended for distributed/production environments. Set verbose: true to suppress this warning.`
4356
+ );
4357
+ }
4358
+ if (this.config.verbose && !options.cohort?.timezone) {
4359
+ console.log(
4360
+ `[EventualConsistency] Auto-detected timezone: ${this.config.cohort.timezone} (from ${process.env.TZ ? "TZ env var" : "system Intl API"})`
4361
+ );
4362
+ }
4085
4363
  }
4086
- async stop() {
4087
- await this.saveIndexes();
4088
- this.removeDatabaseHooks();
4364
+ async onSetup() {
4365
+ this.targetResource = this.database.resources[this.config.resource];
4366
+ if (!this.targetResource) {
4367
+ this.deferredSetup = true;
4368
+ this.watchForResource();
4369
+ return;
4370
+ }
4371
+ await this.completeSetup();
4089
4372
  }
4090
- async loadIndexes() {
4091
- if (!this.indexResource) return;
4092
- const [ok, err, allIndexes] = await tryFn(() => this.indexResource.getAll());
4093
- if (ok) {
4094
- for (const indexRecord of allIndexes) {
4095
- const key = `${indexRecord.resourceName}:${indexRecord.fieldName}:${indexRecord.word}`;
4096
- this.indexes.set(key, {
4097
- recordIds: indexRecord.recordIds || [],
4098
- count: indexRecord.count || 0
4099
- });
4373
+ watchForResource() {
4374
+ const hookCallback = async ({ resource, config }) => {
4375
+ if (config.name === this.config.resource && this.deferredSetup) {
4376
+ this.targetResource = resource;
4377
+ this.deferredSetup = false;
4378
+ await this.completeSetup();
4100
4379
  }
4380
+ };
4381
+ this.database.addHook("afterCreateResource", hookCallback);
4382
+ }
4383
+ async completeSetup() {
4384
+ if (!this.targetResource) return;
4385
+ const transactionResourceName = `${this.config.resource}_transactions_${this.config.field}`;
4386
+ const partitionConfig = this.createPartitionConfig();
4387
+ const [ok, err, transactionResource] = await tryFn(
4388
+ () => this.database.createResource({
4389
+ name: transactionResourceName,
4390
+ attributes: {
4391
+ id: "string|required",
4392
+ originalId: "string|required",
4393
+ field: "string|required",
4394
+ value: "number|required",
4395
+ operation: "string|required",
4396
+ // 'set', 'add', or 'sub'
4397
+ timestamp: "string|required",
4398
+ cohortDate: "string|required",
4399
+ // For daily partitioning
4400
+ cohortHour: "string|required",
4401
+ // For hourly partitioning
4402
+ cohortMonth: "string|optional",
4403
+ // For monthly partitioning
4404
+ source: "string|optional",
4405
+ applied: "boolean|optional"
4406
+ // Track if transaction was applied
4407
+ },
4408
+ behavior: "body-overflow",
4409
+ timestamps: true,
4410
+ partitions: partitionConfig,
4411
+ asyncPartitions: true
4412
+ // Use async partitions for better performance
4413
+ })
4414
+ );
4415
+ if (!ok && !this.database.resources[transactionResourceName]) {
4416
+ throw new Error(`Failed to create transaction resource: ${err?.message}`);
4417
+ }
4418
+ this.transactionResource = ok ? transactionResource : this.database.resources[transactionResourceName];
4419
+ const lockResourceName = `${this.config.resource}_consolidation_locks_${this.config.field}`;
4420
+ const [lockOk, lockErr, lockResource] = await tryFn(
4421
+ () => this.database.createResource({
4422
+ name: lockResourceName,
4423
+ attributes: {
4424
+ id: "string|required",
4425
+ lockedAt: "number|required",
4426
+ workerId: "string|optional"
4427
+ },
4428
+ behavior: "body-only",
4429
+ timestamps: false
4430
+ })
4431
+ );
4432
+ if (!lockOk && !this.database.resources[lockResourceName]) {
4433
+ throw new Error(`Failed to create lock resource: ${lockErr?.message}`);
4434
+ }
4435
+ this.lockResource = lockOk ? lockResource : this.database.resources[lockResourceName];
4436
+ this.addHelperMethods();
4437
+ if (this.config.autoConsolidate) {
4438
+ this.startConsolidationTimer();
4101
4439
  }
4440
+ this.startGarbageCollectionTimer();
4102
4441
  }
4103
- async saveIndexes() {
4104
- if (!this.indexResource) return;
4105
- const [ok, err] = await tryFn(async () => {
4106
- const existingIndexes = await this.indexResource.getAll();
4107
- for (const index of existingIndexes) {
4108
- await this.indexResource.delete(index.id);
4442
+ async onStart() {
4443
+ if (this.deferredSetup) {
4444
+ return;
4445
+ }
4446
+ this.emit("eventual-consistency.started", {
4447
+ resource: this.config.resource,
4448
+ field: this.config.field,
4449
+ cohort: this.config.cohort
4450
+ });
4451
+ }
4452
+ async onStop() {
4453
+ if (this.consolidationTimer) {
4454
+ clearInterval(this.consolidationTimer);
4455
+ this.consolidationTimer = null;
4456
+ }
4457
+ if (this.gcTimer) {
4458
+ clearInterval(this.gcTimer);
4459
+ this.gcTimer = null;
4460
+ }
4461
+ await this.flushPendingTransactions();
4462
+ this.emit("eventual-consistency.stopped", {
4463
+ resource: this.config.resource,
4464
+ field: this.config.field
4465
+ });
4466
+ }
4467
+ createPartitionConfig() {
4468
+ const partitions = {
4469
+ byHour: {
4470
+ fields: {
4471
+ cohortHour: "string"
4472
+ }
4473
+ },
4474
+ byDay: {
4475
+ fields: {
4476
+ cohortDate: "string"
4477
+ }
4478
+ },
4479
+ byMonth: {
4480
+ fields: {
4481
+ cohortMonth: "string"
4482
+ }
4483
+ }
4484
+ };
4485
+ return partitions;
4486
+ }
4487
+ /**
4488
+ * Auto-detect timezone from environment or system
4489
+ * @private
4490
+ */
4491
+ _detectTimezone() {
4492
+ if (process.env.TZ) {
4493
+ return process.env.TZ;
4494
+ }
4495
+ try {
4496
+ const systemTimezone = Intl.DateTimeFormat().resolvedOptions().timeZone;
4497
+ if (systemTimezone) {
4498
+ return systemTimezone;
4499
+ }
4500
+ } catch (err) {
4501
+ }
4502
+ return "UTC";
4503
+ }
4504
+ /**
4505
+ * Helper method to resolve field and plugin from arguments
4506
+ * Supports both single-field (field, value) and multi-field (field, value) signatures
4507
+ * @private
4508
+ */
4509
+ _resolveFieldAndPlugin(resource, fieldOrValue, value) {
4510
+ const hasMultipleFields = Object.keys(resource._eventualConsistencyPlugins).length > 1;
4511
+ if (hasMultipleFields && value === void 0) {
4512
+ throw new Error(`Multiple fields have eventual consistency. Please specify the field explicitly.`);
4513
+ }
4514
+ const field = value !== void 0 ? fieldOrValue : this.config.field;
4515
+ const actualValue = value !== void 0 ? value : fieldOrValue;
4516
+ const fieldPlugin = resource._eventualConsistencyPlugins[field];
4517
+ if (!fieldPlugin) {
4518
+ throw new Error(`No eventual consistency plugin found for field "${field}"`);
4519
+ }
4520
+ return { field, value: actualValue, plugin: fieldPlugin };
4521
+ }
4522
+ /**
4523
+ * Helper method to perform atomic consolidation in sync mode
4524
+ * @private
4525
+ */
4526
+ async _syncModeConsolidate(id, field) {
4527
+ const consolidatedValue = await this.consolidateRecord(id);
4528
+ await this.targetResource.update(id, {
4529
+ [field]: consolidatedValue
4530
+ });
4531
+ return consolidatedValue;
4532
+ }
4533
+ /**
4534
+ * Create synthetic 'set' transaction from current value
4535
+ * @private
4536
+ */
4537
+ _createSyntheticSetTransaction(currentValue) {
4538
+ return {
4539
+ id: "__synthetic__",
4540
+ operation: "set",
4541
+ value: currentValue,
4542
+ timestamp: (/* @__PURE__ */ new Date(0)).toISOString(),
4543
+ synthetic: true
4544
+ };
4545
+ }
4546
+ addHelperMethods() {
4547
+ const resource = this.targetResource;
4548
+ const defaultField = this.config.field;
4549
+ const plugin = this;
4550
+ if (!resource._eventualConsistencyPlugins) {
4551
+ resource._eventualConsistencyPlugins = {};
4552
+ }
4553
+ resource._eventualConsistencyPlugins[defaultField] = plugin;
4554
+ resource.set = async (id, fieldOrValue, value) => {
4555
+ const { field, value: actualValue, plugin: fieldPlugin } = plugin._resolveFieldAndPlugin(resource, fieldOrValue, value);
4556
+ await fieldPlugin.createTransaction({
4557
+ originalId: id,
4558
+ operation: "set",
4559
+ value: actualValue,
4560
+ source: "set"
4561
+ });
4562
+ if (fieldPlugin.config.mode === "sync") {
4563
+ return await fieldPlugin._syncModeConsolidate(id, field);
4564
+ }
4565
+ return actualValue;
4566
+ };
4567
+ resource.add = async (id, fieldOrAmount, amount) => {
4568
+ const { field, value: actualAmount, plugin: fieldPlugin } = plugin._resolveFieldAndPlugin(resource, fieldOrAmount, amount);
4569
+ await fieldPlugin.createTransaction({
4570
+ originalId: id,
4571
+ operation: "add",
4572
+ value: actualAmount,
4573
+ source: "add"
4574
+ });
4575
+ if (fieldPlugin.config.mode === "sync") {
4576
+ return await fieldPlugin._syncModeConsolidate(id, field);
4577
+ }
4578
+ const currentValue = await fieldPlugin.getConsolidatedValue(id);
4579
+ return currentValue + actualAmount;
4580
+ };
4581
+ resource.sub = async (id, fieldOrAmount, amount) => {
4582
+ const { field, value: actualAmount, plugin: fieldPlugin } = plugin._resolveFieldAndPlugin(resource, fieldOrAmount, amount);
4583
+ await fieldPlugin.createTransaction({
4584
+ originalId: id,
4585
+ operation: "sub",
4586
+ value: actualAmount,
4587
+ source: "sub"
4588
+ });
4589
+ if (fieldPlugin.config.mode === "sync") {
4590
+ return await fieldPlugin._syncModeConsolidate(id, field);
4591
+ }
4592
+ const currentValue = await fieldPlugin.getConsolidatedValue(id);
4593
+ return currentValue - actualAmount;
4594
+ };
4595
+ resource.consolidate = async (id, field) => {
4596
+ const hasMultipleFields = Object.keys(resource._eventualConsistencyPlugins).length > 1;
4597
+ if (hasMultipleFields && !field) {
4598
+ throw new Error(`Multiple fields have eventual consistency. Please specify the field: consolidate(id, field)`);
4599
+ }
4600
+ const actualField = field || defaultField;
4601
+ const fieldPlugin = resource._eventualConsistencyPlugins[actualField];
4602
+ if (!fieldPlugin) {
4603
+ throw new Error(`No eventual consistency plugin found for field "${actualField}"`);
4604
+ }
4605
+ return await fieldPlugin.consolidateRecord(id);
4606
+ };
4607
+ resource.getConsolidatedValue = async (id, fieldOrOptions, options) => {
4608
+ if (typeof fieldOrOptions === "string") {
4609
+ const field = fieldOrOptions;
4610
+ const fieldPlugin = resource._eventualConsistencyPlugins[field] || plugin;
4611
+ return await fieldPlugin.getConsolidatedValue(id, options || {});
4612
+ } else {
4613
+ return await plugin.getConsolidatedValue(id, fieldOrOptions || {});
4614
+ }
4615
+ };
4616
+ }
4617
+ async createTransaction(data) {
4618
+ const now = /* @__PURE__ */ new Date();
4619
+ const cohortInfo = this.getCohortInfo(now);
4620
+ const watermarkMs = this.config.consolidationWindow * 60 * 60 * 1e3;
4621
+ const watermarkTime = now.getTime() - watermarkMs;
4622
+ const cohortHourDate = /* @__PURE__ */ new Date(cohortInfo.hour + ":00:00Z");
4623
+ if (cohortHourDate.getTime() < watermarkTime) {
4624
+ const hoursLate = Math.floor((now.getTime() - cohortHourDate.getTime()) / (60 * 60 * 1e3));
4625
+ if (this.config.lateArrivalStrategy === "ignore") {
4626
+ if (this.config.verbose) {
4627
+ console.warn(
4628
+ `[EventualConsistency] Late arrival ignored: transaction for ${cohortInfo.hour} is ${hoursLate}h late (watermark: ${this.config.consolidationWindow}h)`
4629
+ );
4630
+ }
4631
+ return null;
4632
+ } else if (this.config.lateArrivalStrategy === "warn") {
4633
+ console.warn(
4634
+ `[EventualConsistency] Late arrival detected: transaction for ${cohortInfo.hour} is ${hoursLate}h late (watermark: ${this.config.consolidationWindow}h). Processing anyway, but consolidation may not pick it up.`
4635
+ );
4636
+ }
4637
+ }
4638
+ const transaction = {
4639
+ id: idGenerator(),
4640
+ // Use nanoid for guaranteed uniqueness
4641
+ originalId: data.originalId,
4642
+ field: this.config.field,
4643
+ value: data.value || 0,
4644
+ operation: data.operation || "set",
4645
+ timestamp: now.toISOString(),
4646
+ cohortDate: cohortInfo.date,
4647
+ cohortHour: cohortInfo.hour,
4648
+ cohortMonth: cohortInfo.month,
4649
+ source: data.source || "unknown",
4650
+ applied: false
4651
+ };
4652
+ if (this.config.batchTransactions) {
4653
+ this.pendingTransactions.set(transaction.id, transaction);
4654
+ if (this.pendingTransactions.size >= this.config.batchSize) {
4655
+ await this.flushPendingTransactions();
4656
+ }
4657
+ } else {
4658
+ await this.transactionResource.insert(transaction);
4659
+ }
4660
+ return transaction;
4661
+ }
4662
+ async flushPendingTransactions() {
4663
+ if (this.pendingTransactions.size === 0) return;
4664
+ const transactions = Array.from(this.pendingTransactions.values());
4665
+ try {
4666
+ await Promise.all(
4667
+ transactions.map(
4668
+ (transaction) => this.transactionResource.insert(transaction)
4669
+ )
4670
+ );
4671
+ this.pendingTransactions.clear();
4672
+ } catch (error) {
4673
+ console.error("Failed to flush pending transactions:", error);
4674
+ throw error;
4675
+ }
4676
+ }
4677
+ getCohortInfo(date) {
4678
+ const tz = this.config.cohort.timezone;
4679
+ const offset = this.getTimezoneOffset(tz);
4680
+ const localDate = new Date(date.getTime() + offset);
4681
+ const year = localDate.getFullYear();
4682
+ const month = String(localDate.getMonth() + 1).padStart(2, "0");
4683
+ const day = String(localDate.getDate()).padStart(2, "0");
4684
+ const hour = String(localDate.getHours()).padStart(2, "0");
4685
+ return {
4686
+ date: `${year}-${month}-${day}`,
4687
+ hour: `${year}-${month}-${day}T${hour}`,
4688
+ // ISO-like format for hour partition
4689
+ month: `${year}-${month}`
4690
+ };
4691
+ }
4692
+ getTimezoneOffset(timezone) {
4693
+ try {
4694
+ const now = /* @__PURE__ */ new Date();
4695
+ const utcDate = new Date(now.toLocaleString("en-US", { timeZone: "UTC" }));
4696
+ const tzDate = new Date(now.toLocaleString("en-US", { timeZone: timezone }));
4697
+ return tzDate.getTime() - utcDate.getTime();
4698
+ } catch (err) {
4699
+ const offsets = {
4700
+ "UTC": 0,
4701
+ "America/New_York": -5 * 36e5,
4702
+ "America/Chicago": -6 * 36e5,
4703
+ "America/Denver": -7 * 36e5,
4704
+ "America/Los_Angeles": -8 * 36e5,
4705
+ "America/Sao_Paulo": -3 * 36e5,
4706
+ "Europe/London": 0,
4707
+ "Europe/Paris": 1 * 36e5,
4708
+ "Europe/Berlin": 1 * 36e5,
4709
+ "Asia/Tokyo": 9 * 36e5,
4710
+ "Asia/Shanghai": 8 * 36e5,
4711
+ "Australia/Sydney": 10 * 36e5
4712
+ };
4713
+ if (this.config.verbose && !offsets[timezone]) {
4714
+ console.warn(
4715
+ `[EventualConsistency] Unknown timezone '${timezone}', using UTC. Consider using a valid IANA timezone (e.g., 'America/New_York')`
4716
+ );
4717
+ }
4718
+ return offsets[timezone] || 0;
4719
+ }
4720
+ }
4721
+ startConsolidationTimer() {
4722
+ const intervalMs = this.config.consolidationInterval * 1e3;
4723
+ this.consolidationTimer = setInterval(async () => {
4724
+ await this.runConsolidation();
4725
+ }, intervalMs);
4726
+ }
4727
+ async runConsolidation() {
4728
+ try {
4729
+ const now = /* @__PURE__ */ new Date();
4730
+ const hoursToCheck = this.config.consolidationWindow || 24;
4731
+ const cohortHours = [];
4732
+ for (let i = 0; i < hoursToCheck; i++) {
4733
+ const date = new Date(now.getTime() - i * 60 * 60 * 1e3);
4734
+ const cohortInfo = this.getCohortInfo(date);
4735
+ cohortHours.push(cohortInfo.hour);
4736
+ }
4737
+ const transactionsByHour = await Promise.all(
4738
+ cohortHours.map(async (cohortHour) => {
4739
+ const [ok, err, txns] = await tryFn(
4740
+ () => this.transactionResource.query({
4741
+ cohortHour,
4742
+ applied: false
4743
+ })
4744
+ );
4745
+ return ok ? txns : [];
4746
+ })
4747
+ );
4748
+ const transactions = transactionsByHour.flat();
4749
+ if (transactions.length === 0) {
4750
+ if (this.config.verbose) {
4751
+ console.log(`[EventualConsistency] No pending transactions to consolidate`);
4752
+ }
4753
+ return;
4754
+ }
4755
+ const uniqueIds = [...new Set(transactions.map((t) => t.originalId))];
4756
+ const { results, errors } = await promisePool.PromisePool.for(uniqueIds).withConcurrency(this.config.consolidationConcurrency).process(async (id) => {
4757
+ return await this.consolidateRecord(id);
4758
+ });
4759
+ if (errors && errors.length > 0) {
4760
+ console.error(`Consolidation completed with ${errors.length} errors:`, errors);
4761
+ }
4762
+ this.emit("eventual-consistency.consolidated", {
4763
+ resource: this.config.resource,
4764
+ field: this.config.field,
4765
+ recordCount: uniqueIds.length,
4766
+ successCount: results.length,
4767
+ errorCount: errors.length
4768
+ });
4769
+ } catch (error) {
4770
+ console.error("Consolidation error:", error);
4771
+ this.emit("eventual-consistency.consolidation-error", error);
4772
+ }
4773
+ }
4774
+ async consolidateRecord(originalId) {
4775
+ await this.cleanupStaleLocks();
4776
+ const lockId = `lock-${originalId}`;
4777
+ const [lockAcquired, lockErr, lock] = await tryFn(
4778
+ () => this.lockResource.insert({
4779
+ id: lockId,
4780
+ lockedAt: Date.now(),
4781
+ workerId: process.pid ? String(process.pid) : "unknown"
4782
+ })
4783
+ );
4784
+ if (!lockAcquired) {
4785
+ if (this.config.verbose) {
4786
+ console.log(`[EventualConsistency] Lock for ${originalId} already held, skipping`);
4787
+ }
4788
+ const [recordOk, recordErr, record] = await tryFn(
4789
+ () => this.targetResource.get(originalId)
4790
+ );
4791
+ return recordOk && record ? record[this.config.field] || 0 : 0;
4792
+ }
4793
+ try {
4794
+ const [recordOk, recordErr, record] = await tryFn(
4795
+ () => this.targetResource.get(originalId)
4796
+ );
4797
+ const currentValue = recordOk && record ? record[this.config.field] || 0 : 0;
4798
+ const [ok, err, transactions] = await tryFn(
4799
+ () => this.transactionResource.query({
4800
+ originalId,
4801
+ applied: false
4802
+ })
4803
+ );
4804
+ if (!ok || !transactions || transactions.length === 0) {
4805
+ return currentValue;
4806
+ }
4807
+ transactions.sort(
4808
+ (a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()
4809
+ );
4810
+ const hasSetOperation = transactions.some((t) => t.operation === "set");
4811
+ if (currentValue !== 0 && !hasSetOperation) {
4812
+ transactions.unshift(this._createSyntheticSetTransaction(currentValue));
4813
+ }
4814
+ const consolidatedValue = this.config.reducer(transactions);
4815
+ const [updateOk, updateErr] = await tryFn(
4816
+ () => this.targetResource.update(originalId, {
4817
+ [this.config.field]: consolidatedValue
4818
+ })
4819
+ );
4820
+ if (updateOk) {
4821
+ const transactionsToUpdate = transactions.filter((txn) => txn.id !== "__synthetic__");
4822
+ const { results, errors } = await promisePool.PromisePool.for(transactionsToUpdate).withConcurrency(10).process(async (txn) => {
4823
+ const [ok2, err2] = await tryFn(
4824
+ () => this.transactionResource.update(txn.id, { applied: true })
4825
+ );
4826
+ if (!ok2 && this.config.verbose) {
4827
+ console.warn(`[EventualConsistency] Failed to mark transaction ${txn.id} as applied:`, err2?.message);
4828
+ }
4829
+ return ok2;
4830
+ });
4831
+ if (errors && errors.length > 0 && this.config.verbose) {
4832
+ console.warn(`[EventualConsistency] ${errors.length} transactions failed to mark as applied`);
4833
+ }
4834
+ }
4835
+ return consolidatedValue;
4836
+ } finally {
4837
+ const [lockReleased, lockReleaseErr] = await tryFn(() => this.lockResource.delete(lockId));
4838
+ if (!lockReleased && this.config.verbose) {
4839
+ console.warn(`[EventualConsistency] Failed to release lock ${lockId}:`, lockReleaseErr?.message);
4840
+ }
4841
+ }
4842
+ }
4843
+ async getConsolidatedValue(originalId, options = {}) {
4844
+ const includeApplied = options.includeApplied || false;
4845
+ const startDate = options.startDate;
4846
+ const endDate = options.endDate;
4847
+ const query = { originalId };
4848
+ if (!includeApplied) {
4849
+ query.applied = false;
4850
+ }
4851
+ const [ok, err, transactions] = await tryFn(
4852
+ () => this.transactionResource.query(query)
4853
+ );
4854
+ if (!ok || !transactions || transactions.length === 0) {
4855
+ const [recordOk2, recordErr2, record2] = await tryFn(
4856
+ () => this.targetResource.get(originalId)
4857
+ );
4858
+ if (recordOk2 && record2) {
4859
+ return record2[this.config.field] || 0;
4860
+ }
4861
+ return 0;
4862
+ }
4863
+ let filtered = transactions;
4864
+ if (startDate || endDate) {
4865
+ filtered = transactions.filter((t) => {
4866
+ const timestamp = new Date(t.timestamp);
4867
+ if (startDate && timestamp < new Date(startDate)) return false;
4868
+ if (endDate && timestamp > new Date(endDate)) return false;
4869
+ return true;
4870
+ });
4871
+ }
4872
+ const [recordOk, recordErr, record] = await tryFn(
4873
+ () => this.targetResource.get(originalId)
4874
+ );
4875
+ const currentValue = recordOk && record ? record[this.config.field] || 0 : 0;
4876
+ const hasSetOperation = filtered.some((t) => t.operation === "set");
4877
+ if (currentValue !== 0 && !hasSetOperation) {
4878
+ filtered.unshift(this._createSyntheticSetTransaction(currentValue));
4879
+ }
4880
+ filtered.sort(
4881
+ (a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()
4882
+ );
4883
+ return this.config.reducer(filtered);
4884
+ }
4885
+ // Helper method to get cohort statistics
4886
+ async getCohortStats(cohortDate) {
4887
+ const [ok, err, transactions] = await tryFn(
4888
+ () => this.transactionResource.query({
4889
+ cohortDate
4890
+ })
4891
+ );
4892
+ if (!ok) return null;
4893
+ const stats = {
4894
+ date: cohortDate,
4895
+ transactionCount: transactions.length,
4896
+ totalValue: 0,
4897
+ byOperation: { set: 0, add: 0, sub: 0 },
4898
+ byOriginalId: {}
4899
+ };
4900
+ for (const txn of transactions) {
4901
+ stats.totalValue += txn.value || 0;
4902
+ stats.byOperation[txn.operation] = (stats.byOperation[txn.operation] || 0) + 1;
4903
+ if (!stats.byOriginalId[txn.originalId]) {
4904
+ stats.byOriginalId[txn.originalId] = {
4905
+ count: 0,
4906
+ value: 0
4907
+ };
4908
+ }
4909
+ stats.byOriginalId[txn.originalId].count++;
4910
+ stats.byOriginalId[txn.originalId].value += txn.value || 0;
4911
+ }
4912
+ return stats;
4913
+ }
4914
+ /**
4915
+ * Clean up stale locks that exceed the configured timeout
4916
+ * Uses distributed locking to prevent multiple containers from cleaning simultaneously
4917
+ */
4918
+ async cleanupStaleLocks() {
4919
+ const now = Date.now();
4920
+ const lockTimeoutMs = this.config.lockTimeout * 1e3;
4921
+ const cutoffTime = now - lockTimeoutMs;
4922
+ const cleanupLockId = `lock-cleanup-${this.config.resource}-${this.config.field}`;
4923
+ const [lockAcquired] = await tryFn(
4924
+ () => this.lockResource.insert({
4925
+ id: cleanupLockId,
4926
+ lockedAt: Date.now(),
4927
+ workerId: process.pid ? String(process.pid) : "unknown"
4928
+ })
4929
+ );
4930
+ if (!lockAcquired) {
4931
+ if (this.config.verbose) {
4932
+ console.log(`[EventualConsistency] Lock cleanup already running in another container`);
4933
+ }
4934
+ return;
4935
+ }
4936
+ try {
4937
+ const [ok, err, locks] = await tryFn(() => this.lockResource.list());
4938
+ if (!ok || !locks || locks.length === 0) return;
4939
+ const staleLocks = locks.filter(
4940
+ (lock) => lock.id !== cleanupLockId && lock.lockedAt < cutoffTime
4941
+ );
4942
+ if (staleLocks.length === 0) return;
4943
+ if (this.config.verbose) {
4944
+ console.log(`[EventualConsistency] Cleaning up ${staleLocks.length} stale locks`);
4945
+ }
4946
+ const { results, errors } = await promisePool.PromisePool.for(staleLocks).withConcurrency(5).process(async (lock) => {
4947
+ const [deleted] = await tryFn(() => this.lockResource.delete(lock.id));
4948
+ return deleted;
4949
+ });
4950
+ if (errors && errors.length > 0 && this.config.verbose) {
4951
+ console.warn(`[EventualConsistency] ${errors.length} stale locks failed to delete`);
4952
+ }
4953
+ } catch (error) {
4954
+ if (this.config.verbose) {
4955
+ console.warn(`[EventualConsistency] Error cleaning up stale locks:`, error.message);
4956
+ }
4957
+ } finally {
4958
+ await tryFn(() => this.lockResource.delete(cleanupLockId));
4959
+ }
4960
+ }
4961
+ /**
4962
+ * Start garbage collection timer for old applied transactions
4963
+ */
4964
+ startGarbageCollectionTimer() {
4965
+ const gcIntervalMs = this.config.gcInterval * 1e3;
4966
+ this.gcTimer = setInterval(async () => {
4967
+ await this.runGarbageCollection();
4968
+ }, gcIntervalMs);
4969
+ }
4970
+ /**
4971
+ * Delete old applied transactions based on retention policy
4972
+ * Uses distributed locking to prevent multiple containers from running GC simultaneously
4973
+ */
4974
+ async runGarbageCollection() {
4975
+ const gcLockId = `lock-gc-${this.config.resource}-${this.config.field}`;
4976
+ const [lockAcquired] = await tryFn(
4977
+ () => this.lockResource.insert({
4978
+ id: gcLockId,
4979
+ lockedAt: Date.now(),
4980
+ workerId: process.pid ? String(process.pid) : "unknown"
4981
+ })
4982
+ );
4983
+ if (!lockAcquired) {
4984
+ if (this.config.verbose) {
4985
+ console.log(`[EventualConsistency] GC already running in another container`);
4986
+ }
4987
+ return;
4988
+ }
4989
+ try {
4990
+ const now = Date.now();
4991
+ const retentionMs = this.config.transactionRetention * 24 * 60 * 60 * 1e3;
4992
+ const cutoffDate = new Date(now - retentionMs);
4993
+ const cutoffIso = cutoffDate.toISOString();
4994
+ if (this.config.verbose) {
4995
+ console.log(`[EventualConsistency] Running GC for transactions older than ${cutoffIso} (${this.config.transactionRetention} days)`);
4996
+ }
4997
+ const cutoffMonth = cutoffDate.toISOString().substring(0, 7);
4998
+ const [ok, err, oldTransactions] = await tryFn(
4999
+ () => this.transactionResource.query({
5000
+ applied: true,
5001
+ timestamp: { "<": cutoffIso }
5002
+ })
5003
+ );
5004
+ if (!ok) {
5005
+ if (this.config.verbose) {
5006
+ console.warn(`[EventualConsistency] GC failed to query transactions:`, err?.message);
5007
+ }
5008
+ return;
5009
+ }
5010
+ if (!oldTransactions || oldTransactions.length === 0) {
5011
+ if (this.config.verbose) {
5012
+ console.log(`[EventualConsistency] No old transactions to clean up`);
5013
+ }
5014
+ return;
5015
+ }
5016
+ if (this.config.verbose) {
5017
+ console.log(`[EventualConsistency] Deleting ${oldTransactions.length} old transactions`);
5018
+ }
5019
+ const { results, errors } = await promisePool.PromisePool.for(oldTransactions).withConcurrency(10).process(async (txn) => {
5020
+ const [deleted] = await tryFn(() => this.transactionResource.delete(txn.id));
5021
+ return deleted;
5022
+ });
5023
+ if (this.config.verbose) {
5024
+ console.log(`[EventualConsistency] GC completed: ${results.length} deleted, ${errors.length} errors`);
5025
+ }
5026
+ this.emit("eventual-consistency.gc-completed", {
5027
+ resource: this.config.resource,
5028
+ field: this.config.field,
5029
+ deletedCount: results.length,
5030
+ errorCount: errors.length
5031
+ });
5032
+ } catch (error) {
5033
+ if (this.config.verbose) {
5034
+ console.warn(`[EventualConsistency] GC error:`, error.message);
5035
+ }
5036
+ this.emit("eventual-consistency.gc-error", error);
5037
+ } finally {
5038
+ await tryFn(() => this.lockResource.delete(gcLockId));
5039
+ }
5040
+ }
5041
+ }
5042
+
5043
+ class FullTextPlugin extends Plugin {
5044
+ constructor(options = {}) {
5045
+ super();
5046
+ this.indexResource = null;
5047
+ this.config = {
5048
+ minWordLength: options.minWordLength || 3,
5049
+ maxResults: options.maxResults || 100,
5050
+ ...options
5051
+ };
5052
+ this.indexes = /* @__PURE__ */ new Map();
5053
+ }
5054
+ async setup(database) {
5055
+ this.database = database;
5056
+ const [ok, err, indexResource] = await tryFn(() => database.createResource({
5057
+ name: "plg_fulltext_indexes",
5058
+ attributes: {
5059
+ id: "string|required",
5060
+ resourceName: "string|required",
5061
+ fieldName: "string|required",
5062
+ word: "string|required",
5063
+ recordIds: "json|required",
5064
+ // Array of record IDs containing this word
5065
+ count: "number|required",
5066
+ lastUpdated: "string|required"
5067
+ }
5068
+ }));
5069
+ this.indexResource = ok ? indexResource : database.resources.fulltext_indexes;
5070
+ await this.loadIndexes();
5071
+ this.installDatabaseHooks();
5072
+ this.installIndexingHooks();
5073
+ }
5074
+ async start() {
5075
+ }
5076
+ async stop() {
5077
+ await this.saveIndexes();
5078
+ this.removeDatabaseHooks();
5079
+ }
5080
+ async loadIndexes() {
5081
+ if (!this.indexResource) return;
5082
+ const [ok, err, allIndexes] = await tryFn(() => this.indexResource.getAll());
5083
+ if (ok) {
5084
+ for (const indexRecord of allIndexes) {
5085
+ const key = `${indexRecord.resourceName}:${indexRecord.fieldName}:${indexRecord.word}`;
5086
+ this.indexes.set(key, {
5087
+ recordIds: indexRecord.recordIds || [],
5088
+ count: indexRecord.count || 0
5089
+ });
5090
+ }
5091
+ }
5092
+ }
5093
+ async saveIndexes() {
5094
+ if (!this.indexResource) return;
5095
+ const [ok, err] = await tryFn(async () => {
5096
+ const existingIndexes = await this.indexResource.getAll();
5097
+ for (const index of existingIndexes) {
5098
+ await this.indexResource.delete(index.id);
4109
5099
  }
4110
5100
  for (const [key, data] of this.indexes.entries()) {
4111
5101
  const [resourceName, fieldName, word] = key.split(":");
@@ -4123,7 +5113,7 @@ class FullTextPlugin extends Plugin {
4123
5113
  }
4124
5114
  installDatabaseHooks() {
4125
5115
  this.database.addHook("afterCreateResource", (resource) => {
4126
- if (resource.name !== "fulltext_indexes") {
5116
+ if (resource.name !== "plg_fulltext_indexes") {
4127
5117
  this.installResourceHooks(resource);
4128
5118
  }
4129
5119
  });
@@ -4137,14 +5127,14 @@ class FullTextPlugin extends Plugin {
4137
5127
  }
4138
5128
  this.database.plugins.fulltext = this;
4139
5129
  for (const resource of Object.values(this.database.resources)) {
4140
- if (resource.name === "fulltext_indexes") continue;
5130
+ if (resource.name === "plg_fulltext_indexes") continue;
4141
5131
  this.installResourceHooks(resource);
4142
5132
  }
4143
5133
  if (!this.database._fulltextProxyInstalled) {
4144
5134
  this.database._previousCreateResourceForFullText = this.database.createResource;
4145
5135
  this.database.createResource = async function(...args) {
4146
5136
  const resource = await this._previousCreateResourceForFullText(...args);
4147
- if (this.plugins?.fulltext && resource.name !== "fulltext_indexes") {
5137
+ if (this.plugins?.fulltext && resource.name !== "plg_fulltext_indexes") {
4148
5138
  this.plugins.fulltext.installResourceHooks(resource);
4149
5139
  }
4150
5140
  return resource;
@@ -4152,7 +5142,7 @@ class FullTextPlugin extends Plugin {
4152
5142
  this.database._fulltextProxyInstalled = true;
4153
5143
  }
4154
5144
  for (const resource of Object.values(this.database.resources)) {
4155
- if (resource.name !== "fulltext_indexes") {
5145
+ if (resource.name !== "plg_fulltext_indexes") {
4156
5146
  this.installResourceHooks(resource);
4157
5147
  }
4158
5148
  }
@@ -4395,7 +5385,7 @@ class FullTextPlugin extends Plugin {
4395
5385
  return this._rebuildAllIndexesInternal();
4396
5386
  }
4397
5387
  async _rebuildAllIndexesInternal() {
4398
- const resourceNames = Object.keys(this.database.resources).filter((name) => name !== "fulltext_indexes");
5388
+ const resourceNames = Object.keys(this.database.resources).filter((name) => name !== "plg_fulltext_indexes");
4399
5389
  for (const resourceName of resourceNames) {
4400
5390
  const [ok, err] = await tryFn(() => this.rebuildIndex(resourceName));
4401
5391
  }
@@ -4447,7 +5437,7 @@ class MetricsPlugin extends Plugin {
4447
5437
  if (typeof process !== "undefined" && process.env.NODE_ENV === "test") return;
4448
5438
  const [ok, err] = await tryFn(async () => {
4449
5439
  const [ok1, err1, metricsResource] = await tryFn(() => database.createResource({
4450
- name: "metrics",
5440
+ name: "plg_metrics",
4451
5441
  attributes: {
4452
5442
  id: "string|required",
4453
5443
  type: "string|required",
@@ -4462,9 +5452,9 @@ class MetricsPlugin extends Plugin {
4462
5452
  metadata: "json"
4463
5453
  }
4464
5454
  }));
4465
- this.metricsResource = ok1 ? metricsResource : database.resources.metrics;
5455
+ this.metricsResource = ok1 ? metricsResource : database.resources.plg_metrics;
4466
5456
  const [ok2, err2, errorsResource] = await tryFn(() => database.createResource({
4467
- name: "error_logs",
5457
+ name: "plg_error_logs",
4468
5458
  attributes: {
4469
5459
  id: "string|required",
4470
5460
  resourceName: "string|required",
@@ -4474,9 +5464,9 @@ class MetricsPlugin extends Plugin {
4474
5464
  metadata: "json"
4475
5465
  }
4476
5466
  }));
4477
- this.errorsResource = ok2 ? errorsResource : database.resources.error_logs;
5467
+ this.errorsResource = ok2 ? errorsResource : database.resources.plg_error_logs;
4478
5468
  const [ok3, err3, performanceResource] = await tryFn(() => database.createResource({
4479
- name: "performance_logs",
5469
+ name: "plg_performance_logs",
4480
5470
  attributes: {
4481
5471
  id: "string|required",
4482
5472
  resourceName: "string|required",
@@ -4486,12 +5476,12 @@ class MetricsPlugin extends Plugin {
4486
5476
  metadata: "json"
4487
5477
  }
4488
5478
  }));
4489
- this.performanceResource = ok3 ? performanceResource : database.resources.performance_logs;
5479
+ this.performanceResource = ok3 ? performanceResource : database.resources.plg_performance_logs;
4490
5480
  });
4491
5481
  if (!ok) {
4492
- this.metricsResource = database.resources.metrics;
4493
- this.errorsResource = database.resources.error_logs;
4494
- this.performanceResource = database.resources.performance_logs;
5482
+ this.metricsResource = database.resources.plg_metrics;
5483
+ this.errorsResource = database.resources.plg_error_logs;
5484
+ this.performanceResource = database.resources.plg_performance_logs;
4495
5485
  }
4496
5486
  this.installDatabaseHooks();
4497
5487
  this.installMetricsHooks();
@@ -4510,7 +5500,7 @@ class MetricsPlugin extends Plugin {
4510
5500
  }
4511
5501
  installDatabaseHooks() {
4512
5502
  this.database.addHook("afterCreateResource", (resource) => {
4513
- if (resource.name !== "metrics" && resource.name !== "error_logs" && resource.name !== "performance_logs") {
5503
+ if (resource.name !== "plg_metrics" && resource.name !== "plg_error_logs" && resource.name !== "plg_performance_logs") {
4514
5504
  this.installResourceHooks(resource);
4515
5505
  }
4516
5506
  });
@@ -4520,7 +5510,7 @@ class MetricsPlugin extends Plugin {
4520
5510
  }
4521
5511
  installMetricsHooks() {
4522
5512
  for (const resource of Object.values(this.database.resources)) {
4523
- if (["metrics", "error_logs", "performance_logs"].includes(resource.name)) {
5513
+ if (["plg_metrics", "plg_error_logs", "plg_performance_logs"].includes(resource.name)) {
4524
5514
  continue;
4525
5515
  }
4526
5516
  this.installResourceHooks(resource);
@@ -4528,7 +5518,7 @@ class MetricsPlugin extends Plugin {
4528
5518
  this.database._createResource = this.database.createResource;
4529
5519
  this.database.createResource = async function(...args) {
4530
5520
  const resource = await this._createResource(...args);
4531
- if (this.plugins?.metrics && !["metrics", "error_logs", "performance_logs"].includes(resource.name)) {
5521
+ if (this.plugins?.metrics && !["plg_metrics", "plg_error_logs", "plg_performance_logs"].includes(resource.name)) {
4532
5522
  this.plugins.metrics.installResourceHooks(resource);
4533
5523
  }
4534
5524
  return resource;
@@ -5834,10 +6824,10 @@ class Client extends EventEmitter {
5834
6824
  // Enabled for better performance
5835
6825
  keepAliveMsecs: 1e3,
5836
6826
  // 1 second keep-alive
5837
- maxSockets: 50,
5838
- // Balanced for most applications
5839
- maxFreeSockets: 10,
5840
- // Good connection reuse
6827
+ maxSockets: httpClientOptions.maxSockets || 500,
6828
+ // High concurrency support
6829
+ maxFreeSockets: httpClientOptions.maxFreeSockets || 100,
6830
+ // Better connection reuse
5841
6831
  timeout: 6e4,
5842
6832
  // 60 second timeout
5843
6833
  ...httpClientOptions
@@ -5899,7 +6889,7 @@ class Client extends EventEmitter {
5899
6889
  this.emit("command.response", command.constructor.name, response, command.input);
5900
6890
  return response;
5901
6891
  }
5902
- async putObject({ key, metadata, contentType, body, contentEncoding, contentLength }) {
6892
+ async putObject({ key, metadata, contentType, body, contentEncoding, contentLength, ifMatch }) {
5903
6893
  const keyPrefix = typeof this.config.keyPrefix === "string" ? this.config.keyPrefix : "";
5904
6894
  keyPrefix ? path.join(keyPrefix, key) : key;
5905
6895
  const stringMetadata = {};
@@ -5919,6 +6909,7 @@ class Client extends EventEmitter {
5919
6909
  if (contentType !== void 0) options.ContentType = contentType;
5920
6910
  if (contentEncoding !== void 0) options.ContentEncoding = contentEncoding;
5921
6911
  if (contentLength !== void 0) options.ContentLength = contentLength;
6912
+ if (ifMatch !== void 0) options.IfMatch = ifMatch;
5922
6913
  let response, error;
5923
6914
  try {
5924
6915
  response = await this.sendCommand(new clientS3.PutObjectCommand(options));
@@ -8082,6 +9073,7 @@ ${errorDetails}`,
8082
9073
  data._lastModified = request.LastModified;
8083
9074
  data._hasContent = request.ContentLength > 0;
8084
9075
  data._mimeType = request.ContentType || null;
9076
+ data._etag = request.ETag;
8085
9077
  data._v = objectVersion;
8086
9078
  if (request.VersionId) data._versionId = request.VersionId;
8087
9079
  if (request.Expiration) data._expiresAt = request.Expiration;
@@ -8293,6 +9285,172 @@ ${errorDetails}`,
8293
9285
  return finalResult;
8294
9286
  }
8295
9287
  }
9288
+ /**
9289
+ * Update with conditional check (If-Match ETag)
9290
+ * @param {string} id - Resource ID
9291
+ * @param {Object} attributes - Attributes to update
9292
+ * @param {Object} options - Options including ifMatch (ETag)
9293
+ * @returns {Promise<Object>} { success: boolean, data?: Object, etag?: string, error?: string }
9294
+ * @example
9295
+ * const msg = await resource.get('msg-123');
9296
+ * const result = await resource.updateConditional('msg-123', { status: 'processing' }, { ifMatch: msg._etag });
9297
+ * if (!result.success) {
9298
+ * console.log('Update failed - object was modified by another process');
9299
+ * }
9300
+ */
9301
+ async updateConditional(id, attributes, options = {}) {
9302
+ if (lodashEs.isEmpty(id)) {
9303
+ throw new Error("id cannot be empty");
9304
+ }
9305
+ const { ifMatch } = options;
9306
+ if (!ifMatch) {
9307
+ throw new Error("updateConditional requires ifMatch option with ETag value");
9308
+ }
9309
+ const exists = await this.exists(id);
9310
+ if (!exists) {
9311
+ return {
9312
+ success: false,
9313
+ error: `Resource with id '${id}' does not exist`
9314
+ };
9315
+ }
9316
+ const originalData = await this.get(id);
9317
+ const attributesClone = lodashEs.cloneDeep(attributes);
9318
+ let mergedData = lodashEs.cloneDeep(originalData);
9319
+ for (const [key2, value] of Object.entries(attributesClone)) {
9320
+ if (key2.includes(".")) {
9321
+ let ref = mergedData;
9322
+ const parts = key2.split(".");
9323
+ for (let i = 0; i < parts.length - 1; i++) {
9324
+ if (typeof ref[parts[i]] !== "object" || ref[parts[i]] === null) {
9325
+ ref[parts[i]] = {};
9326
+ }
9327
+ ref = ref[parts[i]];
9328
+ }
9329
+ ref[parts[parts.length - 1]] = lodashEs.cloneDeep(value);
9330
+ } else if (typeof value === "object" && value !== null && !Array.isArray(value)) {
9331
+ mergedData[key2] = lodashEs.merge({}, mergedData[key2], value);
9332
+ } else {
9333
+ mergedData[key2] = lodashEs.cloneDeep(value);
9334
+ }
9335
+ }
9336
+ if (this.config.timestamps) {
9337
+ const now = (/* @__PURE__ */ new Date()).toISOString();
9338
+ mergedData.updatedAt = now;
9339
+ if (!mergedData.metadata) mergedData.metadata = {};
9340
+ mergedData.metadata.updatedAt = now;
9341
+ }
9342
+ const preProcessedData = await this.executeHooks("beforeUpdate", lodashEs.cloneDeep(mergedData));
9343
+ const completeData = { ...originalData, ...preProcessedData, id };
9344
+ const { isValid, errors, data } = await this.validate(lodashEs.cloneDeep(completeData));
9345
+ if (!isValid) {
9346
+ return {
9347
+ success: false,
9348
+ error: "Validation failed: " + (errors && errors.length ? JSON.stringify(errors) : "unknown"),
9349
+ validationErrors: errors
9350
+ };
9351
+ }
9352
+ const { id: validatedId, ...validatedAttributes } = data;
9353
+ const mappedData = await this.schema.mapper(validatedAttributes);
9354
+ mappedData._v = String(this.version);
9355
+ const behaviorImpl = getBehavior(this.behavior);
9356
+ const { mappedData: processedMetadata, body } = await behaviorImpl.handleUpdate({
9357
+ resource: this,
9358
+ id,
9359
+ data: validatedAttributes,
9360
+ mappedData,
9361
+ originalData: { ...attributesClone, id }
9362
+ });
9363
+ const key = this.getResourceKey(id);
9364
+ let existingContentType = void 0;
9365
+ let finalBody = body;
9366
+ if (body === "" && this.behavior !== "body-overflow") {
9367
+ const [ok2, err2, existingObject] = await tryFn(() => this.client.getObject(key));
9368
+ if (ok2 && existingObject.ContentLength > 0) {
9369
+ const existingBodyBuffer = Buffer.from(await existingObject.Body.transformToByteArray());
9370
+ const existingBodyString = existingBodyBuffer.toString();
9371
+ const [okParse, errParse] = await tryFn(() => Promise.resolve(JSON.parse(existingBodyString)));
9372
+ if (!okParse) {
9373
+ finalBody = existingBodyBuffer;
9374
+ existingContentType = existingObject.ContentType;
9375
+ }
9376
+ }
9377
+ }
9378
+ let finalContentType = existingContentType;
9379
+ if (finalBody && finalBody !== "" && !finalContentType) {
9380
+ const [okParse, errParse] = await tryFn(() => Promise.resolve(JSON.parse(finalBody)));
9381
+ if (okParse) finalContentType = "application/json";
9382
+ }
9383
+ const [ok, err, response] = await tryFn(() => this.client.putObject({
9384
+ key,
9385
+ body: finalBody,
9386
+ contentType: finalContentType,
9387
+ metadata: processedMetadata,
9388
+ ifMatch
9389
+ // ← Conditional write with ETag
9390
+ }));
9391
+ if (!ok) {
9392
+ if (err.name === "PreconditionFailed" || err.$metadata?.httpStatusCode === 412) {
9393
+ return {
9394
+ success: false,
9395
+ error: "ETag mismatch - object was modified by another process"
9396
+ };
9397
+ }
9398
+ return {
9399
+ success: false,
9400
+ error: err.message || "Update failed"
9401
+ };
9402
+ }
9403
+ const updatedData = await this.composeFullObjectFromWrite({
9404
+ id,
9405
+ metadata: processedMetadata,
9406
+ body: finalBody,
9407
+ behavior: this.behavior
9408
+ });
9409
+ const oldData = { ...originalData, id };
9410
+ const newData = { ...validatedAttributes, id };
9411
+ if (this.config.asyncPartitions && this.config.partitions && Object.keys(this.config.partitions).length > 0) {
9412
+ setImmediate(() => {
9413
+ this.handlePartitionReferenceUpdates(oldData, newData).catch((err2) => {
9414
+ this.emit("partitionIndexError", {
9415
+ operation: "updateConditional",
9416
+ id,
9417
+ error: err2,
9418
+ message: err2.message
9419
+ });
9420
+ });
9421
+ });
9422
+ const nonPartitionHooks = this.hooks.afterUpdate.filter(
9423
+ (hook) => !hook.toString().includes("handlePartitionReferenceUpdates")
9424
+ );
9425
+ let finalResult = updatedData;
9426
+ for (const hook of nonPartitionHooks) {
9427
+ finalResult = await hook(finalResult);
9428
+ }
9429
+ this.emit("update", {
9430
+ ...updatedData,
9431
+ $before: { ...originalData },
9432
+ $after: { ...finalResult }
9433
+ });
9434
+ return {
9435
+ success: true,
9436
+ data: finalResult,
9437
+ etag: response.ETag
9438
+ };
9439
+ } else {
9440
+ await this.handlePartitionReferenceUpdates(oldData, newData);
9441
+ const finalResult = await this.executeHooks("afterUpdate", updatedData);
9442
+ this.emit("update", {
9443
+ ...updatedData,
9444
+ $before: { ...originalData },
9445
+ $after: { ...finalResult }
9446
+ });
9447
+ return {
9448
+ success: true,
9449
+ data: finalResult,
9450
+ etag: response.ETag
9451
+ };
9452
+ }
9453
+ }
8296
9454
  /**
8297
9455
  * Delete a resource object by ID
8298
9456
  * @param {string} id - Resource ID
@@ -9700,7 +10858,7 @@ class Database extends EventEmitter {
9700
10858
  this.id = idGenerator(7);
9701
10859
  this.version = "1";
9702
10860
  this.s3dbVersion = (() => {
9703
- const [ok, err, version] = tryFn(() => true ? "9.3.0" : "latest");
10861
+ const [ok, err, version] = tryFn(() => true ? "10.0.1" : "latest");
9704
10862
  return ok ? version : "latest";
9705
10863
  })();
9706
10864
  this.resources = {};
@@ -10953,16 +12111,20 @@ class S3dbReplicator extends BaseReplicator {
10953
12111
  return resource;
10954
12112
  }
10955
12113
  _getDestResourceObj(resource) {
10956
- const available = Object.keys(this.client.resources || {});
12114
+ const db = this.targetDatabase || this.client;
12115
+ const available = Object.keys(db.resources || {});
10957
12116
  const norm = normalizeResourceName$1(resource);
10958
12117
  const found = available.find((r) => normalizeResourceName$1(r) === norm);
10959
12118
  if (!found) {
10960
12119
  throw new Error(`[S3dbReplicator] Destination resource not found: ${resource}. Available: ${available.join(", ")}`);
10961
12120
  }
10962
- return this.client.resources[found];
12121
+ return db.resources[found];
10963
12122
  }
10964
12123
  async replicateBatch(resourceName, records) {
10965
- if (!this.enabled || !this.shouldReplicateResource(resourceName)) {
12124
+ if (this.enabled === false) {
12125
+ return { skipped: true, reason: "replicator_disabled" };
12126
+ }
12127
+ if (!this.shouldReplicateResource(resourceName)) {
10966
12128
  return { skipped: true, reason: "resource_not_included" };
10967
12129
  }
10968
12130
  const results = [];
@@ -11073,11 +12235,12 @@ class SqsReplicator extends BaseReplicator {
11073
12235
  this.client = client;
11074
12236
  this.queueUrl = config.queueUrl;
11075
12237
  this.queues = config.queues || {};
11076
- this.defaultQueue = config.defaultQueue || config.defaultQueueUrl || config.queueUrlDefault;
12238
+ this.defaultQueue = config.defaultQueue || config.defaultQueueUrl || config.queueUrlDefault || null;
11077
12239
  this.region = config.region || "us-east-1";
11078
12240
  this.sqsClient = client || null;
11079
12241
  this.messageGroupId = config.messageGroupId;
11080
12242
  this.deduplicationId = config.deduplicationId;
12243
+ this.resourceQueueMap = config.resourceQueueMap || null;
11081
12244
  if (Array.isArray(resources)) {
11082
12245
  this.resources = {};
11083
12246
  for (const resource of resources) {
@@ -11208,7 +12371,10 @@ class SqsReplicator extends BaseReplicator {
11208
12371
  }
11209
12372
  }
11210
12373
  async replicate(resource, operation, data, id, beforeData = null) {
11211
- if (!this.enabled || !this.shouldReplicateResource(resource)) {
12374
+ if (this.enabled === false) {
12375
+ return { skipped: true, reason: "replicator_disabled" };
12376
+ }
12377
+ if (!this.shouldReplicateResource(resource)) {
11212
12378
  return { skipped: true, reason: "resource_not_included" };
11213
12379
  }
11214
12380
  const [ok, err, result] = await tryFn(async () => {
@@ -11252,7 +12418,10 @@ class SqsReplicator extends BaseReplicator {
11252
12418
  return { success: false, error: err.message };
11253
12419
  }
11254
12420
  async replicateBatch(resource, records) {
11255
- if (!this.enabled || !this.shouldReplicateResource(resource)) {
12421
+ if (this.enabled === false) {
12422
+ return { skipped: true, reason: "replicator_disabled" };
12423
+ }
12424
+ if (!this.shouldReplicateResource(resource)) {
11256
12425
  return { skipped: true, reason: "resource_not_included" };
11257
12426
  }
11258
12427
  const [ok, err, result] = await tryFn(async () => {
@@ -11406,22 +12575,23 @@ class ReplicatorPlugin extends Plugin {
11406
12575
  replicators: options.replicators || [],
11407
12576
  logErrors: options.logErrors !== false,
11408
12577
  replicatorLogResource: options.replicatorLogResource || "replicator_log",
12578
+ persistReplicatorLog: options.persistReplicatorLog || false,
11409
12579
  enabled: options.enabled !== false,
11410
12580
  batchSize: options.batchSize || 100,
11411
12581
  maxRetries: options.maxRetries || 3,
11412
12582
  timeout: options.timeout || 3e4,
11413
- verbose: options.verbose || false,
11414
- ...options
12583
+ verbose: options.verbose || false
11415
12584
  };
11416
12585
  this.replicators = [];
11417
12586
  this.database = null;
11418
12587
  this.eventListenersInstalled = /* @__PURE__ */ new Set();
11419
- }
11420
- /**
11421
- * Decompress data if it was compressed
11422
- */
11423
- async decompressData(data) {
11424
- return data;
12588
+ this.eventHandlers = /* @__PURE__ */ new Map();
12589
+ this.stats = {
12590
+ totalReplications: 0,
12591
+ totalErrors: 0,
12592
+ lastSync: null
12593
+ };
12594
+ this._afterCreateResourceHook = null;
11425
12595
  }
11426
12596
  // Helper to filter out internal S3DB fields
11427
12597
  filterInternalFields(obj) {
@@ -11442,7 +12612,7 @@ class ReplicatorPlugin extends Plugin {
11442
12612
  if (!resource || this.eventListenersInstalled.has(resource.name) || resource.name === this.config.replicatorLogResource) {
11443
12613
  return;
11444
12614
  }
11445
- resource.on("insert", async (data) => {
12615
+ const insertHandler = async (data) => {
11446
12616
  const [ok, error] = await tryFn(async () => {
11447
12617
  const completeData = { ...data, createdAt: (/* @__PURE__ */ new Date()).toISOString() };
11448
12618
  await plugin.processReplicatorEvent("insert", resource.name, completeData.id, completeData);
@@ -11453,8 +12623,8 @@ class ReplicatorPlugin extends Plugin {
11453
12623
  }
11454
12624
  this.emit("error", { operation: "insert", error: error.message, resource: resource.name });
11455
12625
  }
11456
- });
11457
- resource.on("update", async (data, beforeData) => {
12626
+ };
12627
+ const updateHandler = async (data, beforeData) => {
11458
12628
  const [ok, error] = await tryFn(async () => {
11459
12629
  const completeData = await plugin.getCompleteData(resource, data);
11460
12630
  const dataWithTimestamp = { ...completeData, updatedAt: (/* @__PURE__ */ new Date()).toISOString() };
@@ -11466,8 +12636,8 @@ class ReplicatorPlugin extends Plugin {
11466
12636
  }
11467
12637
  this.emit("error", { operation: "update", error: error.message, resource: resource.name });
11468
12638
  }
11469
- });
11470
- resource.on("delete", async (data) => {
12639
+ };
12640
+ const deleteHandler = async (data) => {
11471
12641
  const [ok, error] = await tryFn(async () => {
11472
12642
  await plugin.processReplicatorEvent("delete", resource.name, data.id, data);
11473
12643
  });
@@ -11477,14 +12647,22 @@ class ReplicatorPlugin extends Plugin {
11477
12647
  }
11478
12648
  this.emit("error", { operation: "delete", error: error.message, resource: resource.name });
11479
12649
  }
11480
- });
12650
+ };
12651
+ this.eventHandlers.set(resource.name, {
12652
+ insert: insertHandler,
12653
+ update: updateHandler,
12654
+ delete: deleteHandler
12655
+ });
12656
+ resource.on("insert", insertHandler);
12657
+ resource.on("update", updateHandler);
12658
+ resource.on("delete", deleteHandler);
11481
12659
  this.eventListenersInstalled.add(resource.name);
11482
12660
  }
11483
12661
  async setup(database) {
11484
12662
  this.database = database;
11485
12663
  if (this.config.persistReplicatorLog) {
11486
12664
  const [ok, err, logResource] = await tryFn(() => database.createResource({
11487
- name: this.config.replicatorLogResource || "replicator_logs",
12665
+ name: this.config.replicatorLogResource || "plg_replicator_logs",
11488
12666
  attributes: {
11489
12667
  id: "string|required",
11490
12668
  resource: "string|required",
@@ -11498,13 +12676,13 @@ class ReplicatorPlugin extends Plugin {
11498
12676
  if (ok) {
11499
12677
  this.replicatorLogResource = logResource;
11500
12678
  } else {
11501
- this.replicatorLogResource = database.resources[this.config.replicatorLogResource || "replicator_logs"];
12679
+ this.replicatorLogResource = database.resources[this.config.replicatorLogResource || "plg_replicator_logs"];
11502
12680
  }
11503
12681
  }
11504
12682
  await this.initializeReplicators(database);
11505
12683
  this.installDatabaseHooks();
11506
12684
  for (const resource of Object.values(database.resources)) {
11507
- if (resource.name !== (this.config.replicatorLogResource || "replicator_logs")) {
12685
+ if (resource.name !== (this.config.replicatorLogResource || "plg_replicator_logs")) {
11508
12686
  this.installEventListeners(resource, database, this);
11509
12687
  }
11510
12688
  }
@@ -11520,14 +12698,18 @@ class ReplicatorPlugin extends Plugin {
11520
12698
  this.removeDatabaseHooks();
11521
12699
  }
11522
12700
  installDatabaseHooks() {
11523
- this.database.addHook("afterCreateResource", (resource) => {
11524
- if (resource.name !== (this.config.replicatorLogResource || "replicator_logs")) {
12701
+ this._afterCreateResourceHook = (resource) => {
12702
+ if (resource.name !== (this.config.replicatorLogResource || "plg_replicator_logs")) {
11525
12703
  this.installEventListeners(resource, this.database, this);
11526
12704
  }
11527
- });
12705
+ };
12706
+ this.database.addHook("afterCreateResource", this._afterCreateResourceHook);
11528
12707
  }
11529
12708
  removeDatabaseHooks() {
11530
- this.database.removeHook("afterCreateResource", this.installEventListeners.bind(this));
12709
+ if (this._afterCreateResourceHook) {
12710
+ this.database.removeHook("afterCreateResource", this._afterCreateResourceHook);
12711
+ this._afterCreateResourceHook = null;
12712
+ }
11531
12713
  }
11532
12714
  createReplicator(driver, config, resources, client) {
11533
12715
  return createReplicator(driver, config, resources, client);
@@ -11552,9 +12734,9 @@ class ReplicatorPlugin extends Plugin {
11552
12734
  async retryWithBackoff(operation, maxRetries = 3) {
11553
12735
  let lastError;
11554
12736
  for (let attempt = 1; attempt <= maxRetries; attempt++) {
11555
- const [ok, error] = await tryFn(operation);
12737
+ const [ok, error, result] = await tryFn(operation);
11556
12738
  if (ok) {
11557
- return ok;
12739
+ return result;
11558
12740
  } else {
11559
12741
  lastError = error;
11560
12742
  if (this.config.verbose) {
@@ -11649,7 +12831,7 @@ class ReplicatorPlugin extends Plugin {
11649
12831
  });
11650
12832
  return Promise.allSettled(promises);
11651
12833
  }
11652
- async processreplicatorItem(item) {
12834
+ async processReplicatorItem(item) {
11653
12835
  const applicableReplicators = this.replicators.filter((replicator) => {
11654
12836
  const should = replicator.shouldReplicateResource && replicator.shouldReplicateResource(item.resourceName, item.operation);
11655
12837
  return should;
@@ -11709,12 +12891,9 @@ class ReplicatorPlugin extends Plugin {
11709
12891
  });
11710
12892
  return Promise.allSettled(promises);
11711
12893
  }
11712
- async logreplicator(item) {
12894
+ async logReplicator(item) {
11713
12895
  const logRes = this.replicatorLog || this.database.resources[normalizeResourceName(this.config.replicatorLogResource)];
11714
12896
  if (!logRes) {
11715
- if (this.database) {
11716
- if (this.database.options && this.database.options.connectionString) ;
11717
- }
11718
12897
  this.emit("replicator.log.failed", { error: "replicator log resource not found", item });
11719
12898
  return;
11720
12899
  }
@@ -11736,7 +12915,7 @@ class ReplicatorPlugin extends Plugin {
11736
12915
  this.emit("replicator.log.failed", { error: err, item });
11737
12916
  }
11738
12917
  }
11739
- async updatereplicatorLog(logId, updates) {
12918
+ async updateReplicatorLog(logId, updates) {
11740
12919
  if (!this.replicatorLog) return;
11741
12920
  const [ok, err] = await tryFn(async () => {
11742
12921
  await this.replicatorLog.update(logId, {
@@ -11749,7 +12928,7 @@ class ReplicatorPlugin extends Plugin {
11749
12928
  }
11750
12929
  }
11751
12930
  // Utility methods
11752
- async getreplicatorStats() {
12931
+ async getReplicatorStats() {
11753
12932
  const replicatorStats = await Promise.all(
11754
12933
  this.replicators.map(async (replicator) => {
11755
12934
  const status = await replicator.getStatus();
@@ -11761,116 +12940,669 @@ class ReplicatorPlugin extends Plugin {
11761
12940
  };
11762
12941
  })
11763
12942
  );
11764
- return {
11765
- replicators: replicatorStats,
11766
- queue: {
11767
- length: this.queue.length,
11768
- isProcessing: this.isProcessing
11769
- },
11770
- stats: this.stats,
11771
- lastSync: this.stats.lastSync
11772
- };
11773
- }
11774
- async getreplicatorLogs(options = {}) {
11775
- if (!this.replicatorLog) {
11776
- return [];
11777
- }
11778
- const {
11779
- resourceName,
11780
- operation,
11781
- status,
11782
- limit = 100,
11783
- offset = 0
11784
- } = options;
11785
- let query = {};
11786
- if (resourceName) {
11787
- query.resourceName = resourceName;
12943
+ return {
12944
+ replicators: replicatorStats,
12945
+ stats: this.stats,
12946
+ lastSync: this.stats.lastSync
12947
+ };
12948
+ }
12949
+ async getReplicatorLogs(options = {}) {
12950
+ if (!this.replicatorLog) {
12951
+ return [];
12952
+ }
12953
+ const {
12954
+ resourceName,
12955
+ operation,
12956
+ status,
12957
+ limit = 100,
12958
+ offset = 0
12959
+ } = options;
12960
+ const filter = {};
12961
+ if (resourceName) {
12962
+ filter.resourceName = resourceName;
12963
+ }
12964
+ if (operation) {
12965
+ filter.operation = operation;
12966
+ }
12967
+ if (status) {
12968
+ filter.status = status;
12969
+ }
12970
+ const logs = await this.replicatorLog.query(filter, { limit, offset });
12971
+ return logs || [];
12972
+ }
12973
+ async retryFailedReplicators() {
12974
+ if (!this.replicatorLog) {
12975
+ return { retried: 0 };
12976
+ }
12977
+ const failedLogs = await this.replicatorLog.query({
12978
+ status: "failed"
12979
+ });
12980
+ let retried = 0;
12981
+ for (const log of failedLogs || []) {
12982
+ const [ok, err] = await tryFn(async () => {
12983
+ await this.processReplicatorEvent(
12984
+ log.operation,
12985
+ log.resourceName,
12986
+ log.recordId,
12987
+ log.data
12988
+ );
12989
+ });
12990
+ if (ok) {
12991
+ retried++;
12992
+ }
12993
+ }
12994
+ return { retried };
12995
+ }
12996
+ async syncAllData(replicatorId) {
12997
+ const replicator = this.replicators.find((r) => r.id === replicatorId);
12998
+ if (!replicator) {
12999
+ throw new Error(`Replicator not found: ${replicatorId}`);
13000
+ }
13001
+ this.stats.lastSync = (/* @__PURE__ */ new Date()).toISOString();
13002
+ for (const resourceName in this.database.resources) {
13003
+ if (normalizeResourceName(resourceName) === normalizeResourceName("plg_replicator_logs")) continue;
13004
+ if (replicator.shouldReplicateResource(resourceName)) {
13005
+ this.emit("replicator.sync.resource", { resourceName, replicatorId });
13006
+ const resource = this.database.resources[resourceName];
13007
+ let offset = 0;
13008
+ const pageSize = this.config.batchSize || 100;
13009
+ while (true) {
13010
+ const [ok, err, page] = await tryFn(() => resource.page({ offset, size: pageSize }));
13011
+ if (!ok || !page) break;
13012
+ const records = Array.isArray(page) ? page : page.items || [];
13013
+ if (records.length === 0) break;
13014
+ for (const record of records) {
13015
+ await replicator.replicate(resourceName, "insert", record, record.id);
13016
+ }
13017
+ offset += pageSize;
13018
+ }
13019
+ }
13020
+ }
13021
+ this.emit("replicator.sync.completed", { replicatorId, stats: this.stats });
13022
+ }
13023
+ async cleanup() {
13024
+ const [ok, error] = await tryFn(async () => {
13025
+ if (this.replicators && this.replicators.length > 0) {
13026
+ const cleanupPromises = this.replicators.map(async (replicator) => {
13027
+ const [replicatorOk, replicatorError] = await tryFn(async () => {
13028
+ if (replicator && typeof replicator.cleanup === "function") {
13029
+ await replicator.cleanup();
13030
+ }
13031
+ });
13032
+ if (!replicatorOk) {
13033
+ if (this.config.verbose) {
13034
+ console.warn(`[ReplicatorPlugin] Failed to cleanup replicator ${replicator.name || replicator.id}: ${replicatorError.message}`);
13035
+ }
13036
+ this.emit("replicator_cleanup_error", {
13037
+ replicator: replicator.name || replicator.id || "unknown",
13038
+ driver: replicator.driver || "unknown",
13039
+ error: replicatorError.message
13040
+ });
13041
+ }
13042
+ });
13043
+ await Promise.allSettled(cleanupPromises);
13044
+ }
13045
+ if (this.database && this.database.resources) {
13046
+ for (const resourceName of this.eventListenersInstalled) {
13047
+ const resource = this.database.resources[resourceName];
13048
+ const handlers = this.eventHandlers.get(resourceName);
13049
+ if (resource && handlers) {
13050
+ resource.off("insert", handlers.insert);
13051
+ resource.off("update", handlers.update);
13052
+ resource.off("delete", handlers.delete);
13053
+ }
13054
+ }
13055
+ }
13056
+ this.replicators = [];
13057
+ this.database = null;
13058
+ this.eventListenersInstalled.clear();
13059
+ this.eventHandlers.clear();
13060
+ this.removeAllListeners();
13061
+ });
13062
+ if (!ok) {
13063
+ if (this.config.verbose) {
13064
+ console.warn(`[ReplicatorPlugin] Failed to cleanup plugin: ${error.message}`);
13065
+ }
13066
+ this.emit("replicator_plugin_cleanup_error", {
13067
+ error: error.message
13068
+ });
13069
+ }
13070
+ }
13071
+ }
13072
+
13073
+ class S3QueuePlugin extends Plugin {
13074
+ constructor(options = {}) {
13075
+ super(options);
13076
+ if (!options.resource) {
13077
+ throw new Error('S3QueuePlugin requires "resource" option');
13078
+ }
13079
+ this.config = {
13080
+ resource: options.resource,
13081
+ visibilityTimeout: options.visibilityTimeout || 3e4,
13082
+ // 30 seconds
13083
+ pollInterval: options.pollInterval || 1e3,
13084
+ // 1 second
13085
+ maxAttempts: options.maxAttempts || 3,
13086
+ concurrency: options.concurrency || 1,
13087
+ deadLetterResource: options.deadLetterResource || null,
13088
+ autoStart: options.autoStart !== false,
13089
+ onMessage: options.onMessage,
13090
+ onError: options.onError,
13091
+ onComplete: options.onComplete,
13092
+ verbose: options.verbose || false,
13093
+ ...options
13094
+ };
13095
+ this.queueResource = null;
13096
+ this.targetResource = null;
13097
+ this.deadLetterResourceObj = null;
13098
+ this.workers = [];
13099
+ this.isRunning = false;
13100
+ this.workerId = `worker-${Date.now()}-${Math.random().toString(36).slice(2, 9)}`;
13101
+ this.processedCache = /* @__PURE__ */ new Map();
13102
+ this.cacheCleanupInterval = null;
13103
+ this.lockCleanupInterval = null;
13104
+ }
13105
+ async onSetup() {
13106
+ this.targetResource = this.database.resources[this.config.resource];
13107
+ if (!this.targetResource) {
13108
+ throw new Error(`S3QueuePlugin: resource '${this.config.resource}' not found`);
13109
+ }
13110
+ const queueName = `${this.config.resource}_queue`;
13111
+ const [ok, err] = await tryFn(
13112
+ () => this.database.createResource({
13113
+ name: queueName,
13114
+ attributes: {
13115
+ id: "string|required",
13116
+ originalId: "string|required",
13117
+ // ID do registro original
13118
+ status: "string|required",
13119
+ // pending/processing/completed/failed/dead
13120
+ visibleAt: "number|required",
13121
+ // Timestamp de visibilidade
13122
+ claimedBy: "string|optional",
13123
+ // Worker que claimed
13124
+ claimedAt: "number|optional",
13125
+ // Timestamp do claim
13126
+ attempts: "number|default:0",
13127
+ maxAttempts: "number|default:3",
13128
+ error: "string|optional",
13129
+ result: "json|optional",
13130
+ createdAt: "string|required",
13131
+ completedAt: "number|optional"
13132
+ },
13133
+ behavior: "body-overflow",
13134
+ timestamps: true,
13135
+ asyncPartitions: true,
13136
+ partitions: {
13137
+ byStatus: { fields: { status: "string" } },
13138
+ byDate: { fields: { createdAt: "string|maxlength:10" } }
13139
+ }
13140
+ })
13141
+ );
13142
+ if (!ok && !this.database.resources[queueName]) {
13143
+ throw new Error(`Failed to create queue resource: ${err?.message}`);
13144
+ }
13145
+ this.queueResource = this.database.resources[queueName];
13146
+ const lockName = `${this.config.resource}_locks`;
13147
+ const [okLock, errLock] = await tryFn(
13148
+ () => this.database.createResource({
13149
+ name: lockName,
13150
+ attributes: {
13151
+ id: "string|required",
13152
+ workerId: "string|required",
13153
+ timestamp: "number|required",
13154
+ ttl: "number|default:5000"
13155
+ },
13156
+ behavior: "body-overflow",
13157
+ timestamps: false
13158
+ })
13159
+ );
13160
+ if (okLock || this.database.resources[lockName]) {
13161
+ this.lockResource = this.database.resources[lockName];
13162
+ } else {
13163
+ this.lockResource = null;
13164
+ if (this.config.verbose) {
13165
+ console.log(`[S3QueuePlugin] Lock resource creation failed, locking disabled: ${errLock?.message}`);
13166
+ }
13167
+ }
13168
+ this.addHelperMethods();
13169
+ if (this.config.deadLetterResource) {
13170
+ await this.createDeadLetterResource();
13171
+ }
13172
+ if (this.config.verbose) {
13173
+ console.log(`[S3QueuePlugin] Setup completed for resource '${this.config.resource}'`);
13174
+ }
13175
+ }
13176
+ async onStart() {
13177
+ if (this.config.autoStart && this.config.onMessage) {
13178
+ await this.startProcessing();
13179
+ }
13180
+ }
13181
+ async onStop() {
13182
+ await this.stopProcessing();
13183
+ }
13184
+ addHelperMethods() {
13185
+ const plugin = this;
13186
+ const resource = this.targetResource;
13187
+ resource.enqueue = async function(data, options = {}) {
13188
+ const recordData = {
13189
+ id: data.id || idGenerator(),
13190
+ ...data
13191
+ };
13192
+ const record = await resource.insert(recordData);
13193
+ const queueEntry = {
13194
+ id: idGenerator(),
13195
+ originalId: record.id,
13196
+ status: "pending",
13197
+ visibleAt: Date.now(),
13198
+ attempts: 0,
13199
+ maxAttempts: options.maxAttempts || plugin.config.maxAttempts,
13200
+ createdAt: (/* @__PURE__ */ new Date()).toISOString().slice(0, 10)
13201
+ };
13202
+ await plugin.queueResource.insert(queueEntry);
13203
+ plugin.emit("message.enqueued", { id: record.id, queueId: queueEntry.id });
13204
+ return record;
13205
+ };
13206
+ resource.queueStats = async function() {
13207
+ return await plugin.getStats();
13208
+ };
13209
+ resource.startProcessing = async function(handler, options = {}) {
13210
+ return await plugin.startProcessing(handler, options);
13211
+ };
13212
+ resource.stopProcessing = async function() {
13213
+ return await plugin.stopProcessing();
13214
+ };
13215
+ }
13216
+ async startProcessing(handler = null, options = {}) {
13217
+ if (this.isRunning) {
13218
+ if (this.config.verbose) {
13219
+ console.log("[S3QueuePlugin] Already running");
13220
+ }
13221
+ return;
13222
+ }
13223
+ const messageHandler = handler || this.config.onMessage;
13224
+ if (!messageHandler) {
13225
+ throw new Error("S3QueuePlugin: onMessage handler required");
13226
+ }
13227
+ this.isRunning = true;
13228
+ const concurrency = options.concurrency || this.config.concurrency;
13229
+ this.cacheCleanupInterval = setInterval(() => {
13230
+ const now = Date.now();
13231
+ const maxAge = 3e4;
13232
+ for (const [queueId, timestamp] of this.processedCache.entries()) {
13233
+ if (now - timestamp > maxAge) {
13234
+ this.processedCache.delete(queueId);
13235
+ }
13236
+ }
13237
+ }, 5e3);
13238
+ this.lockCleanupInterval = setInterval(() => {
13239
+ this.cleanupStaleLocks().catch((err) => {
13240
+ if (this.config.verbose) {
13241
+ console.log(`[lockCleanup] Error: ${err.message}`);
13242
+ }
13243
+ });
13244
+ }, 1e4);
13245
+ for (let i = 0; i < concurrency; i++) {
13246
+ const worker = this.createWorker(messageHandler, i);
13247
+ this.workers.push(worker);
13248
+ }
13249
+ if (this.config.verbose) {
13250
+ console.log(`[S3QueuePlugin] Started ${concurrency} workers`);
13251
+ }
13252
+ this.emit("workers.started", { concurrency, workerId: this.workerId });
13253
+ }
13254
+ async stopProcessing() {
13255
+ if (!this.isRunning) return;
13256
+ this.isRunning = false;
13257
+ if (this.cacheCleanupInterval) {
13258
+ clearInterval(this.cacheCleanupInterval);
13259
+ this.cacheCleanupInterval = null;
13260
+ }
13261
+ if (this.lockCleanupInterval) {
13262
+ clearInterval(this.lockCleanupInterval);
13263
+ this.lockCleanupInterval = null;
13264
+ }
13265
+ await Promise.all(this.workers);
13266
+ this.workers = [];
13267
+ this.processedCache.clear();
13268
+ if (this.config.verbose) {
13269
+ console.log("[S3QueuePlugin] Stopped all workers");
13270
+ }
13271
+ this.emit("workers.stopped", { workerId: this.workerId });
13272
+ }
13273
+ createWorker(handler, workerIndex) {
13274
+ return (async () => {
13275
+ while (this.isRunning) {
13276
+ try {
13277
+ const message = await this.claimMessage();
13278
+ if (message) {
13279
+ await this.processMessage(message, handler);
13280
+ } else {
13281
+ await new Promise((resolve) => setTimeout(resolve, this.config.pollInterval));
13282
+ }
13283
+ } catch (error) {
13284
+ if (this.config.verbose) {
13285
+ console.error(`[Worker ${workerIndex}] Error:`, error.message);
13286
+ }
13287
+ await new Promise((resolve) => setTimeout(resolve, 1e3));
13288
+ }
13289
+ }
13290
+ })();
13291
+ }
13292
+ async claimMessage() {
13293
+ const now = Date.now();
13294
+ const [ok, err, messages] = await tryFn(
13295
+ () => this.queueResource.query({
13296
+ status: "pending"
13297
+ })
13298
+ );
13299
+ if (!ok || !messages || messages.length === 0) {
13300
+ return null;
13301
+ }
13302
+ const available = messages.filter((m) => m.visibleAt <= now);
13303
+ if (available.length === 0) {
13304
+ return null;
13305
+ }
13306
+ for (const msg of available) {
13307
+ const claimed = await this.attemptClaim(msg);
13308
+ if (claimed) {
13309
+ return claimed;
13310
+ }
13311
+ }
13312
+ return null;
13313
+ }
13314
+ /**
13315
+ * Acquire a distributed lock using ETag-based conditional updates
13316
+ * This ensures only one worker can claim a message at a time
13317
+ *
13318
+ * Uses a two-step process:
13319
+ * 1. Create lock resource (similar to queue resource) if not exists
13320
+ * 2. Try to claim lock using ETag-based conditional update
13321
+ */
13322
+ async acquireLock(messageId) {
13323
+ if (!this.lockResource) {
13324
+ return true;
13325
+ }
13326
+ const lockId = `lock-${messageId}`;
13327
+ const now = Date.now();
13328
+ try {
13329
+ const [okGet, errGet, existingLock] = await tryFn(
13330
+ () => this.lockResource.get(lockId)
13331
+ );
13332
+ if (existingLock) {
13333
+ const lockAge = now - existingLock.timestamp;
13334
+ if (lockAge < existingLock.ttl) {
13335
+ return false;
13336
+ }
13337
+ const [ok, err, result] = await tryFn(
13338
+ () => this.lockResource.updateConditional(lockId, {
13339
+ workerId: this.workerId,
13340
+ timestamp: now,
13341
+ ttl: 5e3
13342
+ }, {
13343
+ ifMatch: existingLock._etag
13344
+ })
13345
+ );
13346
+ return ok && result.success;
13347
+ }
13348
+ const [okCreate, errCreate] = await tryFn(
13349
+ () => this.lockResource.insert({
13350
+ id: lockId,
13351
+ workerId: this.workerId,
13352
+ timestamp: now,
13353
+ ttl: 5e3
13354
+ })
13355
+ );
13356
+ return okCreate;
13357
+ } catch (error) {
13358
+ if (this.config.verbose) {
13359
+ console.log(`[acquireLock] Error: ${error.message}`);
13360
+ }
13361
+ return false;
13362
+ }
13363
+ }
13364
+ /**
13365
+ * Release a distributed lock by deleting the lock record
13366
+ */
13367
+ async releaseLock(messageId) {
13368
+ if (!this.lockResource) {
13369
+ return;
13370
+ }
13371
+ const lockId = `lock-${messageId}`;
13372
+ try {
13373
+ await this.lockResource.delete(lockId);
13374
+ } catch (error) {
13375
+ if (this.config.verbose) {
13376
+ console.log(`[releaseLock] Failed to release lock for ${messageId}: ${error.message}`);
13377
+ }
13378
+ }
13379
+ }
13380
+ /**
13381
+ * Clean up stale locks (older than TTL)
13382
+ * This prevents deadlocks if a worker crashes while holding a lock
13383
+ */
13384
+ async cleanupStaleLocks() {
13385
+ if (!this.lockResource) {
13386
+ return;
13387
+ }
13388
+ const now = Date.now();
13389
+ try {
13390
+ const locks = await this.lockResource.list();
13391
+ for (const lock of locks) {
13392
+ const lockAge = now - lock.timestamp;
13393
+ if (lockAge > lock.ttl) {
13394
+ await this.lockResource.delete(lock.id);
13395
+ if (this.config.verbose) {
13396
+ console.log(`[cleanupStaleLocks] Removed expired lock: ${lock.id}`);
13397
+ }
13398
+ }
13399
+ }
13400
+ } catch (error) {
13401
+ if (this.config.verbose) {
13402
+ console.log(`[cleanupStaleLocks] Error during cleanup: ${error.message}`);
13403
+ }
13404
+ }
13405
+ }
13406
+ async attemptClaim(msg) {
13407
+ const now = Date.now();
13408
+ const lockAcquired = await this.acquireLock(msg.id);
13409
+ if (!lockAcquired) {
13410
+ return null;
13411
+ }
13412
+ if (this.processedCache.has(msg.id)) {
13413
+ await this.releaseLock(msg.id);
13414
+ if (this.config.verbose) {
13415
+ console.log(`[attemptClaim] Message ${msg.id} already processed (in cache)`);
13416
+ }
13417
+ return null;
13418
+ }
13419
+ this.processedCache.set(msg.id, Date.now());
13420
+ await this.releaseLock(msg.id);
13421
+ const [okGet, errGet, msgWithETag] = await tryFn(
13422
+ () => this.queueResource.get(msg.id)
13423
+ );
13424
+ if (!okGet || !msgWithETag) {
13425
+ this.processedCache.delete(msg.id);
13426
+ if (this.config.verbose) {
13427
+ console.log(`[attemptClaim] Message ${msg.id} not found or error: ${errGet?.message}`);
13428
+ }
13429
+ return null;
13430
+ }
13431
+ if (msgWithETag.status !== "pending" || msgWithETag.visibleAt > now) {
13432
+ this.processedCache.delete(msg.id);
13433
+ if (this.config.verbose) {
13434
+ console.log(`[attemptClaim] Message ${msg.id} not claimable: status=${msgWithETag.status}, visibleAt=${msgWithETag.visibleAt}, now=${now}`);
13435
+ }
13436
+ return null;
13437
+ }
13438
+ if (this.config.verbose) {
13439
+ console.log(`[attemptClaim] Attempting to claim ${msg.id} with ETag: ${msgWithETag._etag}`);
13440
+ }
13441
+ const [ok, err, result] = await tryFn(
13442
+ () => this.queueResource.updateConditional(msgWithETag.id, {
13443
+ status: "processing",
13444
+ claimedBy: this.workerId,
13445
+ claimedAt: now,
13446
+ visibleAt: now + this.config.visibilityTimeout,
13447
+ attempts: msgWithETag.attempts + 1
13448
+ }, {
13449
+ ifMatch: msgWithETag._etag
13450
+ // ← ATOMIC CLAIM using ETag!
13451
+ })
13452
+ );
13453
+ if (!ok || !result.success) {
13454
+ this.processedCache.delete(msg.id);
13455
+ if (this.config.verbose) {
13456
+ console.log(`[attemptClaim] Failed to claim ${msg.id}: ${err?.message || result.error}`);
13457
+ }
13458
+ return null;
11788
13459
  }
11789
- if (operation) {
11790
- query.operation = operation;
13460
+ if (this.config.verbose) {
13461
+ console.log(`[attemptClaim] Successfully claimed ${msg.id}`);
11791
13462
  }
11792
- if (status) {
11793
- query.status = status;
13463
+ const [okRecord, errRecord, record] = await tryFn(
13464
+ () => this.targetResource.get(msgWithETag.originalId)
13465
+ );
13466
+ if (!okRecord) {
13467
+ await this.failMessage(msgWithETag.id, "Original record not found");
13468
+ return null;
11794
13469
  }
11795
- const logs = await this.replicatorLog.list(query);
11796
- return logs.slice(offset, offset + limit);
13470
+ return {
13471
+ queueId: msgWithETag.id,
13472
+ record,
13473
+ attempts: msgWithETag.attempts + 1,
13474
+ maxAttempts: msgWithETag.maxAttempts
13475
+ };
11797
13476
  }
11798
- async retryFailedreplicators() {
11799
- if (!this.replicatorLog) {
11800
- return { retried: 0 };
13477
+ async processMessage(message, handler) {
13478
+ const startTime = Date.now();
13479
+ try {
13480
+ const result = await handler(message.record, {
13481
+ queueId: message.queueId,
13482
+ attempts: message.attempts,
13483
+ workerId: this.workerId
13484
+ });
13485
+ await this.completeMessage(message.queueId, result);
13486
+ const duration = Date.now() - startTime;
13487
+ this.emit("message.completed", {
13488
+ queueId: message.queueId,
13489
+ originalId: message.record.id,
13490
+ duration,
13491
+ attempts: message.attempts
13492
+ });
13493
+ if (this.config.onComplete) {
13494
+ await this.config.onComplete(message.record, result);
13495
+ }
13496
+ } catch (error) {
13497
+ const shouldRetry = message.attempts < message.maxAttempts;
13498
+ if (shouldRetry) {
13499
+ await this.retryMessage(message.queueId, message.attempts, error.message);
13500
+ this.emit("message.retry", {
13501
+ queueId: message.queueId,
13502
+ originalId: message.record.id,
13503
+ attempts: message.attempts,
13504
+ error: error.message
13505
+ });
13506
+ } else {
13507
+ await this.moveToDeadLetter(message.queueId, message.record, error.message);
13508
+ this.emit("message.dead", {
13509
+ queueId: message.queueId,
13510
+ originalId: message.record.id,
13511
+ error: error.message
13512
+ });
13513
+ }
13514
+ if (this.config.onError) {
13515
+ await this.config.onError(error, message.record);
13516
+ }
11801
13517
  }
11802
- const failedLogs = await this.replicatorLog.list({
11803
- status: "failed"
13518
+ }
13519
+ async completeMessage(queueId, result) {
13520
+ await this.queueResource.update(queueId, {
13521
+ status: "completed",
13522
+ completedAt: Date.now(),
13523
+ result
11804
13524
  });
11805
- let retried = 0;
11806
- for (const log of failedLogs) {
11807
- const [ok, err] = await tryFn(async () => {
11808
- await this.processReplicatorEvent(
11809
- log.resourceName,
11810
- log.operation,
11811
- log.recordId,
11812
- log.data
11813
- );
13525
+ }
13526
+ async failMessage(queueId, error) {
13527
+ await this.queueResource.update(queueId, {
13528
+ status: "failed",
13529
+ error
13530
+ });
13531
+ }
13532
+ async retryMessage(queueId, attempts, error) {
13533
+ const backoff = Math.min(Math.pow(2, attempts) * 1e3, 3e4);
13534
+ await this.queueResource.update(queueId, {
13535
+ status: "pending",
13536
+ visibleAt: Date.now() + backoff,
13537
+ error
13538
+ });
13539
+ this.processedCache.delete(queueId);
13540
+ }
13541
+ async moveToDeadLetter(queueId, record, error) {
13542
+ if (this.config.deadLetterResource && this.deadLetterResourceObj) {
13543
+ const msg = await this.queueResource.get(queueId);
13544
+ await this.deadLetterResourceObj.insert({
13545
+ id: idGenerator(),
13546
+ originalId: record.id,
13547
+ queueId,
13548
+ data: record,
13549
+ error,
13550
+ attempts: msg.attempts,
13551
+ createdAt: (/* @__PURE__ */ new Date()).toISOString()
11814
13552
  });
11815
- if (ok) {
11816
- retried++;
11817
- }
11818
13553
  }
11819
- return { retried };
13554
+ await this.queueResource.update(queueId, {
13555
+ status: "dead",
13556
+ error
13557
+ });
11820
13558
  }
11821
- async syncAllData(replicatorId) {
11822
- const replicator = this.replicators.find((r) => r.id === replicatorId);
11823
- if (!replicator) {
11824
- throw new Error(`Replicator not found: ${replicatorId}`);
13559
+ async getStats() {
13560
+ const [ok, err, allMessages] = await tryFn(
13561
+ () => this.queueResource.list()
13562
+ );
13563
+ if (!ok) {
13564
+ if (this.config.verbose) {
13565
+ console.warn("[S3QueuePlugin] Failed to get stats:", err.message);
13566
+ }
13567
+ return null;
11825
13568
  }
11826
- this.stats.lastSync = (/* @__PURE__ */ new Date()).toISOString();
11827
- for (const resourceName in this.database.resources) {
11828
- if (normalizeResourceName(resourceName) === normalizeResourceName("replicator_logs")) continue;
11829
- if (replicator.shouldReplicateResource(resourceName)) {
11830
- this.emit("replicator.sync.resource", { resourceName, replicatorId });
11831
- const resource = this.database.resources[resourceName];
11832
- const allRecords = await resource.getAll();
11833
- for (const record of allRecords) {
11834
- await replicator.replicate(resourceName, "insert", record, record.id);
11835
- }
13569
+ const stats = {
13570
+ total: allMessages.length,
13571
+ pending: 0,
13572
+ processing: 0,
13573
+ completed: 0,
13574
+ failed: 0,
13575
+ dead: 0
13576
+ };
13577
+ for (const msg of allMessages) {
13578
+ if (stats[msg.status] !== void 0) {
13579
+ stats[msg.status]++;
11836
13580
  }
11837
13581
  }
11838
- this.emit("replicator.sync.completed", { replicatorId, stats: this.stats });
13582
+ return stats;
11839
13583
  }
11840
- async cleanup() {
11841
- const [ok, error] = await tryFn(async () => {
11842
- if (this.replicators && this.replicators.length > 0) {
11843
- const cleanupPromises = this.replicators.map(async (replicator) => {
11844
- const [replicatorOk, replicatorError] = await tryFn(async () => {
11845
- if (replicator && typeof replicator.cleanup === "function") {
11846
- await replicator.cleanup();
11847
- }
11848
- });
11849
- if (!replicatorOk) {
11850
- if (this.config.verbose) {
11851
- console.warn(`[ReplicatorPlugin] Failed to cleanup replicator ${replicator.name || replicator.id}: ${replicatorError.message}`);
11852
- }
11853
- this.emit("replicator_cleanup_error", {
11854
- replicator: replicator.name || replicator.id || "unknown",
11855
- driver: replicator.driver || "unknown",
11856
- error: replicatorError.message
11857
- });
11858
- }
11859
- });
11860
- await Promise.allSettled(cleanupPromises);
11861
- }
11862
- this.replicators = [];
11863
- this.database = null;
11864
- this.eventListenersInstalled.clear();
11865
- this.removeAllListeners();
11866
- });
11867
- if (!ok) {
13584
+ async createDeadLetterResource() {
13585
+ const [ok, err] = await tryFn(
13586
+ () => this.database.createResource({
13587
+ name: this.config.deadLetterResource,
13588
+ attributes: {
13589
+ id: "string|required",
13590
+ originalId: "string|required",
13591
+ queueId: "string|required",
13592
+ data: "json|required",
13593
+ error: "string|required",
13594
+ attempts: "number|required",
13595
+ createdAt: "string|required"
13596
+ },
13597
+ behavior: "body-overflow",
13598
+ timestamps: true
13599
+ })
13600
+ );
13601
+ if (ok || this.database.resources[this.config.deadLetterResource]) {
13602
+ this.deadLetterResourceObj = this.database.resources[this.config.deadLetterResource];
11868
13603
  if (this.config.verbose) {
11869
- console.warn(`[ReplicatorPlugin] Failed to cleanup plugin: ${error.message}`);
13604
+ console.log(`[S3QueuePlugin] Dead letter queue created: ${this.config.deadLetterResource}`);
11870
13605
  }
11871
- this.emit("replicator_plugin_cleanup_error", {
11872
- error: error.message
11873
- });
11874
13606
  }
11875
13607
  }
11876
13608
  }
@@ -11884,7 +13616,7 @@ class SchedulerPlugin extends Plugin {
11884
13616
  defaultTimeout: options.defaultTimeout || 3e5,
11885
13617
  // 5 minutes
11886
13618
  defaultRetries: options.defaultRetries || 1,
11887
- jobHistoryResource: options.jobHistoryResource || "job_executions",
13619
+ jobHistoryResource: options.jobHistoryResource || "plg_job_executions",
11888
13620
  persistJobs: options.persistJobs !== false,
11889
13621
  verbose: options.verbose || false,
11890
13622
  onJobStart: options.onJobStart || null,
@@ -11893,12 +13625,20 @@ class SchedulerPlugin extends Plugin {
11893
13625
  ...options
11894
13626
  };
11895
13627
  this.database = null;
13628
+ this.lockResource = null;
11896
13629
  this.jobs = /* @__PURE__ */ new Map();
11897
13630
  this.activeJobs = /* @__PURE__ */ new Map();
11898
13631
  this.timers = /* @__PURE__ */ new Map();
11899
13632
  this.statistics = /* @__PURE__ */ new Map();
11900
13633
  this._validateConfiguration();
11901
13634
  }
13635
+ /**
13636
+ * Helper to detect test environment
13637
+ * @private
13638
+ */
13639
+ _isTestEnvironment() {
13640
+ return process.env.NODE_ENV === "test" || process.env.JEST_WORKER_ID !== void 0 || global.expect !== void 0;
13641
+ }
11902
13642
  _validateConfiguration() {
11903
13643
  if (Object.keys(this.config.jobs).length === 0) {
11904
13644
  throw new Error("SchedulerPlugin: At least one job must be defined");
@@ -11925,6 +13665,7 @@ class SchedulerPlugin extends Plugin {
11925
13665
  }
11926
13666
  async setup(database) {
11927
13667
  this.database = database;
13668
+ await this._createLockResource();
11928
13669
  if (this.config.persistJobs) {
11929
13670
  await this._createJobHistoryResource();
11930
13671
  }
@@ -11953,6 +13694,25 @@ class SchedulerPlugin extends Plugin {
11953
13694
  await this._startScheduling();
11954
13695
  this.emit("initialized", { jobs: this.jobs.size });
11955
13696
  }
13697
+ async _createLockResource() {
13698
+ const [ok, err, lockResource] = await tryFn(
13699
+ () => this.database.createResource({
13700
+ name: "plg_scheduler_job_locks",
13701
+ attributes: {
13702
+ id: "string|required",
13703
+ jobName: "string|required",
13704
+ lockedAt: "number|required",
13705
+ instanceId: "string|optional"
13706
+ },
13707
+ behavior: "body-only",
13708
+ timestamps: false
13709
+ })
13710
+ );
13711
+ if (!ok && !this.database.resources.plg_scheduler_job_locks) {
13712
+ throw new Error(`Failed to create lock resource: ${err?.message}`);
13713
+ }
13714
+ this.lockResource = ok ? lockResource : this.database.resources.plg_scheduler_job_locks;
13715
+ }
11956
13716
  async _createJobHistoryResource() {
11957
13717
  const [ok] = await tryFn(() => this.database.createResource({
11958
13718
  name: this.config.jobHistoryResource,
@@ -12046,18 +13806,37 @@ class SchedulerPlugin extends Plugin {
12046
13806
  next.setHours(next.getHours() + 1);
12047
13807
  }
12048
13808
  }
12049
- const isTestEnvironment = process.env.NODE_ENV === "test" || process.env.JEST_WORKER_ID !== void 0 || global.expect !== void 0;
12050
- if (isTestEnvironment) {
13809
+ if (this._isTestEnvironment()) {
12051
13810
  next.setTime(next.getTime() + 1e3);
12052
13811
  }
12053
13812
  return next;
12054
13813
  }
12055
13814
  async _executeJob(jobName) {
12056
13815
  const job = this.jobs.get(jobName);
12057
- if (!job || this.activeJobs.has(jobName)) {
13816
+ if (!job) {
13817
+ return;
13818
+ }
13819
+ if (this.activeJobs.has(jobName)) {
13820
+ return;
13821
+ }
13822
+ this.activeJobs.set(jobName, "acquiring-lock");
13823
+ const lockId = `lock-${jobName}`;
13824
+ const [lockAcquired, lockErr] = await tryFn(
13825
+ () => this.lockResource.insert({
13826
+ id: lockId,
13827
+ jobName,
13828
+ lockedAt: Date.now(),
13829
+ instanceId: process.pid ? String(process.pid) : "unknown"
13830
+ })
13831
+ );
13832
+ if (!lockAcquired) {
13833
+ if (this.config.verbose) {
13834
+ console.log(`[SchedulerPlugin] Job '${jobName}' already running on another instance`);
13835
+ }
13836
+ this.activeJobs.delete(jobName);
12058
13837
  return;
12059
13838
  }
12060
- const executionId = `${jobName}_${Date.now()}`;
13839
+ const executionId = `${jobName}_${idGenerator()}`;
12061
13840
  const startTime = Date.now();
12062
13841
  const context = {
12063
13842
  jobName,
@@ -12066,91 +13845,95 @@ class SchedulerPlugin extends Plugin {
12066
13845
  database: this.database
12067
13846
  };
12068
13847
  this.activeJobs.set(jobName, executionId);
12069
- if (this.config.onJobStart) {
12070
- await this._executeHook(this.config.onJobStart, jobName, context);
12071
- }
12072
- this.emit("job_start", { jobName, executionId, startTime });
12073
- let attempt = 0;
12074
- let lastError = null;
12075
- let result = null;
12076
- let status = "success";
12077
- const isTestEnvironment = process.env.NODE_ENV === "test" || process.env.JEST_WORKER_ID !== void 0 || global.expect !== void 0;
12078
- while (attempt <= job.retries) {
12079
- try {
12080
- const actualTimeout = isTestEnvironment ? Math.min(job.timeout, 1e3) : job.timeout;
12081
- let timeoutId;
12082
- const timeoutPromise = new Promise((_, reject) => {
12083
- timeoutId = setTimeout(() => reject(new Error("Job execution timeout")), actualTimeout);
12084
- });
12085
- const jobPromise = job.action(this.database, context, this);
13848
+ try {
13849
+ if (this.config.onJobStart) {
13850
+ await this._executeHook(this.config.onJobStart, jobName, context);
13851
+ }
13852
+ this.emit("job_start", { jobName, executionId, startTime });
13853
+ let attempt = 0;
13854
+ let lastError = null;
13855
+ let result = null;
13856
+ let status = "success";
13857
+ const isTestEnvironment = this._isTestEnvironment();
13858
+ while (attempt <= job.retries) {
12086
13859
  try {
12087
- result = await Promise.race([jobPromise, timeoutPromise]);
12088
- clearTimeout(timeoutId);
12089
- } catch (raceError) {
12090
- clearTimeout(timeoutId);
12091
- throw raceError;
12092
- }
12093
- status = "success";
12094
- break;
12095
- } catch (error) {
12096
- lastError = error;
12097
- attempt++;
12098
- if (attempt <= job.retries) {
12099
- if (this.config.verbose) {
12100
- console.warn(`[SchedulerPlugin] Job '${jobName}' failed (attempt ${attempt + 1}):`, error.message);
13860
+ const actualTimeout = isTestEnvironment ? Math.min(job.timeout, 1e3) : job.timeout;
13861
+ let timeoutId;
13862
+ const timeoutPromise = new Promise((_, reject) => {
13863
+ timeoutId = setTimeout(() => reject(new Error("Job execution timeout")), actualTimeout);
13864
+ });
13865
+ const jobPromise = job.action(this.database, context, this);
13866
+ try {
13867
+ result = await Promise.race([jobPromise, timeoutPromise]);
13868
+ clearTimeout(timeoutId);
13869
+ } catch (raceError) {
13870
+ clearTimeout(timeoutId);
13871
+ throw raceError;
13872
+ }
13873
+ status = "success";
13874
+ break;
13875
+ } catch (error) {
13876
+ lastError = error;
13877
+ attempt++;
13878
+ if (attempt <= job.retries) {
13879
+ if (this.config.verbose) {
13880
+ console.warn(`[SchedulerPlugin] Job '${jobName}' failed (attempt ${attempt + 1}):`, error.message);
13881
+ }
13882
+ const baseDelay = Math.min(Math.pow(2, attempt) * 1e3, 5e3);
13883
+ const delay = isTestEnvironment ? 1 : baseDelay;
13884
+ await new Promise((resolve) => setTimeout(resolve, delay));
12101
13885
  }
12102
- const baseDelay = Math.min(Math.pow(2, attempt) * 1e3, 5e3);
12103
- const delay = isTestEnvironment ? 1 : baseDelay;
12104
- await new Promise((resolve) => setTimeout(resolve, delay));
12105
13886
  }
12106
13887
  }
12107
- }
12108
- const endTime = Date.now();
12109
- const duration = Math.max(1, endTime - startTime);
12110
- if (lastError && attempt > job.retries) {
12111
- status = lastError.message.includes("timeout") ? "timeout" : "error";
12112
- }
12113
- job.lastRun = new Date(endTime);
12114
- job.runCount++;
12115
- if (status === "success") {
12116
- job.successCount++;
12117
- } else {
12118
- job.errorCount++;
12119
- }
12120
- const stats = this.statistics.get(jobName);
12121
- stats.totalRuns++;
12122
- stats.lastRun = new Date(endTime);
12123
- if (status === "success") {
12124
- stats.totalSuccesses++;
12125
- stats.lastSuccess = new Date(endTime);
12126
- } else {
12127
- stats.totalErrors++;
12128
- stats.lastError = { time: new Date(endTime), message: lastError?.message };
12129
- }
12130
- stats.avgDuration = (stats.avgDuration * (stats.totalRuns - 1) + duration) / stats.totalRuns;
12131
- if (this.config.persistJobs) {
12132
- await this._persistJobExecution(jobName, executionId, startTime, endTime, duration, status, result, lastError, attempt);
12133
- }
12134
- if (status === "success" && this.config.onJobComplete) {
12135
- await this._executeHook(this.config.onJobComplete, jobName, result, duration);
12136
- } else if (status !== "success" && this.config.onJobError) {
12137
- await this._executeHook(this.config.onJobError, jobName, lastError, attempt);
12138
- }
12139
- this.emit("job_complete", {
12140
- jobName,
12141
- executionId,
12142
- status,
12143
- duration,
12144
- result,
12145
- error: lastError?.message,
12146
- retryCount: attempt
12147
- });
12148
- this.activeJobs.delete(jobName);
12149
- if (job.enabled) {
12150
- this._scheduleNextExecution(jobName);
12151
- }
12152
- if (lastError && status !== "success") {
12153
- throw lastError;
13888
+ const endTime = Date.now();
13889
+ const duration = Math.max(1, endTime - startTime);
13890
+ if (lastError && attempt > job.retries) {
13891
+ status = lastError.message.includes("timeout") ? "timeout" : "error";
13892
+ }
13893
+ job.lastRun = new Date(endTime);
13894
+ job.runCount++;
13895
+ if (status === "success") {
13896
+ job.successCount++;
13897
+ } else {
13898
+ job.errorCount++;
13899
+ }
13900
+ const stats = this.statistics.get(jobName);
13901
+ stats.totalRuns++;
13902
+ stats.lastRun = new Date(endTime);
13903
+ if (status === "success") {
13904
+ stats.totalSuccesses++;
13905
+ stats.lastSuccess = new Date(endTime);
13906
+ } else {
13907
+ stats.totalErrors++;
13908
+ stats.lastError = { time: new Date(endTime), message: lastError?.message };
13909
+ }
13910
+ stats.avgDuration = (stats.avgDuration * (stats.totalRuns - 1) + duration) / stats.totalRuns;
13911
+ if (this.config.persistJobs) {
13912
+ await this._persistJobExecution(jobName, executionId, startTime, endTime, duration, status, result, lastError, attempt);
13913
+ }
13914
+ if (status === "success" && this.config.onJobComplete) {
13915
+ await this._executeHook(this.config.onJobComplete, jobName, result, duration);
13916
+ } else if (status !== "success" && this.config.onJobError) {
13917
+ await this._executeHook(this.config.onJobError, jobName, lastError, attempt);
13918
+ }
13919
+ this.emit("job_complete", {
13920
+ jobName,
13921
+ executionId,
13922
+ status,
13923
+ duration,
13924
+ result,
13925
+ error: lastError?.message,
13926
+ retryCount: attempt
13927
+ });
13928
+ this.activeJobs.delete(jobName);
13929
+ if (job.enabled) {
13930
+ this._scheduleNextExecution(jobName);
13931
+ }
13932
+ if (lastError && status !== "success") {
13933
+ throw lastError;
13934
+ }
13935
+ } finally {
13936
+ await tryFn(() => this.lockResource.delete(lockId));
12154
13937
  }
12155
13938
  }
12156
13939
  async _persistJobExecution(jobName, executionId, startTime, endTime, duration, status, result, error, retryCount) {
@@ -12182,6 +13965,7 @@ class SchedulerPlugin extends Plugin {
12182
13965
  }
12183
13966
  /**
12184
13967
  * Manually trigger a job execution
13968
+ * Note: Race conditions are prevented by distributed locking in _executeJob()
12185
13969
  */
12186
13970
  async runJob(jobName, context = {}) {
12187
13971
  const job = this.jobs.get(jobName);
@@ -12267,12 +14051,15 @@ class SchedulerPlugin extends Plugin {
12267
14051
  return [];
12268
14052
  }
12269
14053
  const { limit = 50, status = null } = options;
12270
- const [ok, err, allHistory] = await tryFn(
12271
- () => this.database.resource(this.config.jobHistoryResource).list({
12272
- orderBy: { startTime: "desc" },
12273
- limit: limit * 2
12274
- // Get more to allow for filtering
12275
- })
14054
+ const queryParams = {
14055
+ jobName
14056
+ // Uses byJob partition for efficient lookup
14057
+ };
14058
+ if (status) {
14059
+ queryParams.status = status;
14060
+ }
14061
+ const [ok, err, history] = await tryFn(
14062
+ () => this.database.resource(this.config.jobHistoryResource).query(queryParams)
12276
14063
  );
12277
14064
  if (!ok) {
12278
14065
  if (this.config.verbose) {
@@ -12280,11 +14067,7 @@ class SchedulerPlugin extends Plugin {
12280
14067
  }
12281
14068
  return [];
12282
14069
  }
12283
- let filtered = allHistory.filter((h) => h.jobName === jobName);
12284
- if (status) {
12285
- filtered = filtered.filter((h) => h.status === status);
12286
- }
12287
- filtered = filtered.sort((a, b) => b.startTime - a.startTime).slice(0, limit);
14070
+ let filtered = history.sort((a, b) => b.startTime - a.startTime).slice(0, limit);
12288
14071
  return filtered.map((h) => {
12289
14072
  let result = null;
12290
14073
  if (h.result) {
@@ -12379,8 +14162,7 @@ class SchedulerPlugin extends Plugin {
12379
14162
  clearTimeout(timer);
12380
14163
  }
12381
14164
  this.timers.clear();
12382
- const isTestEnvironment = process.env.NODE_ENV === "test" || process.env.JEST_WORKER_ID !== void 0 || global.expect !== void 0;
12383
- if (!isTestEnvironment && this.activeJobs.size > 0) {
14165
+ if (!this._isTestEnvironment() && this.activeJobs.size > 0) {
12384
14166
  if (this.config.verbose) {
12385
14167
  console.log(`[SchedulerPlugin] Waiting for ${this.activeJobs.size} active jobs to complete...`);
12386
14168
  }
@@ -12393,7 +14175,7 @@ class SchedulerPlugin extends Plugin {
12393
14175
  console.warn(`[SchedulerPlugin] ${this.activeJobs.size} jobs still running after timeout`);
12394
14176
  }
12395
14177
  }
12396
- if (isTestEnvironment) {
14178
+ if (this._isTestEnvironment()) {
12397
14179
  this.activeJobs.clear();
12398
14180
  }
12399
14181
  }
@@ -12414,14 +14196,14 @@ class StateMachinePlugin extends Plugin {
12414
14196
  actions: options.actions || {},
12415
14197
  guards: options.guards || {},
12416
14198
  persistTransitions: options.persistTransitions !== false,
12417
- transitionLogResource: options.transitionLogResource || "state_transitions",
12418
- stateResource: options.stateResource || "entity_states",
12419
- verbose: options.verbose || false,
12420
- ...options
14199
+ transitionLogResource: options.transitionLogResource || "plg_state_transitions",
14200
+ stateResource: options.stateResource || "plg_entity_states",
14201
+ retryAttempts: options.retryAttempts || 3,
14202
+ retryDelay: options.retryDelay || 100,
14203
+ verbose: options.verbose || false
12421
14204
  };
12422
14205
  this.database = null;
12423
14206
  this.machines = /* @__PURE__ */ new Map();
12424
- this.stateStorage = /* @__PURE__ */ new Map();
12425
14207
  this._validateConfiguration();
12426
14208
  }
12427
14209
  _validateConfiguration() {
@@ -12562,43 +14344,55 @@ class StateMachinePlugin extends Plugin {
12562
14344
  machine.currentStates.set(entityId, toState);
12563
14345
  if (this.config.persistTransitions) {
12564
14346
  const transitionId = `${machineId}_${entityId}_${timestamp}`;
12565
- const [logOk, logErr] = await tryFn(
12566
- () => this.database.resource(this.config.transitionLogResource).insert({
12567
- id: transitionId,
12568
- machineId,
12569
- entityId,
12570
- fromState,
12571
- toState,
12572
- event,
12573
- context,
12574
- timestamp,
12575
- createdAt: now.slice(0, 10)
12576
- // YYYY-MM-DD for partitioning
12577
- })
12578
- );
14347
+ let logOk = false;
14348
+ let lastLogErr;
14349
+ for (let attempt = 0; attempt < this.config.retryAttempts; attempt++) {
14350
+ const [ok, err] = await tryFn(
14351
+ () => this.database.resource(this.config.transitionLogResource).insert({
14352
+ id: transitionId,
14353
+ machineId,
14354
+ entityId,
14355
+ fromState,
14356
+ toState,
14357
+ event,
14358
+ context,
14359
+ timestamp,
14360
+ createdAt: now.slice(0, 10)
14361
+ // YYYY-MM-DD for partitioning
14362
+ })
14363
+ );
14364
+ if (ok) {
14365
+ logOk = true;
14366
+ break;
14367
+ }
14368
+ lastLogErr = err;
14369
+ if (attempt < this.config.retryAttempts - 1) {
14370
+ const delay = this.config.retryDelay * Math.pow(2, attempt);
14371
+ await new Promise((resolve) => setTimeout(resolve, delay));
14372
+ }
14373
+ }
12579
14374
  if (!logOk && this.config.verbose) {
12580
- console.warn(`[StateMachinePlugin] Failed to log transition:`, logErr.message);
14375
+ console.warn(`[StateMachinePlugin] Failed to log transition after ${this.config.retryAttempts} attempts:`, lastLogErr.message);
12581
14376
  }
12582
14377
  const stateId = `${machineId}_${entityId}`;
12583
- const [stateOk, stateErr] = await tryFn(async () => {
12584
- const exists = await this.database.resource(this.config.stateResource).exists(stateId);
12585
- const stateData = {
12586
- id: stateId,
12587
- machineId,
12588
- entityId,
12589
- currentState: toState,
12590
- context,
12591
- lastTransition: transitionId,
12592
- updatedAt: now
12593
- };
12594
- if (exists) {
12595
- await this.database.resource(this.config.stateResource).update(stateId, stateData);
12596
- } else {
12597
- await this.database.resource(this.config.stateResource).insert(stateData);
14378
+ const stateData = {
14379
+ machineId,
14380
+ entityId,
14381
+ currentState: toState,
14382
+ context,
14383
+ lastTransition: transitionId,
14384
+ updatedAt: now
14385
+ };
14386
+ const [updateOk] = await tryFn(
14387
+ () => this.database.resource(this.config.stateResource).update(stateId, stateData)
14388
+ );
14389
+ if (!updateOk) {
14390
+ const [insertOk, insertErr] = await tryFn(
14391
+ () => this.database.resource(this.config.stateResource).insert({ id: stateId, ...stateData })
14392
+ );
14393
+ if (!insertOk && this.config.verbose) {
14394
+ console.warn(`[StateMachinePlugin] Failed to upsert state:`, insertErr.message);
12598
14395
  }
12599
- });
12600
- if (!stateOk && this.config.verbose) {
12601
- console.warn(`[StateMachinePlugin] Failed to update state:`, stateErr.message);
12602
14396
  }
12603
14397
  }
12604
14398
  }
@@ -12629,8 +14423,9 @@ class StateMachinePlugin extends Plugin {
12629
14423
  }
12630
14424
  /**
12631
14425
  * Get valid events for current state
14426
+ * Can accept either a state name (sync) or entityId (async to fetch latest state)
12632
14427
  */
12633
- getValidEvents(machineId, stateOrEntityId) {
14428
+ async getValidEvents(machineId, stateOrEntityId) {
12634
14429
  const machine = this.machines.get(machineId);
12635
14430
  if (!machine) {
12636
14431
  throw new Error(`State machine '${machineId}' not found`);
@@ -12639,7 +14434,7 @@ class StateMachinePlugin extends Plugin {
12639
14434
  if (machine.config.states[stateOrEntityId]) {
12640
14435
  state = stateOrEntityId;
12641
14436
  } else {
12642
- state = machine.currentStates.get(stateOrEntityId) || machine.config.initialState;
14437
+ state = await this.getState(machineId, stateOrEntityId);
12643
14438
  }
12644
14439
  const stateConfig = machine.config.states[state];
12645
14440
  return stateConfig && stateConfig.on ? Object.keys(stateConfig.on) : [];
@@ -12653,9 +14448,10 @@ class StateMachinePlugin extends Plugin {
12653
14448
  }
12654
14449
  const { limit = 50, offset = 0 } = options;
12655
14450
  const [ok, err, transitions] = await tryFn(
12656
- () => this.database.resource(this.config.transitionLogResource).list({
12657
- where: { machineId, entityId },
12658
- orderBy: { timestamp: "desc" },
14451
+ () => this.database.resource(this.config.transitionLogResource).query({
14452
+ machineId,
14453
+ entityId
14454
+ }, {
12659
14455
  limit,
12660
14456
  offset
12661
14457
  })
@@ -12666,8 +14462,8 @@ class StateMachinePlugin extends Plugin {
12666
14462
  }
12667
14463
  return [];
12668
14464
  }
12669
- const sortedTransitions = transitions.sort((a, b) => b.timestamp - a.timestamp);
12670
- return sortedTransitions.map((t) => ({
14465
+ const sorted = (transitions || []).sort((a, b) => b.timestamp - a.timestamp);
14466
+ return sorted.map((t) => ({
12671
14467
  from: t.fromState,
12672
14468
  to: t.toState,
12673
14469
  event: t.event,
@@ -12688,15 +14484,20 @@ class StateMachinePlugin extends Plugin {
12688
14484
  if (this.config.persistTransitions) {
12689
14485
  const now = (/* @__PURE__ */ new Date()).toISOString();
12690
14486
  const stateId = `${machineId}_${entityId}`;
12691
- await this.database.resource(this.config.stateResource).insert({
12692
- id: stateId,
12693
- machineId,
12694
- entityId,
12695
- currentState: initialState,
12696
- context,
12697
- lastTransition: null,
12698
- updatedAt: now
12699
- });
14487
+ const [ok, err] = await tryFn(
14488
+ () => this.database.resource(this.config.stateResource).insert({
14489
+ id: stateId,
14490
+ machineId,
14491
+ entityId,
14492
+ currentState: initialState,
14493
+ context,
14494
+ lastTransition: null,
14495
+ updatedAt: now
14496
+ })
14497
+ );
14498
+ if (!ok && err && !err.message?.includes("already exists")) {
14499
+ throw new Error(`Failed to initialize entity state: ${err.message}`);
14500
+ }
12700
14501
  }
12701
14502
  const initialStateConfig = machine.config.states[initialState];
12702
14503
  if (initialStateConfig && initialStateConfig.entry) {
@@ -12761,7 +14562,6 @@ class StateMachinePlugin extends Plugin {
12761
14562
  }
12762
14563
  async stop() {
12763
14564
  this.machines.clear();
12764
- this.stateStorage.clear();
12765
14565
  }
12766
14566
  async cleanup() {
12767
14567
  await this.stop();
@@ -12785,6 +14585,7 @@ exports.Database = Database;
12785
14585
  exports.DatabaseError = DatabaseError;
12786
14586
  exports.EncryptionError = EncryptionError;
12787
14587
  exports.ErrorMap = ErrorMap;
14588
+ exports.EventualConsistencyPlugin = EventualConsistencyPlugin;
12788
14589
  exports.FullTextPlugin = FullTextPlugin;
12789
14590
  exports.InvalidResourceItem = InvalidResourceItem;
12790
14591
  exports.MetricsPlugin = MetricsPlugin;
@@ -12804,6 +14605,7 @@ exports.ResourceIdsReader = ResourceIdsReader;
12804
14605
  exports.ResourceNotFound = ResourceNotFound;
12805
14606
  exports.ResourceReader = ResourceReader;
12806
14607
  exports.ResourceWriter = ResourceWriter;
14608
+ exports.S3QueuePlugin = S3QueuePlugin;
12807
14609
  exports.S3db = Database;
12808
14610
  exports.S3dbError = S3dbError;
12809
14611
  exports.SchedulerPlugin = SchedulerPlugin;