s3db.js 9.3.0 → 10.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/s3db.es.js CHANGED
@@ -6,11 +6,12 @@ import { pipeline } from 'stream/promises';
6
6
  import path, { join } from 'path';
7
7
  import crypto, { createHash } from 'crypto';
8
8
  import zlib from 'node:zlib';
9
+ import os from 'os';
10
+ import jsonStableStringify from 'json-stable-stringify';
9
11
  import { Transform, Writable } from 'stream';
10
12
  import { PromisePool } from '@supercharge/promise-pool';
11
13
  import { ReadableStream } from 'node:stream/web';
12
14
  import { chunk, merge, isString, isEmpty, invert, uniq, cloneDeep, get, set, isObject, isFunction } from 'lodash-es';
13
- import jsonStableStringify from 'json-stable-stringify';
14
15
  import { Agent } from 'http';
15
16
  import { Agent as Agent$1 } from 'https';
16
17
  import { NodeHttpHandler } from '@smithy/node-http-handler';
@@ -419,8 +420,14 @@ function mapAwsError(err, context = {}) {
419
420
  suggestion = "Check if the object metadata is present and valid.";
420
421
  return new MissingMetadata({ ...context, original: err, metadata, commandName, commandInput, suggestion });
421
422
  }
422
- suggestion = "Check the error details and AWS documentation.";
423
- return new UnknownError("Unknown error", { ...context, original: err, metadata, commandName, commandInput, suggestion });
423
+ const errorDetails = [
424
+ `Unknown error: ${err.message || err.toString()}`,
425
+ err.code && `Code: ${err.code}`,
426
+ err.statusCode && `Status: ${err.statusCode}`,
427
+ err.stack && `Stack: ${err.stack.split("\n")[0]}`
428
+ ].filter(Boolean).join(" | ");
429
+ suggestion = `Check the error details and AWS documentation. Original error: ${err.message || err.toString()}`;
430
+ return new UnknownError(errorDetails, { ...context, original: err, metadata, commandName, commandInput, suggestion });
424
431
  }
425
432
  class ConnectionStringError extends S3dbError {
426
433
  constructor(message, details = {}) {
@@ -832,7 +839,7 @@ class AuditPlugin extends Plugin {
832
839
  }
833
840
  async onSetup() {
834
841
  const [ok, err, auditResource] = await tryFn(() => this.database.createResource({
835
- name: "audits",
842
+ name: "plg_audits",
836
843
  attributes: {
837
844
  id: "string|required",
838
845
  resourceName: "string|required",
@@ -848,15 +855,15 @@ class AuditPlugin extends Plugin {
848
855
  },
849
856
  behavior: "body-overflow"
850
857
  }));
851
- this.auditResource = ok ? auditResource : this.database.resources.audits || null;
858
+ this.auditResource = ok ? auditResource : this.database.resources.plg_audits || null;
852
859
  if (!ok && !this.auditResource) return;
853
860
  this.database.addHook("afterCreateResource", (context) => {
854
- if (context.resource.name !== "audits") {
861
+ if (context.resource.name !== "plg_audits") {
855
862
  this.setupResourceAuditing(context.resource);
856
863
  }
857
864
  });
858
865
  for (const resource of Object.values(this.database.resources)) {
859
- if (resource.name !== "audits") {
866
+ if (resource.name !== "plg_audits") {
860
867
  this.setupResourceAuditing(resource);
861
868
  }
862
869
  }
@@ -1876,11 +1883,10 @@ function validateBackupConfig(driver, config = {}) {
1876
1883
  class BackupPlugin extends Plugin {
1877
1884
  constructor(options = {}) {
1878
1885
  super();
1879
- this.driverName = options.driver || "filesystem";
1880
- this.driverConfig = options.config || {};
1881
1886
  this.config = {
1882
- // Legacy destinations support (will be converted to multi driver)
1883
- destinations: options.destinations || null,
1887
+ // Driver configuration
1888
+ driver: options.driver || "filesystem",
1889
+ driverConfig: options.config || {},
1884
1890
  // Scheduling configuration
1885
1891
  schedule: options.schedule || {},
1886
1892
  // Retention policy (Grandfather-Father-Son)
@@ -1898,8 +1904,8 @@ class BackupPlugin extends Plugin {
1898
1904
  parallelism: options.parallelism || 4,
1899
1905
  include: options.include || null,
1900
1906
  exclude: options.exclude || [],
1901
- backupMetadataResource: options.backupMetadataResource || "backup_metadata",
1902
- tempDir: options.tempDir || "./tmp/backups",
1907
+ backupMetadataResource: options.backupMetadataResource || "plg_backup_metadata",
1908
+ tempDir: options.tempDir || path.join(os.tmpdir(), "s3db", "backups"),
1903
1909
  verbose: options.verbose || false,
1904
1910
  // Hooks
1905
1911
  onBackupStart: options.onBackupStart || null,
@@ -1911,32 +1917,9 @@ class BackupPlugin extends Plugin {
1911
1917
  };
1912
1918
  this.driver = null;
1913
1919
  this.activeBackups = /* @__PURE__ */ new Set();
1914
- this._handleLegacyDestinations();
1915
- validateBackupConfig(this.driverName, this.driverConfig);
1920
+ validateBackupConfig(this.config.driver, this.config.driverConfig);
1916
1921
  this._validateConfiguration();
1917
1922
  }
1918
- /**
1919
- * Convert legacy destinations format to multi driver format
1920
- */
1921
- _handleLegacyDestinations() {
1922
- if (this.config.destinations && Array.isArray(this.config.destinations)) {
1923
- this.driverName = "multi";
1924
- this.driverConfig = {
1925
- strategy: "all",
1926
- destinations: this.config.destinations.map((dest) => {
1927
- const { type, ...config } = dest;
1928
- return {
1929
- driver: type,
1930
- config
1931
- };
1932
- })
1933
- };
1934
- this.config.destinations = null;
1935
- if (this.config.verbose) {
1936
- console.log("[BackupPlugin] Converted legacy destinations format to multi driver");
1937
- }
1938
- }
1939
- }
1940
1923
  _validateConfiguration() {
1941
1924
  if (this.config.encryption && (!this.config.encryption.key || !this.config.encryption.algorithm)) {
1942
1925
  throw new Error("BackupPlugin: Encryption requires both key and algorithm");
@@ -1946,7 +1929,7 @@ class BackupPlugin extends Plugin {
1946
1929
  }
1947
1930
  }
1948
1931
  async onSetup() {
1949
- this.driver = createBackupDriver(this.driverName, this.driverConfig);
1932
+ this.driver = createBackupDriver(this.config.driver, this.config.driverConfig);
1950
1933
  await this.driver.setup(this.database);
1951
1934
  await mkdir(this.config.tempDir, { recursive: true });
1952
1935
  await this._createBackupMetadataResource();
@@ -1994,6 +1977,9 @@ class BackupPlugin extends Plugin {
1994
1977
  async backup(type = "full", options = {}) {
1995
1978
  const backupId = this._generateBackupId(type);
1996
1979
  const startTime = Date.now();
1980
+ if (this.activeBackups.has(backupId)) {
1981
+ throw new Error(`Backup '${backupId}' is already in progress`);
1982
+ }
1997
1983
  try {
1998
1984
  this.activeBackups.add(backupId);
1999
1985
  if (this.config.onBackupStart) {
@@ -2009,16 +1995,9 @@ class BackupPlugin extends Plugin {
2009
1995
  if (exportedFiles.length === 0) {
2010
1996
  throw new Error("No resources were exported for backup");
2011
1997
  }
2012
- let finalPath;
2013
- let totalSize = 0;
2014
- if (this.config.compression !== "none") {
2015
- finalPath = path.join(tempBackupDir, `${backupId}.tar.gz`);
2016
- totalSize = await this._createCompressedArchive(exportedFiles, finalPath);
2017
- } else {
2018
- finalPath = exportedFiles[0];
2019
- const [statOk, , stats] = await tryFn(() => stat(finalPath));
2020
- totalSize = statOk ? stats.size : 0;
2021
- }
1998
+ const archiveExtension = this.config.compression !== "none" ? ".tar.gz" : ".json";
1999
+ const finalPath = path.join(tempBackupDir, `${backupId}${archiveExtension}`);
2000
+ const totalSize = await this._createArchive(exportedFiles, finalPath, this.config.compression);
2022
2001
  const checksum = await this._generateChecksum(finalPath);
2023
2002
  const uploadResult = await this.driver.upload(finalPath, backupId, manifest);
2024
2003
  if (this.config.verification) {
@@ -2127,15 +2106,35 @@ class BackupPlugin extends Plugin {
2127
2106
  for (const resourceName of resourceNames) {
2128
2107
  const resource = this.database.resources[resourceName];
2129
2108
  if (!resource) {
2130
- console.warn(`[BackupPlugin] Resource '${resourceName}' not found, skipping`);
2109
+ if (this.config.verbose) {
2110
+ console.warn(`[BackupPlugin] Resource '${resourceName}' not found, skipping`);
2111
+ }
2131
2112
  continue;
2132
2113
  }
2133
2114
  const exportPath = path.join(tempDir, `${resourceName}.json`);
2134
2115
  let records;
2135
2116
  if (type === "incremental") {
2136
- const yesterday = new Date(Date.now() - 24 * 60 * 60 * 1e3);
2117
+ const [lastBackupOk, , lastBackups] = await tryFn(
2118
+ () => this.database.resource(this.config.backupMetadataResource).list({
2119
+ filter: {
2120
+ status: "completed",
2121
+ type: { $in: ["full", "incremental"] }
2122
+ },
2123
+ sort: { timestamp: -1 },
2124
+ limit: 1
2125
+ })
2126
+ );
2127
+ let sinceTimestamp;
2128
+ if (lastBackupOk && lastBackups && lastBackups.length > 0) {
2129
+ sinceTimestamp = new Date(lastBackups[0].timestamp);
2130
+ } else {
2131
+ sinceTimestamp = new Date(Date.now() - 24 * 60 * 60 * 1e3);
2132
+ }
2133
+ if (this.config.verbose) {
2134
+ console.log(`[BackupPlugin] Incremental backup for '${resourceName}' since ${sinceTimestamp.toISOString()}`);
2135
+ }
2137
2136
  records = await resource.list({
2138
- filter: { updatedAt: { ">": yesterday.toISOString() } }
2137
+ filter: { updatedAt: { ">": sinceTimestamp.toISOString() } }
2139
2138
  });
2140
2139
  } else {
2141
2140
  records = await resource.list();
@@ -2155,29 +2154,57 @@ class BackupPlugin extends Plugin {
2155
2154
  }
2156
2155
  return exportedFiles;
2157
2156
  }
2158
- async _createCompressedArchive(files, targetPath) {
2159
- const output = createWriteStream(targetPath);
2160
- const gzip = zlib.createGzip({ level: 6 });
2157
+ async _createArchive(files, targetPath, compressionType) {
2158
+ const archive = {
2159
+ version: "1.0",
2160
+ created: (/* @__PURE__ */ new Date()).toISOString(),
2161
+ files: []
2162
+ };
2161
2163
  let totalSize = 0;
2162
- await pipeline(
2163
- async function* () {
2164
- for (const filePath of files) {
2165
- const content = await readFile(filePath);
2166
- totalSize += content.length;
2167
- yield content;
2164
+ for (const filePath of files) {
2165
+ const [readOk, readErr, content] = await tryFn(() => readFile(filePath, "utf8"));
2166
+ if (!readOk) {
2167
+ if (this.config.verbose) {
2168
+ console.warn(`[BackupPlugin] Failed to read ${filePath}: ${readErr?.message}`);
2168
2169
  }
2169
- },
2170
- gzip,
2171
- output
2172
- );
2170
+ continue;
2171
+ }
2172
+ const fileName = path.basename(filePath);
2173
+ totalSize += content.length;
2174
+ archive.files.push({
2175
+ name: fileName,
2176
+ size: content.length,
2177
+ content
2178
+ });
2179
+ }
2180
+ const archiveJson = JSON.stringify(archive);
2181
+ if (compressionType === "none") {
2182
+ await writeFile(targetPath, archiveJson, "utf8");
2183
+ } else {
2184
+ const output = createWriteStream(targetPath);
2185
+ const gzip = zlib.createGzip({ level: 6 });
2186
+ await pipeline(
2187
+ async function* () {
2188
+ yield Buffer.from(archiveJson, "utf8");
2189
+ },
2190
+ gzip,
2191
+ output
2192
+ );
2193
+ }
2173
2194
  const [statOk, , stats] = await tryFn(() => stat(targetPath));
2174
2195
  return statOk ? stats.size : totalSize;
2175
2196
  }
2176
2197
  async _generateChecksum(filePath) {
2177
- const hash = crypto.createHash("sha256");
2178
- const stream = createReadStream(filePath);
2179
- await pipeline(stream, hash);
2180
- return hash.digest("hex");
2198
+ const [ok, err, result] = await tryFn(async () => {
2199
+ const hash = crypto.createHash("sha256");
2200
+ const stream = createReadStream(filePath);
2201
+ await pipeline(stream, hash);
2202
+ return hash.digest("hex");
2203
+ });
2204
+ if (!ok) {
2205
+ throw new Error(`Failed to generate checksum for ${filePath}: ${err?.message}`);
2206
+ }
2207
+ return result;
2181
2208
  }
2182
2209
  async _cleanupTempFiles(tempDir) {
2183
2210
  const [ok] = await tryFn(
@@ -2239,7 +2266,109 @@ class BackupPlugin extends Plugin {
2239
2266
  }
2240
2267
  async _restoreFromBackup(backupPath, options) {
2241
2268
  const restoredResources = [];
2242
- return restoredResources;
2269
+ try {
2270
+ let archiveData = "";
2271
+ if (this.config.compression !== "none") {
2272
+ const input = createReadStream(backupPath);
2273
+ const gunzip = zlib.createGunzip();
2274
+ const chunks = [];
2275
+ await new Promise((resolve, reject) => {
2276
+ input.pipe(gunzip).on("data", (chunk) => chunks.push(chunk)).on("end", resolve).on("error", reject);
2277
+ });
2278
+ archiveData = Buffer.concat(chunks).toString("utf8");
2279
+ } else {
2280
+ archiveData = await readFile(backupPath, "utf8");
2281
+ }
2282
+ let archive;
2283
+ try {
2284
+ archive = JSON.parse(archiveData);
2285
+ } catch (parseError) {
2286
+ throw new Error(`Failed to parse backup archive: ${parseError.message}`);
2287
+ }
2288
+ if (!archive || typeof archive !== "object") {
2289
+ throw new Error("Invalid backup archive: not a valid JSON object");
2290
+ }
2291
+ if (!archive.version || !archive.files) {
2292
+ throw new Error("Invalid backup archive format: missing version or files array");
2293
+ }
2294
+ if (this.config.verbose) {
2295
+ console.log(`[BackupPlugin] Restoring ${archive.files.length} files from backup`);
2296
+ }
2297
+ for (const file of archive.files) {
2298
+ try {
2299
+ const resourceData = JSON.parse(file.content);
2300
+ if (!resourceData.resourceName || !resourceData.definition) {
2301
+ if (this.config.verbose) {
2302
+ console.warn(`[BackupPlugin] Skipping invalid file: ${file.name}`);
2303
+ }
2304
+ continue;
2305
+ }
2306
+ const resourceName = resourceData.resourceName;
2307
+ if (options.resources && !options.resources.includes(resourceName)) {
2308
+ continue;
2309
+ }
2310
+ let resource = this.database.resources[resourceName];
2311
+ if (!resource) {
2312
+ if (this.config.verbose) {
2313
+ console.log(`[BackupPlugin] Creating resource '${resourceName}'`);
2314
+ }
2315
+ const [createOk, createErr] = await tryFn(
2316
+ () => this.database.createResource(resourceData.definition)
2317
+ );
2318
+ if (!createOk) {
2319
+ if (this.config.verbose) {
2320
+ console.warn(`[BackupPlugin] Failed to create resource '${resourceName}': ${createErr?.message}`);
2321
+ }
2322
+ continue;
2323
+ }
2324
+ resource = this.database.resources[resourceName];
2325
+ }
2326
+ if (resourceData.records && Array.isArray(resourceData.records)) {
2327
+ const mode = options.mode || "merge";
2328
+ if (mode === "replace") {
2329
+ const ids = await resource.listIds();
2330
+ for (const id of ids) {
2331
+ await resource.delete(id);
2332
+ }
2333
+ }
2334
+ let insertedCount = 0;
2335
+ for (const record of resourceData.records) {
2336
+ const [insertOk] = await tryFn(async () => {
2337
+ if (mode === "skip") {
2338
+ const existing = await resource.get(record.id);
2339
+ if (existing) {
2340
+ return false;
2341
+ }
2342
+ }
2343
+ await resource.insert(record);
2344
+ return true;
2345
+ });
2346
+ if (insertOk) {
2347
+ insertedCount++;
2348
+ }
2349
+ }
2350
+ restoredResources.push({
2351
+ name: resourceName,
2352
+ recordsRestored: insertedCount,
2353
+ totalRecords: resourceData.records.length
2354
+ });
2355
+ if (this.config.verbose) {
2356
+ console.log(`[BackupPlugin] Restored ${insertedCount}/${resourceData.records.length} records to '${resourceName}'`);
2357
+ }
2358
+ }
2359
+ } catch (fileError) {
2360
+ if (this.config.verbose) {
2361
+ console.warn(`[BackupPlugin] Error processing file ${file.name}: ${fileError.message}`);
2362
+ }
2363
+ }
2364
+ }
2365
+ return restoredResources;
2366
+ } catch (error) {
2367
+ if (this.config.verbose) {
2368
+ console.error(`[BackupPlugin] Error restoring backup: ${error.message}`);
2369
+ }
2370
+ throw new Error(`Failed to restore backup: ${error.message}`);
2371
+ }
2243
2372
  }
2244
2373
  /**
2245
2374
  * List available backups
@@ -2283,6 +2412,90 @@ class BackupPlugin extends Plugin {
2283
2412
  return ok ? backup : null;
2284
2413
  }
2285
2414
  async _cleanupOldBackups() {
2415
+ try {
2416
+ const [listOk, , allBackups] = await tryFn(
2417
+ () => this.database.resource(this.config.backupMetadataResource).list({
2418
+ filter: { status: "completed" },
2419
+ sort: { timestamp: -1 }
2420
+ })
2421
+ );
2422
+ if (!listOk || !allBackups || allBackups.length === 0) {
2423
+ return;
2424
+ }
2425
+ const now = Date.now();
2426
+ const msPerDay = 24 * 60 * 60 * 1e3;
2427
+ const msPerWeek = 7 * msPerDay;
2428
+ const msPerMonth = 30 * msPerDay;
2429
+ const msPerYear = 365 * msPerDay;
2430
+ const categorized = {
2431
+ daily: [],
2432
+ weekly: [],
2433
+ monthly: [],
2434
+ yearly: []
2435
+ };
2436
+ for (const backup of allBackups) {
2437
+ const age = now - backup.timestamp;
2438
+ if (age <= msPerDay * this.config.retention.daily) {
2439
+ categorized.daily.push(backup);
2440
+ } else if (age <= msPerWeek * this.config.retention.weekly) {
2441
+ categorized.weekly.push(backup);
2442
+ } else if (age <= msPerMonth * this.config.retention.monthly) {
2443
+ categorized.monthly.push(backup);
2444
+ } else if (age <= msPerYear * this.config.retention.yearly) {
2445
+ categorized.yearly.push(backup);
2446
+ }
2447
+ }
2448
+ const toKeep = /* @__PURE__ */ new Set();
2449
+ categorized.daily.forEach((b) => toKeep.add(b.id));
2450
+ const weeklyByWeek = /* @__PURE__ */ new Map();
2451
+ for (const backup of categorized.weekly) {
2452
+ const weekNum = Math.floor((now - backup.timestamp) / msPerWeek);
2453
+ if (!weeklyByWeek.has(weekNum)) {
2454
+ weeklyByWeek.set(weekNum, backup);
2455
+ toKeep.add(backup.id);
2456
+ }
2457
+ }
2458
+ const monthlyByMonth = /* @__PURE__ */ new Map();
2459
+ for (const backup of categorized.monthly) {
2460
+ const monthNum = Math.floor((now - backup.timestamp) / msPerMonth);
2461
+ if (!monthlyByMonth.has(monthNum)) {
2462
+ monthlyByMonth.set(monthNum, backup);
2463
+ toKeep.add(backup.id);
2464
+ }
2465
+ }
2466
+ const yearlyByYear = /* @__PURE__ */ new Map();
2467
+ for (const backup of categorized.yearly) {
2468
+ const yearNum = Math.floor((now - backup.timestamp) / msPerYear);
2469
+ if (!yearlyByYear.has(yearNum)) {
2470
+ yearlyByYear.set(yearNum, backup);
2471
+ toKeep.add(backup.id);
2472
+ }
2473
+ }
2474
+ const backupsToDelete = allBackups.filter((b) => !toKeep.has(b.id));
2475
+ if (backupsToDelete.length === 0) {
2476
+ return;
2477
+ }
2478
+ if (this.config.verbose) {
2479
+ console.log(`[BackupPlugin] Cleaning up ${backupsToDelete.length} old backups (keeping ${toKeep.size})`);
2480
+ }
2481
+ for (const backup of backupsToDelete) {
2482
+ try {
2483
+ await this.driver.delete(backup.id, backup.driverInfo);
2484
+ await this.database.resource(this.config.backupMetadataResource).delete(backup.id);
2485
+ if (this.config.verbose) {
2486
+ console.log(`[BackupPlugin] Deleted old backup: ${backup.id}`);
2487
+ }
2488
+ } catch (deleteError) {
2489
+ if (this.config.verbose) {
2490
+ console.warn(`[BackupPlugin] Failed to delete backup ${backup.id}: ${deleteError.message}`);
2491
+ }
2492
+ }
2493
+ }
2494
+ } catch (error) {
2495
+ if (this.config.verbose) {
2496
+ console.warn(`[BackupPlugin] Error during cleanup: ${error.message}`);
2497
+ }
2498
+ }
2286
2499
  }
2287
2500
  async _executeHook(hook, ...args) {
2288
2501
  if (typeof hook === "function") {
@@ -3572,81 +3785,58 @@ class PartitionAwareFilesystemCache extends FilesystemCache {
3572
3785
  class CachePlugin extends Plugin {
3573
3786
  constructor(options = {}) {
3574
3787
  super(options);
3575
- this.driverName = options.driver || "s3";
3576
- this.ttl = options.ttl;
3577
- this.maxSize = options.maxSize;
3578
- this.config = options.config || {};
3579
- this.includePartitions = options.includePartitions !== false;
3580
- this.partitionStrategy = options.partitionStrategy || "hierarchical";
3581
- this.partitionAware = options.partitionAware !== false;
3582
- this.trackUsage = options.trackUsage !== false;
3583
- this.preloadRelated = options.preloadRelated !== false;
3584
- this.legacyConfig = {
3585
- memoryOptions: options.memoryOptions,
3586
- filesystemOptions: options.filesystemOptions,
3587
- s3Options: options.s3Options,
3588
- driver: options.driver
3788
+ this.config = {
3789
+ // Driver configuration
3790
+ driver: options.driver || "s3",
3791
+ config: {
3792
+ ttl: options.ttl,
3793
+ maxSize: options.maxSize,
3794
+ ...options.config
3795
+ // Driver-specific config (can override ttl/maxSize)
3796
+ },
3797
+ // Resource filtering
3798
+ include: options.include || null,
3799
+ // Array of resource names to cache (null = all)
3800
+ exclude: options.exclude || [],
3801
+ // Array of resource names to exclude
3802
+ // Partition settings
3803
+ includePartitions: options.includePartitions !== false,
3804
+ partitionStrategy: options.partitionStrategy || "hierarchical",
3805
+ partitionAware: options.partitionAware !== false,
3806
+ trackUsage: options.trackUsage !== false,
3807
+ preloadRelated: options.preloadRelated !== false,
3808
+ // Retry configuration
3809
+ retryAttempts: options.retryAttempts || 3,
3810
+ retryDelay: options.retryDelay || 100,
3811
+ // ms
3812
+ // Logging
3813
+ verbose: options.verbose || false
3589
3814
  };
3590
3815
  }
3591
3816
  async setup(database) {
3592
3817
  await super.setup(database);
3593
3818
  }
3594
3819
  async onSetup() {
3595
- if (this.driverName && typeof this.driverName === "object") {
3596
- this.driver = this.driverName;
3597
- } else if (this.driverName === "memory") {
3598
- const driverConfig = {
3599
- ...this.legacyConfig.memoryOptions,
3600
- // Legacy support (lowest priority)
3601
- ...this.config
3602
- // New config format (medium priority)
3603
- };
3604
- if (this.ttl !== void 0) {
3605
- driverConfig.ttl = this.ttl;
3606
- }
3607
- if (this.maxSize !== void 0) {
3608
- driverConfig.maxSize = this.maxSize;
3609
- }
3610
- this.driver = new MemoryCache(driverConfig);
3611
- } else if (this.driverName === "filesystem") {
3612
- const driverConfig = {
3613
- ...this.legacyConfig.filesystemOptions,
3614
- // Legacy support (lowest priority)
3615
- ...this.config
3616
- // New config format (medium priority)
3617
- };
3618
- if (this.ttl !== void 0) {
3619
- driverConfig.ttl = this.ttl;
3620
- }
3621
- if (this.maxSize !== void 0) {
3622
- driverConfig.maxSize = this.maxSize;
3623
- }
3624
- if (this.partitionAware) {
3820
+ if (this.config.driver && typeof this.config.driver === "object") {
3821
+ this.driver = this.config.driver;
3822
+ } else if (this.config.driver === "memory") {
3823
+ this.driver = new MemoryCache(this.config.config);
3824
+ } else if (this.config.driver === "filesystem") {
3825
+ if (this.config.partitionAware) {
3625
3826
  this.driver = new PartitionAwareFilesystemCache({
3626
- partitionStrategy: this.partitionStrategy,
3627
- trackUsage: this.trackUsage,
3628
- preloadRelated: this.preloadRelated,
3629
- ...driverConfig
3827
+ partitionStrategy: this.config.partitionStrategy,
3828
+ trackUsage: this.config.trackUsage,
3829
+ preloadRelated: this.config.preloadRelated,
3830
+ ...this.config.config
3630
3831
  });
3631
3832
  } else {
3632
- this.driver = new FilesystemCache(driverConfig);
3833
+ this.driver = new FilesystemCache(this.config.config);
3633
3834
  }
3634
3835
  } else {
3635
- const driverConfig = {
3836
+ this.driver = new S3Cache({
3636
3837
  client: this.database.client,
3637
- // Required for S3Cache
3638
- ...this.legacyConfig.s3Options,
3639
- // Legacy support (lowest priority)
3640
- ...this.config
3641
- // New config format (medium priority)
3642
- };
3643
- if (this.ttl !== void 0) {
3644
- driverConfig.ttl = this.ttl;
3645
- }
3646
- if (this.maxSize !== void 0) {
3647
- driverConfig.maxSize = this.maxSize;
3648
- }
3649
- this.driver = new S3Cache(driverConfig);
3838
+ ...this.config.config
3839
+ });
3650
3840
  }
3651
3841
  this.installDatabaseHooks();
3652
3842
  this.installResourceHooks();
@@ -3656,7 +3846,9 @@ class CachePlugin extends Plugin {
3656
3846
  */
3657
3847
  installDatabaseHooks() {
3658
3848
  this.database.addHook("afterCreateResource", async ({ resource }) => {
3659
- this.installResourceHooksForResource(resource);
3849
+ if (this.shouldCacheResource(resource.name)) {
3850
+ this.installResourceHooksForResource(resource);
3851
+ }
3660
3852
  });
3661
3853
  }
3662
3854
  async onStart() {
@@ -3666,9 +3858,24 @@ class CachePlugin extends Plugin {
3666
3858
  // Remove the old installDatabaseProxy method
3667
3859
  installResourceHooks() {
3668
3860
  for (const resource of Object.values(this.database.resources)) {
3861
+ if (!this.shouldCacheResource(resource.name)) {
3862
+ continue;
3863
+ }
3669
3864
  this.installResourceHooksForResource(resource);
3670
3865
  }
3671
3866
  }
3867
+ shouldCacheResource(resourceName) {
3868
+ if (resourceName.startsWith("plg_") && !this.config.include) {
3869
+ return false;
3870
+ }
3871
+ if (this.config.exclude.includes(resourceName)) {
3872
+ return false;
3873
+ }
3874
+ if (this.config.include && !this.config.include.includes(resourceName)) {
3875
+ return false;
3876
+ }
3877
+ return true;
3878
+ }
3672
3879
  installResourceHooksForResource(resource) {
3673
3880
  if (!this.driver) return;
3674
3881
  Object.defineProperty(resource, "cache", {
@@ -3817,37 +4024,74 @@ class CachePlugin extends Plugin {
3817
4024
  if (data && data.id) {
3818
4025
  const itemSpecificMethods = ["get", "exists", "content", "hasContent"];
3819
4026
  for (const method of itemSpecificMethods) {
3820
- try {
3821
- const specificKey = await this.generateCacheKey(resource, method, { id: data.id });
3822
- await resource.cache.clear(specificKey.replace(".json.gz", ""));
3823
- } catch (error) {
4027
+ const specificKey = await this.generateCacheKey(resource, method, { id: data.id });
4028
+ const [ok2, err2] = await this.clearCacheWithRetry(resource.cache, specificKey);
4029
+ if (!ok2) {
4030
+ this.emit("cache_clear_error", {
4031
+ resource: resource.name,
4032
+ method,
4033
+ id: data.id,
4034
+ error: err2.message
4035
+ });
4036
+ if (this.config.verbose) {
4037
+ console.warn(`[CachePlugin] Failed to clear ${method} cache for ${resource.name}:${data.id}:`, err2.message);
4038
+ }
3824
4039
  }
3825
4040
  }
3826
4041
  if (this.config.includePartitions === true && resource.config?.partitions && Object.keys(resource.config.partitions).length > 0) {
3827
4042
  const partitionValues = this.getPartitionValues(data, resource);
3828
4043
  for (const [partitionName, values] of Object.entries(partitionValues)) {
3829
4044
  if (values && Object.keys(values).length > 0 && Object.values(values).some((v) => v !== null && v !== void 0)) {
3830
- try {
3831
- const partitionKeyPrefix = join(keyPrefix, `partition=${partitionName}`);
3832
- await resource.cache.clear(partitionKeyPrefix);
3833
- } catch (error) {
4045
+ const partitionKeyPrefix = join(keyPrefix, `partition=${partitionName}`);
4046
+ const [ok2, err2] = await this.clearCacheWithRetry(resource.cache, partitionKeyPrefix);
4047
+ if (!ok2) {
4048
+ this.emit("cache_clear_error", {
4049
+ resource: resource.name,
4050
+ partition: partitionName,
4051
+ error: err2.message
4052
+ });
4053
+ if (this.config.verbose) {
4054
+ console.warn(`[CachePlugin] Failed to clear partition cache for ${resource.name}/${partitionName}:`, err2.message);
4055
+ }
3834
4056
  }
3835
4057
  }
3836
4058
  }
3837
4059
  }
3838
4060
  }
3839
- try {
3840
- await resource.cache.clear(keyPrefix);
3841
- } catch (error) {
4061
+ const [ok, err] = await this.clearCacheWithRetry(resource.cache, keyPrefix);
4062
+ if (!ok) {
4063
+ this.emit("cache_clear_error", {
4064
+ resource: resource.name,
4065
+ type: "broad",
4066
+ error: err.message
4067
+ });
4068
+ if (this.config.verbose) {
4069
+ console.warn(`[CachePlugin] Failed to clear broad cache for ${resource.name}, trying specific methods:`, err.message);
4070
+ }
3842
4071
  const aggregateMethods = ["count", "list", "listIds", "getAll", "page", "query"];
3843
4072
  for (const method of aggregateMethods) {
3844
- try {
3845
- await resource.cache.clear(`${keyPrefix}/action=${method}`);
3846
- await resource.cache.clear(`resource=${resource.name}/action=${method}`);
3847
- } catch (methodError) {
3848
- }
4073
+ await this.clearCacheWithRetry(resource.cache, `${keyPrefix}/action=${method}`);
4074
+ await this.clearCacheWithRetry(resource.cache, `resource=${resource.name}/action=${method}`);
4075
+ }
4076
+ }
4077
+ }
4078
+ async clearCacheWithRetry(cache, key) {
4079
+ let lastError;
4080
+ for (let attempt = 0; attempt < this.config.retryAttempts; attempt++) {
4081
+ const [ok, err] = await tryFn(() => cache.clear(key));
4082
+ if (ok) {
4083
+ return [true, null];
4084
+ }
4085
+ lastError = err;
4086
+ if (err.name === "NoSuchKey" || err.code === "NoSuchKey") {
4087
+ return [true, null];
4088
+ }
4089
+ if (attempt < this.config.retryAttempts - 1) {
4090
+ const delay = this.config.retryDelay * Math.pow(2, attempt);
4091
+ await new Promise((resolve) => setTimeout(resolve, delay));
3849
4092
  }
3850
4093
  }
4094
+ return [false, lastError];
3851
4095
  }
3852
4096
  async generateCacheKey(resource, action, params = {}, partition = null, partitionValues = null) {
3853
4097
  const keyParts = [
@@ -3863,14 +4107,14 @@ class CachePlugin extends Plugin {
3863
4107
  }
3864
4108
  }
3865
4109
  if (Object.keys(params).length > 0) {
3866
- const paramsHash = await this.hashParams(params);
4110
+ const paramsHash = this.hashParams(params);
3867
4111
  keyParts.push(paramsHash);
3868
4112
  }
3869
4113
  return join(...keyParts) + ".json.gz";
3870
4114
  }
3871
- async hashParams(params) {
3872
- const sortedParams = Object.keys(params).sort().map((key) => `${key}:${JSON.stringify(params[key])}`).join("|") || "empty";
3873
- return await sha256(sortedParams);
4115
+ hashParams(params) {
4116
+ const serialized = jsonStableStringify(params) || "empty";
4117
+ return crypto.createHash("md5").update(serialized).digest("hex").substring(0, 16);
3874
4118
  }
3875
4119
  // Utility methods
3876
4120
  async getCacheStats() {
@@ -3895,50 +4139,48 @@ class CachePlugin extends Plugin {
3895
4139
  if (!resource) {
3896
4140
  throw new Error(`Resource '${resourceName}' not found`);
3897
4141
  }
3898
- const { includePartitions = true } = options;
4142
+ const { includePartitions = true, sampleSize = 100 } = options;
3899
4143
  if (this.driver instanceof PartitionAwareFilesystemCache && resource.warmPartitionCache) {
3900
4144
  const partitionNames = resource.config.partitions ? Object.keys(resource.config.partitions) : [];
3901
4145
  return await resource.warmPartitionCache(partitionNames, options);
3902
4146
  }
3903
- await resource.getAll();
3904
- if (includePartitions && resource.config.partitions) {
4147
+ let offset = 0;
4148
+ const pageSize = 100;
4149
+ const sampledRecords = [];
4150
+ while (sampledRecords.length < sampleSize) {
4151
+ const [ok, err, pageResult] = await tryFn(() => resource.page({ offset, size: pageSize }));
4152
+ if (!ok || !pageResult) {
4153
+ break;
4154
+ }
4155
+ const pageItems = Array.isArray(pageResult) ? pageResult : pageResult.items || [];
4156
+ if (pageItems.length === 0) {
4157
+ break;
4158
+ }
4159
+ sampledRecords.push(...pageItems);
4160
+ offset += pageSize;
4161
+ }
4162
+ if (includePartitions && resource.config.partitions && sampledRecords.length > 0) {
3905
4163
  for (const [partitionName, partitionDef] of Object.entries(resource.config.partitions)) {
3906
4164
  if (partitionDef.fields) {
3907
- const allRecords = await resource.getAll();
3908
- const recordsArray = Array.isArray(allRecords) ? allRecords : [];
3909
- const partitionValues = /* @__PURE__ */ new Set();
3910
- for (const record of recordsArray.slice(0, 10)) {
4165
+ const partitionValuesSet = /* @__PURE__ */ new Set();
4166
+ for (const record of sampledRecords) {
3911
4167
  const values = this.getPartitionValues(record, resource);
3912
4168
  if (values[partitionName]) {
3913
- partitionValues.add(JSON.stringify(values[partitionName]));
4169
+ partitionValuesSet.add(JSON.stringify(values[partitionName]));
3914
4170
  }
3915
4171
  }
3916
- for (const partitionValueStr of partitionValues) {
3917
- const partitionValues2 = JSON.parse(partitionValueStr);
3918
- await resource.list({ partition: partitionName, partitionValues: partitionValues2 });
4172
+ for (const partitionValueStr of partitionValuesSet) {
4173
+ const partitionValues = JSON.parse(partitionValueStr);
4174
+ await tryFn(() => resource.list({ partition: partitionName, partitionValues }));
3919
4175
  }
3920
4176
  }
3921
4177
  }
3922
4178
  }
3923
- }
3924
- // Partition-specific methods
3925
- async getPartitionCacheStats(resourceName, partition = null) {
3926
- if (!(this.driver instanceof PartitionAwareFilesystemCache)) {
3927
- throw new Error("Partition cache statistics are only available with PartitionAwareFilesystemCache");
3928
- }
3929
- return await this.driver.getPartitionStats(resourceName, partition);
3930
- }
3931
- async getCacheRecommendations(resourceName) {
3932
- if (!(this.driver instanceof PartitionAwareFilesystemCache)) {
3933
- throw new Error("Cache recommendations are only available with PartitionAwareFilesystemCache");
3934
- }
3935
- return await this.driver.getCacheRecommendations(resourceName);
3936
- }
3937
- async clearPartitionCache(resourceName, partition, partitionValues = {}) {
3938
- if (!(this.driver instanceof PartitionAwareFilesystemCache)) {
3939
- throw new Error("Partition cache clearing is only available with PartitionAwareFilesystemCache");
3940
- }
3941
- return await this.driver.clearPartition(resourceName, partition, partitionValues);
4179
+ return {
4180
+ resourceName,
4181
+ recordsSampled: sampledRecords.length,
4182
+ partitionsWarmed: includePartitions && resource.config.partitions ? Object.keys(resource.config.partitions).length : 0
4183
+ };
3942
4184
  }
3943
4185
  async analyzeCacheUsage() {
3944
4186
  if (!(this.driver instanceof PartitionAwareFilesystemCache)) {
@@ -3955,6 +4197,9 @@ class CachePlugin extends Plugin {
3955
4197
  }
3956
4198
  };
3957
4199
  for (const [resourceName, resource] of Object.entries(this.database.resources)) {
4200
+ if (!this.shouldCacheResource(resourceName)) {
4201
+ continue;
4202
+ }
3958
4203
  try {
3959
4204
  analysis.resourceStats[resourceName] = await this.driver.getPartitionStats(resourceName);
3960
4205
  analysis.recommendations[resourceName] = await this.driver.getCacheRecommendations(resourceName);
@@ -4046,62 +4291,807 @@ const CostsPlugin = {
4046
4291
  }
4047
4292
  };
4048
4293
 
4049
- class FullTextPlugin extends Plugin {
4294
+ class EventualConsistencyPlugin extends Plugin {
4050
4295
  constructor(options = {}) {
4051
- super();
4052
- this.indexResource = null;
4296
+ super(options);
4297
+ if (!options.resource) {
4298
+ throw new Error("EventualConsistencyPlugin requires 'resource' option");
4299
+ }
4300
+ if (!options.field) {
4301
+ throw new Error("EventualConsistencyPlugin requires 'field' option");
4302
+ }
4303
+ const detectedTimezone = this._detectTimezone();
4053
4304
  this.config = {
4054
- minWordLength: options.minWordLength || 3,
4055
- maxResults: options.maxResults || 100,
4056
- ...options
4305
+ resource: options.resource,
4306
+ field: options.field,
4307
+ cohort: {
4308
+ timezone: options.cohort?.timezone || detectedTimezone
4309
+ },
4310
+ reducer: options.reducer || ((transactions) => {
4311
+ let baseValue = 0;
4312
+ for (const t of transactions) {
4313
+ if (t.operation === "set") {
4314
+ baseValue = t.value;
4315
+ } else if (t.operation === "add") {
4316
+ baseValue += t.value;
4317
+ } else if (t.operation === "sub") {
4318
+ baseValue -= t.value;
4319
+ }
4320
+ }
4321
+ return baseValue;
4322
+ }),
4323
+ consolidationInterval: options.consolidationInterval ?? 300,
4324
+ // 5 minutes (in seconds)
4325
+ consolidationConcurrency: options.consolidationConcurrency || 5,
4326
+ consolidationWindow: options.consolidationWindow || 24,
4327
+ // Hours to look back for pending transactions (watermark)
4328
+ autoConsolidate: options.autoConsolidate !== false,
4329
+ lateArrivalStrategy: options.lateArrivalStrategy || "warn",
4330
+ // 'ignore', 'warn', 'process'
4331
+ batchTransactions: options.batchTransactions || false,
4332
+ // CAUTION: Not safe in distributed environments! Loses data on container crash
4333
+ batchSize: options.batchSize || 100,
4334
+ mode: options.mode || "async",
4335
+ // 'async' or 'sync'
4336
+ lockTimeout: options.lockTimeout || 300,
4337
+ // 5 minutes (in seconds, configurable)
4338
+ transactionRetention: options.transactionRetention || 30,
4339
+ // Days to keep applied transactions
4340
+ gcInterval: options.gcInterval || 86400,
4341
+ // 24 hours (in seconds)
4342
+ verbose: options.verbose || false
4057
4343
  };
4058
- this.indexes = /* @__PURE__ */ new Map();
4059
- }
4060
- async setup(database) {
4061
- this.database = database;
4062
- const [ok, err, indexResource] = await tryFn(() => database.createResource({
4063
- name: "fulltext_indexes",
4064
- attributes: {
4065
- id: "string|required",
4066
- resourceName: "string|required",
4067
- fieldName: "string|required",
4068
- word: "string|required",
4069
- recordIds: "json|required",
4070
- // Array of record IDs containing this word
4071
- count: "number|required",
4072
- lastUpdated: "string|required"
4073
- }
4074
- }));
4075
- this.indexResource = ok ? indexResource : database.resources.fulltext_indexes;
4076
- await this.loadIndexes();
4077
- this.installDatabaseHooks();
4078
- this.installIndexingHooks();
4079
- }
4080
- async start() {
4344
+ this.transactionResource = null;
4345
+ this.targetResource = null;
4346
+ this.consolidationTimer = null;
4347
+ this.gcTimer = null;
4348
+ this.pendingTransactions = /* @__PURE__ */ new Map();
4349
+ if (this.config.batchTransactions && !this.config.verbose) {
4350
+ console.warn(
4351
+ `[EventualConsistency] WARNING: batchTransactions is enabled. This stores transactions in memory and will lose data if container crashes. Not recommended for distributed/production environments. Set verbose: true to suppress this warning.`
4352
+ );
4353
+ }
4354
+ if (this.config.verbose && !options.cohort?.timezone) {
4355
+ console.log(
4356
+ `[EventualConsistency] Auto-detected timezone: ${this.config.cohort.timezone} (from ${process.env.TZ ? "TZ env var" : "system Intl API"})`
4357
+ );
4358
+ }
4081
4359
  }
4082
- async stop() {
4083
- await this.saveIndexes();
4084
- this.removeDatabaseHooks();
4360
+ async onSetup() {
4361
+ this.targetResource = this.database.resources[this.config.resource];
4362
+ if (!this.targetResource) {
4363
+ this.deferredSetup = true;
4364
+ this.watchForResource();
4365
+ return;
4366
+ }
4367
+ await this.completeSetup();
4085
4368
  }
4086
- async loadIndexes() {
4087
- if (!this.indexResource) return;
4088
- const [ok, err, allIndexes] = await tryFn(() => this.indexResource.getAll());
4089
- if (ok) {
4090
- for (const indexRecord of allIndexes) {
4091
- const key = `${indexRecord.resourceName}:${indexRecord.fieldName}:${indexRecord.word}`;
4092
- this.indexes.set(key, {
4093
- recordIds: indexRecord.recordIds || [],
4094
- count: indexRecord.count || 0
4095
- });
4369
+ watchForResource() {
4370
+ const hookCallback = async ({ resource, config }) => {
4371
+ if (config.name === this.config.resource && this.deferredSetup) {
4372
+ this.targetResource = resource;
4373
+ this.deferredSetup = false;
4374
+ await this.completeSetup();
4096
4375
  }
4376
+ };
4377
+ this.database.addHook("afterCreateResource", hookCallback);
4378
+ }
4379
+ async completeSetup() {
4380
+ if (!this.targetResource) return;
4381
+ const transactionResourceName = `${this.config.resource}_transactions_${this.config.field}`;
4382
+ const partitionConfig = this.createPartitionConfig();
4383
+ const [ok, err, transactionResource] = await tryFn(
4384
+ () => this.database.createResource({
4385
+ name: transactionResourceName,
4386
+ attributes: {
4387
+ id: "string|required",
4388
+ originalId: "string|required",
4389
+ field: "string|required",
4390
+ value: "number|required",
4391
+ operation: "string|required",
4392
+ // 'set', 'add', or 'sub'
4393
+ timestamp: "string|required",
4394
+ cohortDate: "string|required",
4395
+ // For daily partitioning
4396
+ cohortHour: "string|required",
4397
+ // For hourly partitioning
4398
+ cohortMonth: "string|optional",
4399
+ // For monthly partitioning
4400
+ source: "string|optional",
4401
+ applied: "boolean|optional"
4402
+ // Track if transaction was applied
4403
+ },
4404
+ behavior: "body-overflow",
4405
+ timestamps: true,
4406
+ partitions: partitionConfig,
4407
+ asyncPartitions: true
4408
+ // Use async partitions for better performance
4409
+ })
4410
+ );
4411
+ if (!ok && !this.database.resources[transactionResourceName]) {
4412
+ throw new Error(`Failed to create transaction resource: ${err?.message}`);
4413
+ }
4414
+ this.transactionResource = ok ? transactionResource : this.database.resources[transactionResourceName];
4415
+ const lockResourceName = `${this.config.resource}_consolidation_locks_${this.config.field}`;
4416
+ const [lockOk, lockErr, lockResource] = await tryFn(
4417
+ () => this.database.createResource({
4418
+ name: lockResourceName,
4419
+ attributes: {
4420
+ id: "string|required",
4421
+ lockedAt: "number|required",
4422
+ workerId: "string|optional"
4423
+ },
4424
+ behavior: "body-only",
4425
+ timestamps: false
4426
+ })
4427
+ );
4428
+ if (!lockOk && !this.database.resources[lockResourceName]) {
4429
+ throw new Error(`Failed to create lock resource: ${lockErr?.message}`);
4430
+ }
4431
+ this.lockResource = lockOk ? lockResource : this.database.resources[lockResourceName];
4432
+ this.addHelperMethods();
4433
+ if (this.config.autoConsolidate) {
4434
+ this.startConsolidationTimer();
4097
4435
  }
4436
+ this.startGarbageCollectionTimer();
4098
4437
  }
4099
- async saveIndexes() {
4100
- if (!this.indexResource) return;
4101
- const [ok, err] = await tryFn(async () => {
4102
- const existingIndexes = await this.indexResource.getAll();
4103
- for (const index of existingIndexes) {
4104
- await this.indexResource.delete(index.id);
4438
+ async onStart() {
4439
+ if (this.deferredSetup) {
4440
+ return;
4441
+ }
4442
+ this.emit("eventual-consistency.started", {
4443
+ resource: this.config.resource,
4444
+ field: this.config.field,
4445
+ cohort: this.config.cohort
4446
+ });
4447
+ }
4448
+ async onStop() {
4449
+ if (this.consolidationTimer) {
4450
+ clearInterval(this.consolidationTimer);
4451
+ this.consolidationTimer = null;
4452
+ }
4453
+ if (this.gcTimer) {
4454
+ clearInterval(this.gcTimer);
4455
+ this.gcTimer = null;
4456
+ }
4457
+ await this.flushPendingTransactions();
4458
+ this.emit("eventual-consistency.stopped", {
4459
+ resource: this.config.resource,
4460
+ field: this.config.field
4461
+ });
4462
+ }
4463
+ createPartitionConfig() {
4464
+ const partitions = {
4465
+ byHour: {
4466
+ fields: {
4467
+ cohortHour: "string"
4468
+ }
4469
+ },
4470
+ byDay: {
4471
+ fields: {
4472
+ cohortDate: "string"
4473
+ }
4474
+ },
4475
+ byMonth: {
4476
+ fields: {
4477
+ cohortMonth: "string"
4478
+ }
4479
+ }
4480
+ };
4481
+ return partitions;
4482
+ }
4483
+ /**
4484
+ * Auto-detect timezone from environment or system
4485
+ * @private
4486
+ */
4487
+ _detectTimezone() {
4488
+ if (process.env.TZ) {
4489
+ return process.env.TZ;
4490
+ }
4491
+ try {
4492
+ const systemTimezone = Intl.DateTimeFormat().resolvedOptions().timeZone;
4493
+ if (systemTimezone) {
4494
+ return systemTimezone;
4495
+ }
4496
+ } catch (err) {
4497
+ }
4498
+ return "UTC";
4499
+ }
4500
+ /**
4501
+ * Helper method to resolve field and plugin from arguments
4502
+ * Supports both single-field (field, value) and multi-field (field, value) signatures
4503
+ * @private
4504
+ */
4505
+ _resolveFieldAndPlugin(resource, fieldOrValue, value) {
4506
+ const hasMultipleFields = Object.keys(resource._eventualConsistencyPlugins).length > 1;
4507
+ if (hasMultipleFields && value === void 0) {
4508
+ throw new Error(`Multiple fields have eventual consistency. Please specify the field explicitly.`);
4509
+ }
4510
+ const field = value !== void 0 ? fieldOrValue : this.config.field;
4511
+ const actualValue = value !== void 0 ? value : fieldOrValue;
4512
+ const fieldPlugin = resource._eventualConsistencyPlugins[field];
4513
+ if (!fieldPlugin) {
4514
+ throw new Error(`No eventual consistency plugin found for field "${field}"`);
4515
+ }
4516
+ return { field, value: actualValue, plugin: fieldPlugin };
4517
+ }
4518
+ /**
4519
+ * Helper method to perform atomic consolidation in sync mode
4520
+ * @private
4521
+ */
4522
+ async _syncModeConsolidate(id, field) {
4523
+ const consolidatedValue = await this.consolidateRecord(id);
4524
+ await this.targetResource.update(id, {
4525
+ [field]: consolidatedValue
4526
+ });
4527
+ return consolidatedValue;
4528
+ }
4529
+ /**
4530
+ * Create synthetic 'set' transaction from current value
4531
+ * @private
4532
+ */
4533
+ _createSyntheticSetTransaction(currentValue) {
4534
+ return {
4535
+ id: "__synthetic__",
4536
+ operation: "set",
4537
+ value: currentValue,
4538
+ timestamp: (/* @__PURE__ */ new Date(0)).toISOString(),
4539
+ synthetic: true
4540
+ };
4541
+ }
4542
+ addHelperMethods() {
4543
+ const resource = this.targetResource;
4544
+ const defaultField = this.config.field;
4545
+ const plugin = this;
4546
+ if (!resource._eventualConsistencyPlugins) {
4547
+ resource._eventualConsistencyPlugins = {};
4548
+ }
4549
+ resource._eventualConsistencyPlugins[defaultField] = plugin;
4550
+ resource.set = async (id, fieldOrValue, value) => {
4551
+ const { field, value: actualValue, plugin: fieldPlugin } = plugin._resolveFieldAndPlugin(resource, fieldOrValue, value);
4552
+ await fieldPlugin.createTransaction({
4553
+ originalId: id,
4554
+ operation: "set",
4555
+ value: actualValue,
4556
+ source: "set"
4557
+ });
4558
+ if (fieldPlugin.config.mode === "sync") {
4559
+ return await fieldPlugin._syncModeConsolidate(id, field);
4560
+ }
4561
+ return actualValue;
4562
+ };
4563
+ resource.add = async (id, fieldOrAmount, amount) => {
4564
+ const { field, value: actualAmount, plugin: fieldPlugin } = plugin._resolveFieldAndPlugin(resource, fieldOrAmount, amount);
4565
+ await fieldPlugin.createTransaction({
4566
+ originalId: id,
4567
+ operation: "add",
4568
+ value: actualAmount,
4569
+ source: "add"
4570
+ });
4571
+ if (fieldPlugin.config.mode === "sync") {
4572
+ return await fieldPlugin._syncModeConsolidate(id, field);
4573
+ }
4574
+ const currentValue = await fieldPlugin.getConsolidatedValue(id);
4575
+ return currentValue + actualAmount;
4576
+ };
4577
+ resource.sub = async (id, fieldOrAmount, amount) => {
4578
+ const { field, value: actualAmount, plugin: fieldPlugin } = plugin._resolveFieldAndPlugin(resource, fieldOrAmount, amount);
4579
+ await fieldPlugin.createTransaction({
4580
+ originalId: id,
4581
+ operation: "sub",
4582
+ value: actualAmount,
4583
+ source: "sub"
4584
+ });
4585
+ if (fieldPlugin.config.mode === "sync") {
4586
+ return await fieldPlugin._syncModeConsolidate(id, field);
4587
+ }
4588
+ const currentValue = await fieldPlugin.getConsolidatedValue(id);
4589
+ return currentValue - actualAmount;
4590
+ };
4591
+ resource.consolidate = async (id, field) => {
4592
+ const hasMultipleFields = Object.keys(resource._eventualConsistencyPlugins).length > 1;
4593
+ if (hasMultipleFields && !field) {
4594
+ throw new Error(`Multiple fields have eventual consistency. Please specify the field: consolidate(id, field)`);
4595
+ }
4596
+ const actualField = field || defaultField;
4597
+ const fieldPlugin = resource._eventualConsistencyPlugins[actualField];
4598
+ if (!fieldPlugin) {
4599
+ throw new Error(`No eventual consistency plugin found for field "${actualField}"`);
4600
+ }
4601
+ return await fieldPlugin.consolidateRecord(id);
4602
+ };
4603
+ resource.getConsolidatedValue = async (id, fieldOrOptions, options) => {
4604
+ if (typeof fieldOrOptions === "string") {
4605
+ const field = fieldOrOptions;
4606
+ const fieldPlugin = resource._eventualConsistencyPlugins[field] || plugin;
4607
+ return await fieldPlugin.getConsolidatedValue(id, options || {});
4608
+ } else {
4609
+ return await plugin.getConsolidatedValue(id, fieldOrOptions || {});
4610
+ }
4611
+ };
4612
+ }
4613
+ async createTransaction(data) {
4614
+ const now = /* @__PURE__ */ new Date();
4615
+ const cohortInfo = this.getCohortInfo(now);
4616
+ const watermarkMs = this.config.consolidationWindow * 60 * 60 * 1e3;
4617
+ const watermarkTime = now.getTime() - watermarkMs;
4618
+ const cohortHourDate = /* @__PURE__ */ new Date(cohortInfo.hour + ":00:00Z");
4619
+ if (cohortHourDate.getTime() < watermarkTime) {
4620
+ const hoursLate = Math.floor((now.getTime() - cohortHourDate.getTime()) / (60 * 60 * 1e3));
4621
+ if (this.config.lateArrivalStrategy === "ignore") {
4622
+ if (this.config.verbose) {
4623
+ console.warn(
4624
+ `[EventualConsistency] Late arrival ignored: transaction for ${cohortInfo.hour} is ${hoursLate}h late (watermark: ${this.config.consolidationWindow}h)`
4625
+ );
4626
+ }
4627
+ return null;
4628
+ } else if (this.config.lateArrivalStrategy === "warn") {
4629
+ console.warn(
4630
+ `[EventualConsistency] Late arrival detected: transaction for ${cohortInfo.hour} is ${hoursLate}h late (watermark: ${this.config.consolidationWindow}h). Processing anyway, but consolidation may not pick it up.`
4631
+ );
4632
+ }
4633
+ }
4634
+ const transaction = {
4635
+ id: idGenerator(),
4636
+ // Use nanoid for guaranteed uniqueness
4637
+ originalId: data.originalId,
4638
+ field: this.config.field,
4639
+ value: data.value || 0,
4640
+ operation: data.operation || "set",
4641
+ timestamp: now.toISOString(),
4642
+ cohortDate: cohortInfo.date,
4643
+ cohortHour: cohortInfo.hour,
4644
+ cohortMonth: cohortInfo.month,
4645
+ source: data.source || "unknown",
4646
+ applied: false
4647
+ };
4648
+ if (this.config.batchTransactions) {
4649
+ this.pendingTransactions.set(transaction.id, transaction);
4650
+ if (this.pendingTransactions.size >= this.config.batchSize) {
4651
+ await this.flushPendingTransactions();
4652
+ }
4653
+ } else {
4654
+ await this.transactionResource.insert(transaction);
4655
+ }
4656
+ return transaction;
4657
+ }
4658
+ async flushPendingTransactions() {
4659
+ if (this.pendingTransactions.size === 0) return;
4660
+ const transactions = Array.from(this.pendingTransactions.values());
4661
+ try {
4662
+ await Promise.all(
4663
+ transactions.map(
4664
+ (transaction) => this.transactionResource.insert(transaction)
4665
+ )
4666
+ );
4667
+ this.pendingTransactions.clear();
4668
+ } catch (error) {
4669
+ console.error("Failed to flush pending transactions:", error);
4670
+ throw error;
4671
+ }
4672
+ }
4673
+ getCohortInfo(date) {
4674
+ const tz = this.config.cohort.timezone;
4675
+ const offset = this.getTimezoneOffset(tz);
4676
+ const localDate = new Date(date.getTime() + offset);
4677
+ const year = localDate.getFullYear();
4678
+ const month = String(localDate.getMonth() + 1).padStart(2, "0");
4679
+ const day = String(localDate.getDate()).padStart(2, "0");
4680
+ const hour = String(localDate.getHours()).padStart(2, "0");
4681
+ return {
4682
+ date: `${year}-${month}-${day}`,
4683
+ hour: `${year}-${month}-${day}T${hour}`,
4684
+ // ISO-like format for hour partition
4685
+ month: `${year}-${month}`
4686
+ };
4687
+ }
4688
+ getTimezoneOffset(timezone) {
4689
+ try {
4690
+ const now = /* @__PURE__ */ new Date();
4691
+ const utcDate = new Date(now.toLocaleString("en-US", { timeZone: "UTC" }));
4692
+ const tzDate = new Date(now.toLocaleString("en-US", { timeZone: timezone }));
4693
+ return tzDate.getTime() - utcDate.getTime();
4694
+ } catch (err) {
4695
+ const offsets = {
4696
+ "UTC": 0,
4697
+ "America/New_York": -5 * 36e5,
4698
+ "America/Chicago": -6 * 36e5,
4699
+ "America/Denver": -7 * 36e5,
4700
+ "America/Los_Angeles": -8 * 36e5,
4701
+ "America/Sao_Paulo": -3 * 36e5,
4702
+ "Europe/London": 0,
4703
+ "Europe/Paris": 1 * 36e5,
4704
+ "Europe/Berlin": 1 * 36e5,
4705
+ "Asia/Tokyo": 9 * 36e5,
4706
+ "Asia/Shanghai": 8 * 36e5,
4707
+ "Australia/Sydney": 10 * 36e5
4708
+ };
4709
+ if (this.config.verbose && !offsets[timezone]) {
4710
+ console.warn(
4711
+ `[EventualConsistency] Unknown timezone '${timezone}', using UTC. Consider using a valid IANA timezone (e.g., 'America/New_York')`
4712
+ );
4713
+ }
4714
+ return offsets[timezone] || 0;
4715
+ }
4716
+ }
4717
+ startConsolidationTimer() {
4718
+ const intervalMs = this.config.consolidationInterval * 1e3;
4719
+ this.consolidationTimer = setInterval(async () => {
4720
+ await this.runConsolidation();
4721
+ }, intervalMs);
4722
+ }
4723
+ async runConsolidation() {
4724
+ try {
4725
+ const now = /* @__PURE__ */ new Date();
4726
+ const hoursToCheck = this.config.consolidationWindow || 24;
4727
+ const cohortHours = [];
4728
+ for (let i = 0; i < hoursToCheck; i++) {
4729
+ const date = new Date(now.getTime() - i * 60 * 60 * 1e3);
4730
+ const cohortInfo = this.getCohortInfo(date);
4731
+ cohortHours.push(cohortInfo.hour);
4732
+ }
4733
+ const transactionsByHour = await Promise.all(
4734
+ cohortHours.map(async (cohortHour) => {
4735
+ const [ok, err, txns] = await tryFn(
4736
+ () => this.transactionResource.query({
4737
+ cohortHour,
4738
+ applied: false
4739
+ })
4740
+ );
4741
+ return ok ? txns : [];
4742
+ })
4743
+ );
4744
+ const transactions = transactionsByHour.flat();
4745
+ if (transactions.length === 0) {
4746
+ if (this.config.verbose) {
4747
+ console.log(`[EventualConsistency] No pending transactions to consolidate`);
4748
+ }
4749
+ return;
4750
+ }
4751
+ const uniqueIds = [...new Set(transactions.map((t) => t.originalId))];
4752
+ const { results, errors } = await PromisePool.for(uniqueIds).withConcurrency(this.config.consolidationConcurrency).process(async (id) => {
4753
+ return await this.consolidateRecord(id);
4754
+ });
4755
+ if (errors && errors.length > 0) {
4756
+ console.error(`Consolidation completed with ${errors.length} errors:`, errors);
4757
+ }
4758
+ this.emit("eventual-consistency.consolidated", {
4759
+ resource: this.config.resource,
4760
+ field: this.config.field,
4761
+ recordCount: uniqueIds.length,
4762
+ successCount: results.length,
4763
+ errorCount: errors.length
4764
+ });
4765
+ } catch (error) {
4766
+ console.error("Consolidation error:", error);
4767
+ this.emit("eventual-consistency.consolidation-error", error);
4768
+ }
4769
+ }
4770
+ async consolidateRecord(originalId) {
4771
+ await this.cleanupStaleLocks();
4772
+ const lockId = `lock-${originalId}`;
4773
+ const [lockAcquired, lockErr, lock] = await tryFn(
4774
+ () => this.lockResource.insert({
4775
+ id: lockId,
4776
+ lockedAt: Date.now(),
4777
+ workerId: process.pid ? String(process.pid) : "unknown"
4778
+ })
4779
+ );
4780
+ if (!lockAcquired) {
4781
+ if (this.config.verbose) {
4782
+ console.log(`[EventualConsistency] Lock for ${originalId} already held, skipping`);
4783
+ }
4784
+ const [recordOk, recordErr, record] = await tryFn(
4785
+ () => this.targetResource.get(originalId)
4786
+ );
4787
+ return recordOk && record ? record[this.config.field] || 0 : 0;
4788
+ }
4789
+ try {
4790
+ const [recordOk, recordErr, record] = await tryFn(
4791
+ () => this.targetResource.get(originalId)
4792
+ );
4793
+ const currentValue = recordOk && record ? record[this.config.field] || 0 : 0;
4794
+ const [ok, err, transactions] = await tryFn(
4795
+ () => this.transactionResource.query({
4796
+ originalId,
4797
+ applied: false
4798
+ })
4799
+ );
4800
+ if (!ok || !transactions || transactions.length === 0) {
4801
+ return currentValue;
4802
+ }
4803
+ transactions.sort(
4804
+ (a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()
4805
+ );
4806
+ const hasSetOperation = transactions.some((t) => t.operation === "set");
4807
+ if (currentValue !== 0 && !hasSetOperation) {
4808
+ transactions.unshift(this._createSyntheticSetTransaction(currentValue));
4809
+ }
4810
+ const consolidatedValue = this.config.reducer(transactions);
4811
+ const [updateOk, updateErr] = await tryFn(
4812
+ () => this.targetResource.update(originalId, {
4813
+ [this.config.field]: consolidatedValue
4814
+ })
4815
+ );
4816
+ if (updateOk) {
4817
+ const transactionsToUpdate = transactions.filter((txn) => txn.id !== "__synthetic__");
4818
+ const { results, errors } = await PromisePool.for(transactionsToUpdate).withConcurrency(10).process(async (txn) => {
4819
+ const [ok2, err2] = await tryFn(
4820
+ () => this.transactionResource.update(txn.id, { applied: true })
4821
+ );
4822
+ if (!ok2 && this.config.verbose) {
4823
+ console.warn(`[EventualConsistency] Failed to mark transaction ${txn.id} as applied:`, err2?.message);
4824
+ }
4825
+ return ok2;
4826
+ });
4827
+ if (errors && errors.length > 0 && this.config.verbose) {
4828
+ console.warn(`[EventualConsistency] ${errors.length} transactions failed to mark as applied`);
4829
+ }
4830
+ }
4831
+ return consolidatedValue;
4832
+ } finally {
4833
+ const [lockReleased, lockReleaseErr] = await tryFn(() => this.lockResource.delete(lockId));
4834
+ if (!lockReleased && this.config.verbose) {
4835
+ console.warn(`[EventualConsistency] Failed to release lock ${lockId}:`, lockReleaseErr?.message);
4836
+ }
4837
+ }
4838
+ }
4839
+ async getConsolidatedValue(originalId, options = {}) {
4840
+ const includeApplied = options.includeApplied || false;
4841
+ const startDate = options.startDate;
4842
+ const endDate = options.endDate;
4843
+ const query = { originalId };
4844
+ if (!includeApplied) {
4845
+ query.applied = false;
4846
+ }
4847
+ const [ok, err, transactions] = await tryFn(
4848
+ () => this.transactionResource.query(query)
4849
+ );
4850
+ if (!ok || !transactions || transactions.length === 0) {
4851
+ const [recordOk2, recordErr2, record2] = await tryFn(
4852
+ () => this.targetResource.get(originalId)
4853
+ );
4854
+ if (recordOk2 && record2) {
4855
+ return record2[this.config.field] || 0;
4856
+ }
4857
+ return 0;
4858
+ }
4859
+ let filtered = transactions;
4860
+ if (startDate || endDate) {
4861
+ filtered = transactions.filter((t) => {
4862
+ const timestamp = new Date(t.timestamp);
4863
+ if (startDate && timestamp < new Date(startDate)) return false;
4864
+ if (endDate && timestamp > new Date(endDate)) return false;
4865
+ return true;
4866
+ });
4867
+ }
4868
+ const [recordOk, recordErr, record] = await tryFn(
4869
+ () => this.targetResource.get(originalId)
4870
+ );
4871
+ const currentValue = recordOk && record ? record[this.config.field] || 0 : 0;
4872
+ const hasSetOperation = filtered.some((t) => t.operation === "set");
4873
+ if (currentValue !== 0 && !hasSetOperation) {
4874
+ filtered.unshift(this._createSyntheticSetTransaction(currentValue));
4875
+ }
4876
+ filtered.sort(
4877
+ (a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()
4878
+ );
4879
+ return this.config.reducer(filtered);
4880
+ }
4881
+ // Helper method to get cohort statistics
4882
+ async getCohortStats(cohortDate) {
4883
+ const [ok, err, transactions] = await tryFn(
4884
+ () => this.transactionResource.query({
4885
+ cohortDate
4886
+ })
4887
+ );
4888
+ if (!ok) return null;
4889
+ const stats = {
4890
+ date: cohortDate,
4891
+ transactionCount: transactions.length,
4892
+ totalValue: 0,
4893
+ byOperation: { set: 0, add: 0, sub: 0 },
4894
+ byOriginalId: {}
4895
+ };
4896
+ for (const txn of transactions) {
4897
+ stats.totalValue += txn.value || 0;
4898
+ stats.byOperation[txn.operation] = (stats.byOperation[txn.operation] || 0) + 1;
4899
+ if (!stats.byOriginalId[txn.originalId]) {
4900
+ stats.byOriginalId[txn.originalId] = {
4901
+ count: 0,
4902
+ value: 0
4903
+ };
4904
+ }
4905
+ stats.byOriginalId[txn.originalId].count++;
4906
+ stats.byOriginalId[txn.originalId].value += txn.value || 0;
4907
+ }
4908
+ return stats;
4909
+ }
4910
+ /**
4911
+ * Clean up stale locks that exceed the configured timeout
4912
+ * Uses distributed locking to prevent multiple containers from cleaning simultaneously
4913
+ */
4914
+ async cleanupStaleLocks() {
4915
+ const now = Date.now();
4916
+ const lockTimeoutMs = this.config.lockTimeout * 1e3;
4917
+ const cutoffTime = now - lockTimeoutMs;
4918
+ const cleanupLockId = `lock-cleanup-${this.config.resource}-${this.config.field}`;
4919
+ const [lockAcquired] = await tryFn(
4920
+ () => this.lockResource.insert({
4921
+ id: cleanupLockId,
4922
+ lockedAt: Date.now(),
4923
+ workerId: process.pid ? String(process.pid) : "unknown"
4924
+ })
4925
+ );
4926
+ if (!lockAcquired) {
4927
+ if (this.config.verbose) {
4928
+ console.log(`[EventualConsistency] Lock cleanup already running in another container`);
4929
+ }
4930
+ return;
4931
+ }
4932
+ try {
4933
+ const [ok, err, locks] = await tryFn(() => this.lockResource.list());
4934
+ if (!ok || !locks || locks.length === 0) return;
4935
+ const staleLocks = locks.filter(
4936
+ (lock) => lock.id !== cleanupLockId && lock.lockedAt < cutoffTime
4937
+ );
4938
+ if (staleLocks.length === 0) return;
4939
+ if (this.config.verbose) {
4940
+ console.log(`[EventualConsistency] Cleaning up ${staleLocks.length} stale locks`);
4941
+ }
4942
+ const { results, errors } = await PromisePool.for(staleLocks).withConcurrency(5).process(async (lock) => {
4943
+ const [deleted] = await tryFn(() => this.lockResource.delete(lock.id));
4944
+ return deleted;
4945
+ });
4946
+ if (errors && errors.length > 0 && this.config.verbose) {
4947
+ console.warn(`[EventualConsistency] ${errors.length} stale locks failed to delete`);
4948
+ }
4949
+ } catch (error) {
4950
+ if (this.config.verbose) {
4951
+ console.warn(`[EventualConsistency] Error cleaning up stale locks:`, error.message);
4952
+ }
4953
+ } finally {
4954
+ await tryFn(() => this.lockResource.delete(cleanupLockId));
4955
+ }
4956
+ }
4957
+ /**
4958
+ * Start garbage collection timer for old applied transactions
4959
+ */
4960
+ startGarbageCollectionTimer() {
4961
+ const gcIntervalMs = this.config.gcInterval * 1e3;
4962
+ this.gcTimer = setInterval(async () => {
4963
+ await this.runGarbageCollection();
4964
+ }, gcIntervalMs);
4965
+ }
4966
+ /**
4967
+ * Delete old applied transactions based on retention policy
4968
+ * Uses distributed locking to prevent multiple containers from running GC simultaneously
4969
+ */
4970
+ async runGarbageCollection() {
4971
+ const gcLockId = `lock-gc-${this.config.resource}-${this.config.field}`;
4972
+ const [lockAcquired] = await tryFn(
4973
+ () => this.lockResource.insert({
4974
+ id: gcLockId,
4975
+ lockedAt: Date.now(),
4976
+ workerId: process.pid ? String(process.pid) : "unknown"
4977
+ })
4978
+ );
4979
+ if (!lockAcquired) {
4980
+ if (this.config.verbose) {
4981
+ console.log(`[EventualConsistency] GC already running in another container`);
4982
+ }
4983
+ return;
4984
+ }
4985
+ try {
4986
+ const now = Date.now();
4987
+ const retentionMs = this.config.transactionRetention * 24 * 60 * 60 * 1e3;
4988
+ const cutoffDate = new Date(now - retentionMs);
4989
+ const cutoffIso = cutoffDate.toISOString();
4990
+ if (this.config.verbose) {
4991
+ console.log(`[EventualConsistency] Running GC for transactions older than ${cutoffIso} (${this.config.transactionRetention} days)`);
4992
+ }
4993
+ const cutoffMonth = cutoffDate.toISOString().substring(0, 7);
4994
+ const [ok, err, oldTransactions] = await tryFn(
4995
+ () => this.transactionResource.query({
4996
+ applied: true,
4997
+ timestamp: { "<": cutoffIso }
4998
+ })
4999
+ );
5000
+ if (!ok) {
5001
+ if (this.config.verbose) {
5002
+ console.warn(`[EventualConsistency] GC failed to query transactions:`, err?.message);
5003
+ }
5004
+ return;
5005
+ }
5006
+ if (!oldTransactions || oldTransactions.length === 0) {
5007
+ if (this.config.verbose) {
5008
+ console.log(`[EventualConsistency] No old transactions to clean up`);
5009
+ }
5010
+ return;
5011
+ }
5012
+ if (this.config.verbose) {
5013
+ console.log(`[EventualConsistency] Deleting ${oldTransactions.length} old transactions`);
5014
+ }
5015
+ const { results, errors } = await PromisePool.for(oldTransactions).withConcurrency(10).process(async (txn) => {
5016
+ const [deleted] = await tryFn(() => this.transactionResource.delete(txn.id));
5017
+ return deleted;
5018
+ });
5019
+ if (this.config.verbose) {
5020
+ console.log(`[EventualConsistency] GC completed: ${results.length} deleted, ${errors.length} errors`);
5021
+ }
5022
+ this.emit("eventual-consistency.gc-completed", {
5023
+ resource: this.config.resource,
5024
+ field: this.config.field,
5025
+ deletedCount: results.length,
5026
+ errorCount: errors.length
5027
+ });
5028
+ } catch (error) {
5029
+ if (this.config.verbose) {
5030
+ console.warn(`[EventualConsistency] GC error:`, error.message);
5031
+ }
5032
+ this.emit("eventual-consistency.gc-error", error);
5033
+ } finally {
5034
+ await tryFn(() => this.lockResource.delete(gcLockId));
5035
+ }
5036
+ }
5037
+ }
5038
+
5039
+ class FullTextPlugin extends Plugin {
5040
+ constructor(options = {}) {
5041
+ super();
5042
+ this.indexResource = null;
5043
+ this.config = {
5044
+ minWordLength: options.minWordLength || 3,
5045
+ maxResults: options.maxResults || 100,
5046
+ ...options
5047
+ };
5048
+ this.indexes = /* @__PURE__ */ new Map();
5049
+ }
5050
+ async setup(database) {
5051
+ this.database = database;
5052
+ const [ok, err, indexResource] = await tryFn(() => database.createResource({
5053
+ name: "plg_fulltext_indexes",
5054
+ attributes: {
5055
+ id: "string|required",
5056
+ resourceName: "string|required",
5057
+ fieldName: "string|required",
5058
+ word: "string|required",
5059
+ recordIds: "json|required",
5060
+ // Array of record IDs containing this word
5061
+ count: "number|required",
5062
+ lastUpdated: "string|required"
5063
+ }
5064
+ }));
5065
+ this.indexResource = ok ? indexResource : database.resources.fulltext_indexes;
5066
+ await this.loadIndexes();
5067
+ this.installDatabaseHooks();
5068
+ this.installIndexingHooks();
5069
+ }
5070
+ async start() {
5071
+ }
5072
+ async stop() {
5073
+ await this.saveIndexes();
5074
+ this.removeDatabaseHooks();
5075
+ }
5076
+ async loadIndexes() {
5077
+ if (!this.indexResource) return;
5078
+ const [ok, err, allIndexes] = await tryFn(() => this.indexResource.getAll());
5079
+ if (ok) {
5080
+ for (const indexRecord of allIndexes) {
5081
+ const key = `${indexRecord.resourceName}:${indexRecord.fieldName}:${indexRecord.word}`;
5082
+ this.indexes.set(key, {
5083
+ recordIds: indexRecord.recordIds || [],
5084
+ count: indexRecord.count || 0
5085
+ });
5086
+ }
5087
+ }
5088
+ }
5089
+ async saveIndexes() {
5090
+ if (!this.indexResource) return;
5091
+ const [ok, err] = await tryFn(async () => {
5092
+ const existingIndexes = await this.indexResource.getAll();
5093
+ for (const index of existingIndexes) {
5094
+ await this.indexResource.delete(index.id);
4105
5095
  }
4106
5096
  for (const [key, data] of this.indexes.entries()) {
4107
5097
  const [resourceName, fieldName, word] = key.split(":");
@@ -4119,7 +5109,7 @@ class FullTextPlugin extends Plugin {
4119
5109
  }
4120
5110
  installDatabaseHooks() {
4121
5111
  this.database.addHook("afterCreateResource", (resource) => {
4122
- if (resource.name !== "fulltext_indexes") {
5112
+ if (resource.name !== "plg_fulltext_indexes") {
4123
5113
  this.installResourceHooks(resource);
4124
5114
  }
4125
5115
  });
@@ -4133,14 +5123,14 @@ class FullTextPlugin extends Plugin {
4133
5123
  }
4134
5124
  this.database.plugins.fulltext = this;
4135
5125
  for (const resource of Object.values(this.database.resources)) {
4136
- if (resource.name === "fulltext_indexes") continue;
5126
+ if (resource.name === "plg_fulltext_indexes") continue;
4137
5127
  this.installResourceHooks(resource);
4138
5128
  }
4139
5129
  if (!this.database._fulltextProxyInstalled) {
4140
5130
  this.database._previousCreateResourceForFullText = this.database.createResource;
4141
5131
  this.database.createResource = async function(...args) {
4142
5132
  const resource = await this._previousCreateResourceForFullText(...args);
4143
- if (this.plugins?.fulltext && resource.name !== "fulltext_indexes") {
5133
+ if (this.plugins?.fulltext && resource.name !== "plg_fulltext_indexes") {
4144
5134
  this.plugins.fulltext.installResourceHooks(resource);
4145
5135
  }
4146
5136
  return resource;
@@ -4148,7 +5138,7 @@ class FullTextPlugin extends Plugin {
4148
5138
  this.database._fulltextProxyInstalled = true;
4149
5139
  }
4150
5140
  for (const resource of Object.values(this.database.resources)) {
4151
- if (resource.name !== "fulltext_indexes") {
5141
+ if (resource.name !== "plg_fulltext_indexes") {
4152
5142
  this.installResourceHooks(resource);
4153
5143
  }
4154
5144
  }
@@ -4391,7 +5381,7 @@ class FullTextPlugin extends Plugin {
4391
5381
  return this._rebuildAllIndexesInternal();
4392
5382
  }
4393
5383
  async _rebuildAllIndexesInternal() {
4394
- const resourceNames = Object.keys(this.database.resources).filter((name) => name !== "fulltext_indexes");
5384
+ const resourceNames = Object.keys(this.database.resources).filter((name) => name !== "plg_fulltext_indexes");
4395
5385
  for (const resourceName of resourceNames) {
4396
5386
  const [ok, err] = await tryFn(() => this.rebuildIndex(resourceName));
4397
5387
  }
@@ -4443,7 +5433,7 @@ class MetricsPlugin extends Plugin {
4443
5433
  if (typeof process !== "undefined" && process.env.NODE_ENV === "test") return;
4444
5434
  const [ok, err] = await tryFn(async () => {
4445
5435
  const [ok1, err1, metricsResource] = await tryFn(() => database.createResource({
4446
- name: "metrics",
5436
+ name: "plg_metrics",
4447
5437
  attributes: {
4448
5438
  id: "string|required",
4449
5439
  type: "string|required",
@@ -4458,9 +5448,9 @@ class MetricsPlugin extends Plugin {
4458
5448
  metadata: "json"
4459
5449
  }
4460
5450
  }));
4461
- this.metricsResource = ok1 ? metricsResource : database.resources.metrics;
5451
+ this.metricsResource = ok1 ? metricsResource : database.resources.plg_metrics;
4462
5452
  const [ok2, err2, errorsResource] = await tryFn(() => database.createResource({
4463
- name: "error_logs",
5453
+ name: "plg_error_logs",
4464
5454
  attributes: {
4465
5455
  id: "string|required",
4466
5456
  resourceName: "string|required",
@@ -4470,9 +5460,9 @@ class MetricsPlugin extends Plugin {
4470
5460
  metadata: "json"
4471
5461
  }
4472
5462
  }));
4473
- this.errorsResource = ok2 ? errorsResource : database.resources.error_logs;
5463
+ this.errorsResource = ok2 ? errorsResource : database.resources.plg_error_logs;
4474
5464
  const [ok3, err3, performanceResource] = await tryFn(() => database.createResource({
4475
- name: "performance_logs",
5465
+ name: "plg_performance_logs",
4476
5466
  attributes: {
4477
5467
  id: "string|required",
4478
5468
  resourceName: "string|required",
@@ -4482,12 +5472,12 @@ class MetricsPlugin extends Plugin {
4482
5472
  metadata: "json"
4483
5473
  }
4484
5474
  }));
4485
- this.performanceResource = ok3 ? performanceResource : database.resources.performance_logs;
5475
+ this.performanceResource = ok3 ? performanceResource : database.resources.plg_performance_logs;
4486
5476
  });
4487
5477
  if (!ok) {
4488
- this.metricsResource = database.resources.metrics;
4489
- this.errorsResource = database.resources.error_logs;
4490
- this.performanceResource = database.resources.performance_logs;
5478
+ this.metricsResource = database.resources.plg_metrics;
5479
+ this.errorsResource = database.resources.plg_error_logs;
5480
+ this.performanceResource = database.resources.plg_performance_logs;
4491
5481
  }
4492
5482
  this.installDatabaseHooks();
4493
5483
  this.installMetricsHooks();
@@ -4506,7 +5496,7 @@ class MetricsPlugin extends Plugin {
4506
5496
  }
4507
5497
  installDatabaseHooks() {
4508
5498
  this.database.addHook("afterCreateResource", (resource) => {
4509
- if (resource.name !== "metrics" && resource.name !== "error_logs" && resource.name !== "performance_logs") {
5499
+ if (resource.name !== "plg_metrics" && resource.name !== "plg_error_logs" && resource.name !== "plg_performance_logs") {
4510
5500
  this.installResourceHooks(resource);
4511
5501
  }
4512
5502
  });
@@ -4516,7 +5506,7 @@ class MetricsPlugin extends Plugin {
4516
5506
  }
4517
5507
  installMetricsHooks() {
4518
5508
  for (const resource of Object.values(this.database.resources)) {
4519
- if (["metrics", "error_logs", "performance_logs"].includes(resource.name)) {
5509
+ if (["plg_metrics", "plg_error_logs", "plg_performance_logs"].includes(resource.name)) {
4520
5510
  continue;
4521
5511
  }
4522
5512
  this.installResourceHooks(resource);
@@ -4524,7 +5514,7 @@ class MetricsPlugin extends Plugin {
4524
5514
  this.database._createResource = this.database.createResource;
4525
5515
  this.database.createResource = async function(...args) {
4526
5516
  const resource = await this._createResource(...args);
4527
- if (this.plugins?.metrics && !["metrics", "error_logs", "performance_logs"].includes(resource.name)) {
5517
+ if (this.plugins?.metrics && !["plg_metrics", "plg_error_logs", "plg_performance_logs"].includes(resource.name)) {
4528
5518
  this.plugins.metrics.installResourceHooks(resource);
4529
5519
  }
4530
5520
  return resource;
@@ -5830,10 +6820,10 @@ class Client extends EventEmitter {
5830
6820
  // Enabled for better performance
5831
6821
  keepAliveMsecs: 1e3,
5832
6822
  // 1 second keep-alive
5833
- maxSockets: 50,
5834
- // Balanced for most applications
5835
- maxFreeSockets: 10,
5836
- // Good connection reuse
6823
+ maxSockets: httpClientOptions.maxSockets || 500,
6824
+ // High concurrency support
6825
+ maxFreeSockets: httpClientOptions.maxFreeSockets || 100,
6826
+ // Better connection reuse
5837
6827
  timeout: 6e4,
5838
6828
  // 60 second timeout
5839
6829
  ...httpClientOptions
@@ -5895,7 +6885,7 @@ class Client extends EventEmitter {
5895
6885
  this.emit("command.response", command.constructor.name, response, command.input);
5896
6886
  return response;
5897
6887
  }
5898
- async putObject({ key, metadata, contentType, body, contentEncoding, contentLength }) {
6888
+ async putObject({ key, metadata, contentType, body, contentEncoding, contentLength, ifMatch }) {
5899
6889
  const keyPrefix = typeof this.config.keyPrefix === "string" ? this.config.keyPrefix : "";
5900
6890
  keyPrefix ? path.join(keyPrefix, key) : key;
5901
6891
  const stringMetadata = {};
@@ -5915,6 +6905,7 @@ class Client extends EventEmitter {
5915
6905
  if (contentType !== void 0) options.ContentType = contentType;
5916
6906
  if (contentEncoding !== void 0) options.ContentEncoding = contentEncoding;
5917
6907
  if (contentLength !== void 0) options.ContentLength = contentLength;
6908
+ if (ifMatch !== void 0) options.IfMatch = ifMatch;
5918
6909
  let response, error;
5919
6910
  try {
5920
6911
  response = await this.sendCommand(new PutObjectCommand(options));
@@ -8078,6 +9069,7 @@ ${errorDetails}`,
8078
9069
  data._lastModified = request.LastModified;
8079
9070
  data._hasContent = request.ContentLength > 0;
8080
9071
  data._mimeType = request.ContentType || null;
9072
+ data._etag = request.ETag;
8081
9073
  data._v = objectVersion;
8082
9074
  if (request.VersionId) data._versionId = request.VersionId;
8083
9075
  if (request.Expiration) data._expiresAt = request.Expiration;
@@ -8289,6 +9281,172 @@ ${errorDetails}`,
8289
9281
  return finalResult;
8290
9282
  }
8291
9283
  }
9284
+ /**
9285
+ * Update with conditional check (If-Match ETag)
9286
+ * @param {string} id - Resource ID
9287
+ * @param {Object} attributes - Attributes to update
9288
+ * @param {Object} options - Options including ifMatch (ETag)
9289
+ * @returns {Promise<Object>} { success: boolean, data?: Object, etag?: string, error?: string }
9290
+ * @example
9291
+ * const msg = await resource.get('msg-123');
9292
+ * const result = await resource.updateConditional('msg-123', { status: 'processing' }, { ifMatch: msg._etag });
9293
+ * if (!result.success) {
9294
+ * console.log('Update failed - object was modified by another process');
9295
+ * }
9296
+ */
9297
+ async updateConditional(id, attributes, options = {}) {
9298
+ if (isEmpty(id)) {
9299
+ throw new Error("id cannot be empty");
9300
+ }
9301
+ const { ifMatch } = options;
9302
+ if (!ifMatch) {
9303
+ throw new Error("updateConditional requires ifMatch option with ETag value");
9304
+ }
9305
+ const exists = await this.exists(id);
9306
+ if (!exists) {
9307
+ return {
9308
+ success: false,
9309
+ error: `Resource with id '${id}' does not exist`
9310
+ };
9311
+ }
9312
+ const originalData = await this.get(id);
9313
+ const attributesClone = cloneDeep(attributes);
9314
+ let mergedData = cloneDeep(originalData);
9315
+ for (const [key2, value] of Object.entries(attributesClone)) {
9316
+ if (key2.includes(".")) {
9317
+ let ref = mergedData;
9318
+ const parts = key2.split(".");
9319
+ for (let i = 0; i < parts.length - 1; i++) {
9320
+ if (typeof ref[parts[i]] !== "object" || ref[parts[i]] === null) {
9321
+ ref[parts[i]] = {};
9322
+ }
9323
+ ref = ref[parts[i]];
9324
+ }
9325
+ ref[parts[parts.length - 1]] = cloneDeep(value);
9326
+ } else if (typeof value === "object" && value !== null && !Array.isArray(value)) {
9327
+ mergedData[key2] = merge({}, mergedData[key2], value);
9328
+ } else {
9329
+ mergedData[key2] = cloneDeep(value);
9330
+ }
9331
+ }
9332
+ if (this.config.timestamps) {
9333
+ const now = (/* @__PURE__ */ new Date()).toISOString();
9334
+ mergedData.updatedAt = now;
9335
+ if (!mergedData.metadata) mergedData.metadata = {};
9336
+ mergedData.metadata.updatedAt = now;
9337
+ }
9338
+ const preProcessedData = await this.executeHooks("beforeUpdate", cloneDeep(mergedData));
9339
+ const completeData = { ...originalData, ...preProcessedData, id };
9340
+ const { isValid, errors, data } = await this.validate(cloneDeep(completeData));
9341
+ if (!isValid) {
9342
+ return {
9343
+ success: false,
9344
+ error: "Validation failed: " + (errors && errors.length ? JSON.stringify(errors) : "unknown"),
9345
+ validationErrors: errors
9346
+ };
9347
+ }
9348
+ const { id: validatedId, ...validatedAttributes } = data;
9349
+ const mappedData = await this.schema.mapper(validatedAttributes);
9350
+ mappedData._v = String(this.version);
9351
+ const behaviorImpl = getBehavior(this.behavior);
9352
+ const { mappedData: processedMetadata, body } = await behaviorImpl.handleUpdate({
9353
+ resource: this,
9354
+ id,
9355
+ data: validatedAttributes,
9356
+ mappedData,
9357
+ originalData: { ...attributesClone, id }
9358
+ });
9359
+ const key = this.getResourceKey(id);
9360
+ let existingContentType = void 0;
9361
+ let finalBody = body;
9362
+ if (body === "" && this.behavior !== "body-overflow") {
9363
+ const [ok2, err2, existingObject] = await tryFn(() => this.client.getObject(key));
9364
+ if (ok2 && existingObject.ContentLength > 0) {
9365
+ const existingBodyBuffer = Buffer.from(await existingObject.Body.transformToByteArray());
9366
+ const existingBodyString = existingBodyBuffer.toString();
9367
+ const [okParse, errParse] = await tryFn(() => Promise.resolve(JSON.parse(existingBodyString)));
9368
+ if (!okParse) {
9369
+ finalBody = existingBodyBuffer;
9370
+ existingContentType = existingObject.ContentType;
9371
+ }
9372
+ }
9373
+ }
9374
+ let finalContentType = existingContentType;
9375
+ if (finalBody && finalBody !== "" && !finalContentType) {
9376
+ const [okParse, errParse] = await tryFn(() => Promise.resolve(JSON.parse(finalBody)));
9377
+ if (okParse) finalContentType = "application/json";
9378
+ }
9379
+ const [ok, err, response] = await tryFn(() => this.client.putObject({
9380
+ key,
9381
+ body: finalBody,
9382
+ contentType: finalContentType,
9383
+ metadata: processedMetadata,
9384
+ ifMatch
9385
+ // ← Conditional write with ETag
9386
+ }));
9387
+ if (!ok) {
9388
+ if (err.name === "PreconditionFailed" || err.$metadata?.httpStatusCode === 412) {
9389
+ return {
9390
+ success: false,
9391
+ error: "ETag mismatch - object was modified by another process"
9392
+ };
9393
+ }
9394
+ return {
9395
+ success: false,
9396
+ error: err.message || "Update failed"
9397
+ };
9398
+ }
9399
+ const updatedData = await this.composeFullObjectFromWrite({
9400
+ id,
9401
+ metadata: processedMetadata,
9402
+ body: finalBody,
9403
+ behavior: this.behavior
9404
+ });
9405
+ const oldData = { ...originalData, id };
9406
+ const newData = { ...validatedAttributes, id };
9407
+ if (this.config.asyncPartitions && this.config.partitions && Object.keys(this.config.partitions).length > 0) {
9408
+ setImmediate(() => {
9409
+ this.handlePartitionReferenceUpdates(oldData, newData).catch((err2) => {
9410
+ this.emit("partitionIndexError", {
9411
+ operation: "updateConditional",
9412
+ id,
9413
+ error: err2,
9414
+ message: err2.message
9415
+ });
9416
+ });
9417
+ });
9418
+ const nonPartitionHooks = this.hooks.afterUpdate.filter(
9419
+ (hook) => !hook.toString().includes("handlePartitionReferenceUpdates")
9420
+ );
9421
+ let finalResult = updatedData;
9422
+ for (const hook of nonPartitionHooks) {
9423
+ finalResult = await hook(finalResult);
9424
+ }
9425
+ this.emit("update", {
9426
+ ...updatedData,
9427
+ $before: { ...originalData },
9428
+ $after: { ...finalResult }
9429
+ });
9430
+ return {
9431
+ success: true,
9432
+ data: finalResult,
9433
+ etag: response.ETag
9434
+ };
9435
+ } else {
9436
+ await this.handlePartitionReferenceUpdates(oldData, newData);
9437
+ const finalResult = await this.executeHooks("afterUpdate", updatedData);
9438
+ this.emit("update", {
9439
+ ...updatedData,
9440
+ $before: { ...originalData },
9441
+ $after: { ...finalResult }
9442
+ });
9443
+ return {
9444
+ success: true,
9445
+ data: finalResult,
9446
+ etag: response.ETag
9447
+ };
9448
+ }
9449
+ }
8292
9450
  /**
8293
9451
  * Delete a resource object by ID
8294
9452
  * @param {string} id - Resource ID
@@ -9696,7 +10854,7 @@ class Database extends EventEmitter {
9696
10854
  this.id = idGenerator(7);
9697
10855
  this.version = "1";
9698
10856
  this.s3dbVersion = (() => {
9699
- const [ok, err, version] = tryFn(() => true ? "9.3.0" : "latest");
10857
+ const [ok, err, version] = tryFn(() => true ? "10.0.1" : "latest");
9700
10858
  return ok ? version : "latest";
9701
10859
  })();
9702
10860
  this.resources = {};
@@ -10949,16 +12107,20 @@ class S3dbReplicator extends BaseReplicator {
10949
12107
  return resource;
10950
12108
  }
10951
12109
  _getDestResourceObj(resource) {
10952
- const available = Object.keys(this.client.resources || {});
12110
+ const db = this.targetDatabase || this.client;
12111
+ const available = Object.keys(db.resources || {});
10953
12112
  const norm = normalizeResourceName$1(resource);
10954
12113
  const found = available.find((r) => normalizeResourceName$1(r) === norm);
10955
12114
  if (!found) {
10956
12115
  throw new Error(`[S3dbReplicator] Destination resource not found: ${resource}. Available: ${available.join(", ")}`);
10957
12116
  }
10958
- return this.client.resources[found];
12117
+ return db.resources[found];
10959
12118
  }
10960
12119
  async replicateBatch(resourceName, records) {
10961
- if (!this.enabled || !this.shouldReplicateResource(resourceName)) {
12120
+ if (this.enabled === false) {
12121
+ return { skipped: true, reason: "replicator_disabled" };
12122
+ }
12123
+ if (!this.shouldReplicateResource(resourceName)) {
10962
12124
  return { skipped: true, reason: "resource_not_included" };
10963
12125
  }
10964
12126
  const results = [];
@@ -11069,11 +12231,12 @@ class SqsReplicator extends BaseReplicator {
11069
12231
  this.client = client;
11070
12232
  this.queueUrl = config.queueUrl;
11071
12233
  this.queues = config.queues || {};
11072
- this.defaultQueue = config.defaultQueue || config.defaultQueueUrl || config.queueUrlDefault;
12234
+ this.defaultQueue = config.defaultQueue || config.defaultQueueUrl || config.queueUrlDefault || null;
11073
12235
  this.region = config.region || "us-east-1";
11074
12236
  this.sqsClient = client || null;
11075
12237
  this.messageGroupId = config.messageGroupId;
11076
12238
  this.deduplicationId = config.deduplicationId;
12239
+ this.resourceQueueMap = config.resourceQueueMap || null;
11077
12240
  if (Array.isArray(resources)) {
11078
12241
  this.resources = {};
11079
12242
  for (const resource of resources) {
@@ -11204,7 +12367,10 @@ class SqsReplicator extends BaseReplicator {
11204
12367
  }
11205
12368
  }
11206
12369
  async replicate(resource, operation, data, id, beforeData = null) {
11207
- if (!this.enabled || !this.shouldReplicateResource(resource)) {
12370
+ if (this.enabled === false) {
12371
+ return { skipped: true, reason: "replicator_disabled" };
12372
+ }
12373
+ if (!this.shouldReplicateResource(resource)) {
11208
12374
  return { skipped: true, reason: "resource_not_included" };
11209
12375
  }
11210
12376
  const [ok, err, result] = await tryFn(async () => {
@@ -11248,7 +12414,10 @@ class SqsReplicator extends BaseReplicator {
11248
12414
  return { success: false, error: err.message };
11249
12415
  }
11250
12416
  async replicateBatch(resource, records) {
11251
- if (!this.enabled || !this.shouldReplicateResource(resource)) {
12417
+ if (this.enabled === false) {
12418
+ return { skipped: true, reason: "replicator_disabled" };
12419
+ }
12420
+ if (!this.shouldReplicateResource(resource)) {
11252
12421
  return { skipped: true, reason: "resource_not_included" };
11253
12422
  }
11254
12423
  const [ok, err, result] = await tryFn(async () => {
@@ -11402,22 +12571,23 @@ class ReplicatorPlugin extends Plugin {
11402
12571
  replicators: options.replicators || [],
11403
12572
  logErrors: options.logErrors !== false,
11404
12573
  replicatorLogResource: options.replicatorLogResource || "replicator_log",
12574
+ persistReplicatorLog: options.persistReplicatorLog || false,
11405
12575
  enabled: options.enabled !== false,
11406
12576
  batchSize: options.batchSize || 100,
11407
12577
  maxRetries: options.maxRetries || 3,
11408
12578
  timeout: options.timeout || 3e4,
11409
- verbose: options.verbose || false,
11410
- ...options
12579
+ verbose: options.verbose || false
11411
12580
  };
11412
12581
  this.replicators = [];
11413
12582
  this.database = null;
11414
12583
  this.eventListenersInstalled = /* @__PURE__ */ new Set();
11415
- }
11416
- /**
11417
- * Decompress data if it was compressed
11418
- */
11419
- async decompressData(data) {
11420
- return data;
12584
+ this.eventHandlers = /* @__PURE__ */ new Map();
12585
+ this.stats = {
12586
+ totalReplications: 0,
12587
+ totalErrors: 0,
12588
+ lastSync: null
12589
+ };
12590
+ this._afterCreateResourceHook = null;
11421
12591
  }
11422
12592
  // Helper to filter out internal S3DB fields
11423
12593
  filterInternalFields(obj) {
@@ -11438,7 +12608,7 @@ class ReplicatorPlugin extends Plugin {
11438
12608
  if (!resource || this.eventListenersInstalled.has(resource.name) || resource.name === this.config.replicatorLogResource) {
11439
12609
  return;
11440
12610
  }
11441
- resource.on("insert", async (data) => {
12611
+ const insertHandler = async (data) => {
11442
12612
  const [ok, error] = await tryFn(async () => {
11443
12613
  const completeData = { ...data, createdAt: (/* @__PURE__ */ new Date()).toISOString() };
11444
12614
  await plugin.processReplicatorEvent("insert", resource.name, completeData.id, completeData);
@@ -11449,8 +12619,8 @@ class ReplicatorPlugin extends Plugin {
11449
12619
  }
11450
12620
  this.emit("error", { operation: "insert", error: error.message, resource: resource.name });
11451
12621
  }
11452
- });
11453
- resource.on("update", async (data, beforeData) => {
12622
+ };
12623
+ const updateHandler = async (data, beforeData) => {
11454
12624
  const [ok, error] = await tryFn(async () => {
11455
12625
  const completeData = await plugin.getCompleteData(resource, data);
11456
12626
  const dataWithTimestamp = { ...completeData, updatedAt: (/* @__PURE__ */ new Date()).toISOString() };
@@ -11462,8 +12632,8 @@ class ReplicatorPlugin extends Plugin {
11462
12632
  }
11463
12633
  this.emit("error", { operation: "update", error: error.message, resource: resource.name });
11464
12634
  }
11465
- });
11466
- resource.on("delete", async (data) => {
12635
+ };
12636
+ const deleteHandler = async (data) => {
11467
12637
  const [ok, error] = await tryFn(async () => {
11468
12638
  await plugin.processReplicatorEvent("delete", resource.name, data.id, data);
11469
12639
  });
@@ -11473,14 +12643,22 @@ class ReplicatorPlugin extends Plugin {
11473
12643
  }
11474
12644
  this.emit("error", { operation: "delete", error: error.message, resource: resource.name });
11475
12645
  }
11476
- });
12646
+ };
12647
+ this.eventHandlers.set(resource.name, {
12648
+ insert: insertHandler,
12649
+ update: updateHandler,
12650
+ delete: deleteHandler
12651
+ });
12652
+ resource.on("insert", insertHandler);
12653
+ resource.on("update", updateHandler);
12654
+ resource.on("delete", deleteHandler);
11477
12655
  this.eventListenersInstalled.add(resource.name);
11478
12656
  }
11479
12657
  async setup(database) {
11480
12658
  this.database = database;
11481
12659
  if (this.config.persistReplicatorLog) {
11482
12660
  const [ok, err, logResource] = await tryFn(() => database.createResource({
11483
- name: this.config.replicatorLogResource || "replicator_logs",
12661
+ name: this.config.replicatorLogResource || "plg_replicator_logs",
11484
12662
  attributes: {
11485
12663
  id: "string|required",
11486
12664
  resource: "string|required",
@@ -11494,13 +12672,13 @@ class ReplicatorPlugin extends Plugin {
11494
12672
  if (ok) {
11495
12673
  this.replicatorLogResource = logResource;
11496
12674
  } else {
11497
- this.replicatorLogResource = database.resources[this.config.replicatorLogResource || "replicator_logs"];
12675
+ this.replicatorLogResource = database.resources[this.config.replicatorLogResource || "plg_replicator_logs"];
11498
12676
  }
11499
12677
  }
11500
12678
  await this.initializeReplicators(database);
11501
12679
  this.installDatabaseHooks();
11502
12680
  for (const resource of Object.values(database.resources)) {
11503
- if (resource.name !== (this.config.replicatorLogResource || "replicator_logs")) {
12681
+ if (resource.name !== (this.config.replicatorLogResource || "plg_replicator_logs")) {
11504
12682
  this.installEventListeners(resource, database, this);
11505
12683
  }
11506
12684
  }
@@ -11516,14 +12694,18 @@ class ReplicatorPlugin extends Plugin {
11516
12694
  this.removeDatabaseHooks();
11517
12695
  }
11518
12696
  installDatabaseHooks() {
11519
- this.database.addHook("afterCreateResource", (resource) => {
11520
- if (resource.name !== (this.config.replicatorLogResource || "replicator_logs")) {
12697
+ this._afterCreateResourceHook = (resource) => {
12698
+ if (resource.name !== (this.config.replicatorLogResource || "plg_replicator_logs")) {
11521
12699
  this.installEventListeners(resource, this.database, this);
11522
12700
  }
11523
- });
12701
+ };
12702
+ this.database.addHook("afterCreateResource", this._afterCreateResourceHook);
11524
12703
  }
11525
12704
  removeDatabaseHooks() {
11526
- this.database.removeHook("afterCreateResource", this.installEventListeners.bind(this));
12705
+ if (this._afterCreateResourceHook) {
12706
+ this.database.removeHook("afterCreateResource", this._afterCreateResourceHook);
12707
+ this._afterCreateResourceHook = null;
12708
+ }
11527
12709
  }
11528
12710
  createReplicator(driver, config, resources, client) {
11529
12711
  return createReplicator(driver, config, resources, client);
@@ -11548,9 +12730,9 @@ class ReplicatorPlugin extends Plugin {
11548
12730
  async retryWithBackoff(operation, maxRetries = 3) {
11549
12731
  let lastError;
11550
12732
  for (let attempt = 1; attempt <= maxRetries; attempt++) {
11551
- const [ok, error] = await tryFn(operation);
12733
+ const [ok, error, result] = await tryFn(operation);
11552
12734
  if (ok) {
11553
- return ok;
12735
+ return result;
11554
12736
  } else {
11555
12737
  lastError = error;
11556
12738
  if (this.config.verbose) {
@@ -11645,7 +12827,7 @@ class ReplicatorPlugin extends Plugin {
11645
12827
  });
11646
12828
  return Promise.allSettled(promises);
11647
12829
  }
11648
- async processreplicatorItem(item) {
12830
+ async processReplicatorItem(item) {
11649
12831
  const applicableReplicators = this.replicators.filter((replicator) => {
11650
12832
  const should = replicator.shouldReplicateResource && replicator.shouldReplicateResource(item.resourceName, item.operation);
11651
12833
  return should;
@@ -11705,12 +12887,9 @@ class ReplicatorPlugin extends Plugin {
11705
12887
  });
11706
12888
  return Promise.allSettled(promises);
11707
12889
  }
11708
- async logreplicator(item) {
12890
+ async logReplicator(item) {
11709
12891
  const logRes = this.replicatorLog || this.database.resources[normalizeResourceName(this.config.replicatorLogResource)];
11710
12892
  if (!logRes) {
11711
- if (this.database) {
11712
- if (this.database.options && this.database.options.connectionString) ;
11713
- }
11714
12893
  this.emit("replicator.log.failed", { error: "replicator log resource not found", item });
11715
12894
  return;
11716
12895
  }
@@ -11732,7 +12911,7 @@ class ReplicatorPlugin extends Plugin {
11732
12911
  this.emit("replicator.log.failed", { error: err, item });
11733
12912
  }
11734
12913
  }
11735
- async updatereplicatorLog(logId, updates) {
12914
+ async updateReplicatorLog(logId, updates) {
11736
12915
  if (!this.replicatorLog) return;
11737
12916
  const [ok, err] = await tryFn(async () => {
11738
12917
  await this.replicatorLog.update(logId, {
@@ -11745,7 +12924,7 @@ class ReplicatorPlugin extends Plugin {
11745
12924
  }
11746
12925
  }
11747
12926
  // Utility methods
11748
- async getreplicatorStats() {
12927
+ async getReplicatorStats() {
11749
12928
  const replicatorStats = await Promise.all(
11750
12929
  this.replicators.map(async (replicator) => {
11751
12930
  const status = await replicator.getStatus();
@@ -11757,116 +12936,669 @@ class ReplicatorPlugin extends Plugin {
11757
12936
  };
11758
12937
  })
11759
12938
  );
11760
- return {
11761
- replicators: replicatorStats,
11762
- queue: {
11763
- length: this.queue.length,
11764
- isProcessing: this.isProcessing
11765
- },
11766
- stats: this.stats,
11767
- lastSync: this.stats.lastSync
11768
- };
11769
- }
11770
- async getreplicatorLogs(options = {}) {
11771
- if (!this.replicatorLog) {
11772
- return [];
11773
- }
11774
- const {
11775
- resourceName,
11776
- operation,
11777
- status,
11778
- limit = 100,
11779
- offset = 0
11780
- } = options;
11781
- let query = {};
11782
- if (resourceName) {
11783
- query.resourceName = resourceName;
12939
+ return {
12940
+ replicators: replicatorStats,
12941
+ stats: this.stats,
12942
+ lastSync: this.stats.lastSync
12943
+ };
12944
+ }
12945
+ async getReplicatorLogs(options = {}) {
12946
+ if (!this.replicatorLog) {
12947
+ return [];
12948
+ }
12949
+ const {
12950
+ resourceName,
12951
+ operation,
12952
+ status,
12953
+ limit = 100,
12954
+ offset = 0
12955
+ } = options;
12956
+ const filter = {};
12957
+ if (resourceName) {
12958
+ filter.resourceName = resourceName;
12959
+ }
12960
+ if (operation) {
12961
+ filter.operation = operation;
12962
+ }
12963
+ if (status) {
12964
+ filter.status = status;
12965
+ }
12966
+ const logs = await this.replicatorLog.query(filter, { limit, offset });
12967
+ return logs || [];
12968
+ }
12969
+ async retryFailedReplicators() {
12970
+ if (!this.replicatorLog) {
12971
+ return { retried: 0 };
12972
+ }
12973
+ const failedLogs = await this.replicatorLog.query({
12974
+ status: "failed"
12975
+ });
12976
+ let retried = 0;
12977
+ for (const log of failedLogs || []) {
12978
+ const [ok, err] = await tryFn(async () => {
12979
+ await this.processReplicatorEvent(
12980
+ log.operation,
12981
+ log.resourceName,
12982
+ log.recordId,
12983
+ log.data
12984
+ );
12985
+ });
12986
+ if (ok) {
12987
+ retried++;
12988
+ }
12989
+ }
12990
+ return { retried };
12991
+ }
12992
+ async syncAllData(replicatorId) {
12993
+ const replicator = this.replicators.find((r) => r.id === replicatorId);
12994
+ if (!replicator) {
12995
+ throw new Error(`Replicator not found: ${replicatorId}`);
12996
+ }
12997
+ this.stats.lastSync = (/* @__PURE__ */ new Date()).toISOString();
12998
+ for (const resourceName in this.database.resources) {
12999
+ if (normalizeResourceName(resourceName) === normalizeResourceName("plg_replicator_logs")) continue;
13000
+ if (replicator.shouldReplicateResource(resourceName)) {
13001
+ this.emit("replicator.sync.resource", { resourceName, replicatorId });
13002
+ const resource = this.database.resources[resourceName];
13003
+ let offset = 0;
13004
+ const pageSize = this.config.batchSize || 100;
13005
+ while (true) {
13006
+ const [ok, err, page] = await tryFn(() => resource.page({ offset, size: pageSize }));
13007
+ if (!ok || !page) break;
13008
+ const records = Array.isArray(page) ? page : page.items || [];
13009
+ if (records.length === 0) break;
13010
+ for (const record of records) {
13011
+ await replicator.replicate(resourceName, "insert", record, record.id);
13012
+ }
13013
+ offset += pageSize;
13014
+ }
13015
+ }
13016
+ }
13017
+ this.emit("replicator.sync.completed", { replicatorId, stats: this.stats });
13018
+ }
13019
+ async cleanup() {
13020
+ const [ok, error] = await tryFn(async () => {
13021
+ if (this.replicators && this.replicators.length > 0) {
13022
+ const cleanupPromises = this.replicators.map(async (replicator) => {
13023
+ const [replicatorOk, replicatorError] = await tryFn(async () => {
13024
+ if (replicator && typeof replicator.cleanup === "function") {
13025
+ await replicator.cleanup();
13026
+ }
13027
+ });
13028
+ if (!replicatorOk) {
13029
+ if (this.config.verbose) {
13030
+ console.warn(`[ReplicatorPlugin] Failed to cleanup replicator ${replicator.name || replicator.id}: ${replicatorError.message}`);
13031
+ }
13032
+ this.emit("replicator_cleanup_error", {
13033
+ replicator: replicator.name || replicator.id || "unknown",
13034
+ driver: replicator.driver || "unknown",
13035
+ error: replicatorError.message
13036
+ });
13037
+ }
13038
+ });
13039
+ await Promise.allSettled(cleanupPromises);
13040
+ }
13041
+ if (this.database && this.database.resources) {
13042
+ for (const resourceName of this.eventListenersInstalled) {
13043
+ const resource = this.database.resources[resourceName];
13044
+ const handlers = this.eventHandlers.get(resourceName);
13045
+ if (resource && handlers) {
13046
+ resource.off("insert", handlers.insert);
13047
+ resource.off("update", handlers.update);
13048
+ resource.off("delete", handlers.delete);
13049
+ }
13050
+ }
13051
+ }
13052
+ this.replicators = [];
13053
+ this.database = null;
13054
+ this.eventListenersInstalled.clear();
13055
+ this.eventHandlers.clear();
13056
+ this.removeAllListeners();
13057
+ });
13058
+ if (!ok) {
13059
+ if (this.config.verbose) {
13060
+ console.warn(`[ReplicatorPlugin] Failed to cleanup plugin: ${error.message}`);
13061
+ }
13062
+ this.emit("replicator_plugin_cleanup_error", {
13063
+ error: error.message
13064
+ });
13065
+ }
13066
+ }
13067
+ }
13068
+
13069
+ class S3QueuePlugin extends Plugin {
13070
+ constructor(options = {}) {
13071
+ super(options);
13072
+ if (!options.resource) {
13073
+ throw new Error('S3QueuePlugin requires "resource" option');
13074
+ }
13075
+ this.config = {
13076
+ resource: options.resource,
13077
+ visibilityTimeout: options.visibilityTimeout || 3e4,
13078
+ // 30 seconds
13079
+ pollInterval: options.pollInterval || 1e3,
13080
+ // 1 second
13081
+ maxAttempts: options.maxAttempts || 3,
13082
+ concurrency: options.concurrency || 1,
13083
+ deadLetterResource: options.deadLetterResource || null,
13084
+ autoStart: options.autoStart !== false,
13085
+ onMessage: options.onMessage,
13086
+ onError: options.onError,
13087
+ onComplete: options.onComplete,
13088
+ verbose: options.verbose || false,
13089
+ ...options
13090
+ };
13091
+ this.queueResource = null;
13092
+ this.targetResource = null;
13093
+ this.deadLetterResourceObj = null;
13094
+ this.workers = [];
13095
+ this.isRunning = false;
13096
+ this.workerId = `worker-${Date.now()}-${Math.random().toString(36).slice(2, 9)}`;
13097
+ this.processedCache = /* @__PURE__ */ new Map();
13098
+ this.cacheCleanupInterval = null;
13099
+ this.lockCleanupInterval = null;
13100
+ }
13101
+ async onSetup() {
13102
+ this.targetResource = this.database.resources[this.config.resource];
13103
+ if (!this.targetResource) {
13104
+ throw new Error(`S3QueuePlugin: resource '${this.config.resource}' not found`);
13105
+ }
13106
+ const queueName = `${this.config.resource}_queue`;
13107
+ const [ok, err] = await tryFn(
13108
+ () => this.database.createResource({
13109
+ name: queueName,
13110
+ attributes: {
13111
+ id: "string|required",
13112
+ originalId: "string|required",
13113
+ // ID do registro original
13114
+ status: "string|required",
13115
+ // pending/processing/completed/failed/dead
13116
+ visibleAt: "number|required",
13117
+ // Timestamp de visibilidade
13118
+ claimedBy: "string|optional",
13119
+ // Worker que claimed
13120
+ claimedAt: "number|optional",
13121
+ // Timestamp do claim
13122
+ attempts: "number|default:0",
13123
+ maxAttempts: "number|default:3",
13124
+ error: "string|optional",
13125
+ result: "json|optional",
13126
+ createdAt: "string|required",
13127
+ completedAt: "number|optional"
13128
+ },
13129
+ behavior: "body-overflow",
13130
+ timestamps: true,
13131
+ asyncPartitions: true,
13132
+ partitions: {
13133
+ byStatus: { fields: { status: "string" } },
13134
+ byDate: { fields: { createdAt: "string|maxlength:10" } }
13135
+ }
13136
+ })
13137
+ );
13138
+ if (!ok && !this.database.resources[queueName]) {
13139
+ throw new Error(`Failed to create queue resource: ${err?.message}`);
13140
+ }
13141
+ this.queueResource = this.database.resources[queueName];
13142
+ const lockName = `${this.config.resource}_locks`;
13143
+ const [okLock, errLock] = await tryFn(
13144
+ () => this.database.createResource({
13145
+ name: lockName,
13146
+ attributes: {
13147
+ id: "string|required",
13148
+ workerId: "string|required",
13149
+ timestamp: "number|required",
13150
+ ttl: "number|default:5000"
13151
+ },
13152
+ behavior: "body-overflow",
13153
+ timestamps: false
13154
+ })
13155
+ );
13156
+ if (okLock || this.database.resources[lockName]) {
13157
+ this.lockResource = this.database.resources[lockName];
13158
+ } else {
13159
+ this.lockResource = null;
13160
+ if (this.config.verbose) {
13161
+ console.log(`[S3QueuePlugin] Lock resource creation failed, locking disabled: ${errLock?.message}`);
13162
+ }
13163
+ }
13164
+ this.addHelperMethods();
13165
+ if (this.config.deadLetterResource) {
13166
+ await this.createDeadLetterResource();
13167
+ }
13168
+ if (this.config.verbose) {
13169
+ console.log(`[S3QueuePlugin] Setup completed for resource '${this.config.resource}'`);
13170
+ }
13171
+ }
13172
+ async onStart() {
13173
+ if (this.config.autoStart && this.config.onMessage) {
13174
+ await this.startProcessing();
13175
+ }
13176
+ }
13177
+ async onStop() {
13178
+ await this.stopProcessing();
13179
+ }
13180
+ addHelperMethods() {
13181
+ const plugin = this;
13182
+ const resource = this.targetResource;
13183
+ resource.enqueue = async function(data, options = {}) {
13184
+ const recordData = {
13185
+ id: data.id || idGenerator(),
13186
+ ...data
13187
+ };
13188
+ const record = await resource.insert(recordData);
13189
+ const queueEntry = {
13190
+ id: idGenerator(),
13191
+ originalId: record.id,
13192
+ status: "pending",
13193
+ visibleAt: Date.now(),
13194
+ attempts: 0,
13195
+ maxAttempts: options.maxAttempts || plugin.config.maxAttempts,
13196
+ createdAt: (/* @__PURE__ */ new Date()).toISOString().slice(0, 10)
13197
+ };
13198
+ await plugin.queueResource.insert(queueEntry);
13199
+ plugin.emit("message.enqueued", { id: record.id, queueId: queueEntry.id });
13200
+ return record;
13201
+ };
13202
+ resource.queueStats = async function() {
13203
+ return await plugin.getStats();
13204
+ };
13205
+ resource.startProcessing = async function(handler, options = {}) {
13206
+ return await plugin.startProcessing(handler, options);
13207
+ };
13208
+ resource.stopProcessing = async function() {
13209
+ return await plugin.stopProcessing();
13210
+ };
13211
+ }
13212
+ async startProcessing(handler = null, options = {}) {
13213
+ if (this.isRunning) {
13214
+ if (this.config.verbose) {
13215
+ console.log("[S3QueuePlugin] Already running");
13216
+ }
13217
+ return;
13218
+ }
13219
+ const messageHandler = handler || this.config.onMessage;
13220
+ if (!messageHandler) {
13221
+ throw new Error("S3QueuePlugin: onMessage handler required");
13222
+ }
13223
+ this.isRunning = true;
13224
+ const concurrency = options.concurrency || this.config.concurrency;
13225
+ this.cacheCleanupInterval = setInterval(() => {
13226
+ const now = Date.now();
13227
+ const maxAge = 3e4;
13228
+ for (const [queueId, timestamp] of this.processedCache.entries()) {
13229
+ if (now - timestamp > maxAge) {
13230
+ this.processedCache.delete(queueId);
13231
+ }
13232
+ }
13233
+ }, 5e3);
13234
+ this.lockCleanupInterval = setInterval(() => {
13235
+ this.cleanupStaleLocks().catch((err) => {
13236
+ if (this.config.verbose) {
13237
+ console.log(`[lockCleanup] Error: ${err.message}`);
13238
+ }
13239
+ });
13240
+ }, 1e4);
13241
+ for (let i = 0; i < concurrency; i++) {
13242
+ const worker = this.createWorker(messageHandler, i);
13243
+ this.workers.push(worker);
13244
+ }
13245
+ if (this.config.verbose) {
13246
+ console.log(`[S3QueuePlugin] Started ${concurrency} workers`);
13247
+ }
13248
+ this.emit("workers.started", { concurrency, workerId: this.workerId });
13249
+ }
13250
+ async stopProcessing() {
13251
+ if (!this.isRunning) return;
13252
+ this.isRunning = false;
13253
+ if (this.cacheCleanupInterval) {
13254
+ clearInterval(this.cacheCleanupInterval);
13255
+ this.cacheCleanupInterval = null;
13256
+ }
13257
+ if (this.lockCleanupInterval) {
13258
+ clearInterval(this.lockCleanupInterval);
13259
+ this.lockCleanupInterval = null;
13260
+ }
13261
+ await Promise.all(this.workers);
13262
+ this.workers = [];
13263
+ this.processedCache.clear();
13264
+ if (this.config.verbose) {
13265
+ console.log("[S3QueuePlugin] Stopped all workers");
13266
+ }
13267
+ this.emit("workers.stopped", { workerId: this.workerId });
13268
+ }
13269
+ createWorker(handler, workerIndex) {
13270
+ return (async () => {
13271
+ while (this.isRunning) {
13272
+ try {
13273
+ const message = await this.claimMessage();
13274
+ if (message) {
13275
+ await this.processMessage(message, handler);
13276
+ } else {
13277
+ await new Promise((resolve) => setTimeout(resolve, this.config.pollInterval));
13278
+ }
13279
+ } catch (error) {
13280
+ if (this.config.verbose) {
13281
+ console.error(`[Worker ${workerIndex}] Error:`, error.message);
13282
+ }
13283
+ await new Promise((resolve) => setTimeout(resolve, 1e3));
13284
+ }
13285
+ }
13286
+ })();
13287
+ }
13288
+ async claimMessage() {
13289
+ const now = Date.now();
13290
+ const [ok, err, messages] = await tryFn(
13291
+ () => this.queueResource.query({
13292
+ status: "pending"
13293
+ })
13294
+ );
13295
+ if (!ok || !messages || messages.length === 0) {
13296
+ return null;
13297
+ }
13298
+ const available = messages.filter((m) => m.visibleAt <= now);
13299
+ if (available.length === 0) {
13300
+ return null;
13301
+ }
13302
+ for (const msg of available) {
13303
+ const claimed = await this.attemptClaim(msg);
13304
+ if (claimed) {
13305
+ return claimed;
13306
+ }
13307
+ }
13308
+ return null;
13309
+ }
13310
+ /**
13311
+ * Acquire a distributed lock using ETag-based conditional updates
13312
+ * This ensures only one worker can claim a message at a time
13313
+ *
13314
+ * Uses a two-step process:
13315
+ * 1. Create lock resource (similar to queue resource) if not exists
13316
+ * 2. Try to claim lock using ETag-based conditional update
13317
+ */
13318
+ async acquireLock(messageId) {
13319
+ if (!this.lockResource) {
13320
+ return true;
13321
+ }
13322
+ const lockId = `lock-${messageId}`;
13323
+ const now = Date.now();
13324
+ try {
13325
+ const [okGet, errGet, existingLock] = await tryFn(
13326
+ () => this.lockResource.get(lockId)
13327
+ );
13328
+ if (existingLock) {
13329
+ const lockAge = now - existingLock.timestamp;
13330
+ if (lockAge < existingLock.ttl) {
13331
+ return false;
13332
+ }
13333
+ const [ok, err, result] = await tryFn(
13334
+ () => this.lockResource.updateConditional(lockId, {
13335
+ workerId: this.workerId,
13336
+ timestamp: now,
13337
+ ttl: 5e3
13338
+ }, {
13339
+ ifMatch: existingLock._etag
13340
+ })
13341
+ );
13342
+ return ok && result.success;
13343
+ }
13344
+ const [okCreate, errCreate] = await tryFn(
13345
+ () => this.lockResource.insert({
13346
+ id: lockId,
13347
+ workerId: this.workerId,
13348
+ timestamp: now,
13349
+ ttl: 5e3
13350
+ })
13351
+ );
13352
+ return okCreate;
13353
+ } catch (error) {
13354
+ if (this.config.verbose) {
13355
+ console.log(`[acquireLock] Error: ${error.message}`);
13356
+ }
13357
+ return false;
13358
+ }
13359
+ }
13360
+ /**
13361
+ * Release a distributed lock by deleting the lock record
13362
+ */
13363
+ async releaseLock(messageId) {
13364
+ if (!this.lockResource) {
13365
+ return;
13366
+ }
13367
+ const lockId = `lock-${messageId}`;
13368
+ try {
13369
+ await this.lockResource.delete(lockId);
13370
+ } catch (error) {
13371
+ if (this.config.verbose) {
13372
+ console.log(`[releaseLock] Failed to release lock for ${messageId}: ${error.message}`);
13373
+ }
13374
+ }
13375
+ }
13376
+ /**
13377
+ * Clean up stale locks (older than TTL)
13378
+ * This prevents deadlocks if a worker crashes while holding a lock
13379
+ */
13380
+ async cleanupStaleLocks() {
13381
+ if (!this.lockResource) {
13382
+ return;
13383
+ }
13384
+ const now = Date.now();
13385
+ try {
13386
+ const locks = await this.lockResource.list();
13387
+ for (const lock of locks) {
13388
+ const lockAge = now - lock.timestamp;
13389
+ if (lockAge > lock.ttl) {
13390
+ await this.lockResource.delete(lock.id);
13391
+ if (this.config.verbose) {
13392
+ console.log(`[cleanupStaleLocks] Removed expired lock: ${lock.id}`);
13393
+ }
13394
+ }
13395
+ }
13396
+ } catch (error) {
13397
+ if (this.config.verbose) {
13398
+ console.log(`[cleanupStaleLocks] Error during cleanup: ${error.message}`);
13399
+ }
13400
+ }
13401
+ }
13402
+ async attemptClaim(msg) {
13403
+ const now = Date.now();
13404
+ const lockAcquired = await this.acquireLock(msg.id);
13405
+ if (!lockAcquired) {
13406
+ return null;
13407
+ }
13408
+ if (this.processedCache.has(msg.id)) {
13409
+ await this.releaseLock(msg.id);
13410
+ if (this.config.verbose) {
13411
+ console.log(`[attemptClaim] Message ${msg.id} already processed (in cache)`);
13412
+ }
13413
+ return null;
13414
+ }
13415
+ this.processedCache.set(msg.id, Date.now());
13416
+ await this.releaseLock(msg.id);
13417
+ const [okGet, errGet, msgWithETag] = await tryFn(
13418
+ () => this.queueResource.get(msg.id)
13419
+ );
13420
+ if (!okGet || !msgWithETag) {
13421
+ this.processedCache.delete(msg.id);
13422
+ if (this.config.verbose) {
13423
+ console.log(`[attemptClaim] Message ${msg.id} not found or error: ${errGet?.message}`);
13424
+ }
13425
+ return null;
13426
+ }
13427
+ if (msgWithETag.status !== "pending" || msgWithETag.visibleAt > now) {
13428
+ this.processedCache.delete(msg.id);
13429
+ if (this.config.verbose) {
13430
+ console.log(`[attemptClaim] Message ${msg.id} not claimable: status=${msgWithETag.status}, visibleAt=${msgWithETag.visibleAt}, now=${now}`);
13431
+ }
13432
+ return null;
13433
+ }
13434
+ if (this.config.verbose) {
13435
+ console.log(`[attemptClaim] Attempting to claim ${msg.id} with ETag: ${msgWithETag._etag}`);
13436
+ }
13437
+ const [ok, err, result] = await tryFn(
13438
+ () => this.queueResource.updateConditional(msgWithETag.id, {
13439
+ status: "processing",
13440
+ claimedBy: this.workerId,
13441
+ claimedAt: now,
13442
+ visibleAt: now + this.config.visibilityTimeout,
13443
+ attempts: msgWithETag.attempts + 1
13444
+ }, {
13445
+ ifMatch: msgWithETag._etag
13446
+ // ← ATOMIC CLAIM using ETag!
13447
+ })
13448
+ );
13449
+ if (!ok || !result.success) {
13450
+ this.processedCache.delete(msg.id);
13451
+ if (this.config.verbose) {
13452
+ console.log(`[attemptClaim] Failed to claim ${msg.id}: ${err?.message || result.error}`);
13453
+ }
13454
+ return null;
11784
13455
  }
11785
- if (operation) {
11786
- query.operation = operation;
13456
+ if (this.config.verbose) {
13457
+ console.log(`[attemptClaim] Successfully claimed ${msg.id}`);
11787
13458
  }
11788
- if (status) {
11789
- query.status = status;
13459
+ const [okRecord, errRecord, record] = await tryFn(
13460
+ () => this.targetResource.get(msgWithETag.originalId)
13461
+ );
13462
+ if (!okRecord) {
13463
+ await this.failMessage(msgWithETag.id, "Original record not found");
13464
+ return null;
11790
13465
  }
11791
- const logs = await this.replicatorLog.list(query);
11792
- return logs.slice(offset, offset + limit);
13466
+ return {
13467
+ queueId: msgWithETag.id,
13468
+ record,
13469
+ attempts: msgWithETag.attempts + 1,
13470
+ maxAttempts: msgWithETag.maxAttempts
13471
+ };
11793
13472
  }
11794
- async retryFailedreplicators() {
11795
- if (!this.replicatorLog) {
11796
- return { retried: 0 };
13473
+ async processMessage(message, handler) {
13474
+ const startTime = Date.now();
13475
+ try {
13476
+ const result = await handler(message.record, {
13477
+ queueId: message.queueId,
13478
+ attempts: message.attempts,
13479
+ workerId: this.workerId
13480
+ });
13481
+ await this.completeMessage(message.queueId, result);
13482
+ const duration = Date.now() - startTime;
13483
+ this.emit("message.completed", {
13484
+ queueId: message.queueId,
13485
+ originalId: message.record.id,
13486
+ duration,
13487
+ attempts: message.attempts
13488
+ });
13489
+ if (this.config.onComplete) {
13490
+ await this.config.onComplete(message.record, result);
13491
+ }
13492
+ } catch (error) {
13493
+ const shouldRetry = message.attempts < message.maxAttempts;
13494
+ if (shouldRetry) {
13495
+ await this.retryMessage(message.queueId, message.attempts, error.message);
13496
+ this.emit("message.retry", {
13497
+ queueId: message.queueId,
13498
+ originalId: message.record.id,
13499
+ attempts: message.attempts,
13500
+ error: error.message
13501
+ });
13502
+ } else {
13503
+ await this.moveToDeadLetter(message.queueId, message.record, error.message);
13504
+ this.emit("message.dead", {
13505
+ queueId: message.queueId,
13506
+ originalId: message.record.id,
13507
+ error: error.message
13508
+ });
13509
+ }
13510
+ if (this.config.onError) {
13511
+ await this.config.onError(error, message.record);
13512
+ }
11797
13513
  }
11798
- const failedLogs = await this.replicatorLog.list({
11799
- status: "failed"
13514
+ }
13515
+ async completeMessage(queueId, result) {
13516
+ await this.queueResource.update(queueId, {
13517
+ status: "completed",
13518
+ completedAt: Date.now(),
13519
+ result
11800
13520
  });
11801
- let retried = 0;
11802
- for (const log of failedLogs) {
11803
- const [ok, err] = await tryFn(async () => {
11804
- await this.processReplicatorEvent(
11805
- log.resourceName,
11806
- log.operation,
11807
- log.recordId,
11808
- log.data
11809
- );
13521
+ }
13522
+ async failMessage(queueId, error) {
13523
+ await this.queueResource.update(queueId, {
13524
+ status: "failed",
13525
+ error
13526
+ });
13527
+ }
13528
+ async retryMessage(queueId, attempts, error) {
13529
+ const backoff = Math.min(Math.pow(2, attempts) * 1e3, 3e4);
13530
+ await this.queueResource.update(queueId, {
13531
+ status: "pending",
13532
+ visibleAt: Date.now() + backoff,
13533
+ error
13534
+ });
13535
+ this.processedCache.delete(queueId);
13536
+ }
13537
+ async moveToDeadLetter(queueId, record, error) {
13538
+ if (this.config.deadLetterResource && this.deadLetterResourceObj) {
13539
+ const msg = await this.queueResource.get(queueId);
13540
+ await this.deadLetterResourceObj.insert({
13541
+ id: idGenerator(),
13542
+ originalId: record.id,
13543
+ queueId,
13544
+ data: record,
13545
+ error,
13546
+ attempts: msg.attempts,
13547
+ createdAt: (/* @__PURE__ */ new Date()).toISOString()
11810
13548
  });
11811
- if (ok) {
11812
- retried++;
11813
- }
11814
13549
  }
11815
- return { retried };
13550
+ await this.queueResource.update(queueId, {
13551
+ status: "dead",
13552
+ error
13553
+ });
11816
13554
  }
11817
- async syncAllData(replicatorId) {
11818
- const replicator = this.replicators.find((r) => r.id === replicatorId);
11819
- if (!replicator) {
11820
- throw new Error(`Replicator not found: ${replicatorId}`);
13555
+ async getStats() {
13556
+ const [ok, err, allMessages] = await tryFn(
13557
+ () => this.queueResource.list()
13558
+ );
13559
+ if (!ok) {
13560
+ if (this.config.verbose) {
13561
+ console.warn("[S3QueuePlugin] Failed to get stats:", err.message);
13562
+ }
13563
+ return null;
11821
13564
  }
11822
- this.stats.lastSync = (/* @__PURE__ */ new Date()).toISOString();
11823
- for (const resourceName in this.database.resources) {
11824
- if (normalizeResourceName(resourceName) === normalizeResourceName("replicator_logs")) continue;
11825
- if (replicator.shouldReplicateResource(resourceName)) {
11826
- this.emit("replicator.sync.resource", { resourceName, replicatorId });
11827
- const resource = this.database.resources[resourceName];
11828
- const allRecords = await resource.getAll();
11829
- for (const record of allRecords) {
11830
- await replicator.replicate(resourceName, "insert", record, record.id);
11831
- }
13565
+ const stats = {
13566
+ total: allMessages.length,
13567
+ pending: 0,
13568
+ processing: 0,
13569
+ completed: 0,
13570
+ failed: 0,
13571
+ dead: 0
13572
+ };
13573
+ for (const msg of allMessages) {
13574
+ if (stats[msg.status] !== void 0) {
13575
+ stats[msg.status]++;
11832
13576
  }
11833
13577
  }
11834
- this.emit("replicator.sync.completed", { replicatorId, stats: this.stats });
13578
+ return stats;
11835
13579
  }
11836
- async cleanup() {
11837
- const [ok, error] = await tryFn(async () => {
11838
- if (this.replicators && this.replicators.length > 0) {
11839
- const cleanupPromises = this.replicators.map(async (replicator) => {
11840
- const [replicatorOk, replicatorError] = await tryFn(async () => {
11841
- if (replicator && typeof replicator.cleanup === "function") {
11842
- await replicator.cleanup();
11843
- }
11844
- });
11845
- if (!replicatorOk) {
11846
- if (this.config.verbose) {
11847
- console.warn(`[ReplicatorPlugin] Failed to cleanup replicator ${replicator.name || replicator.id}: ${replicatorError.message}`);
11848
- }
11849
- this.emit("replicator_cleanup_error", {
11850
- replicator: replicator.name || replicator.id || "unknown",
11851
- driver: replicator.driver || "unknown",
11852
- error: replicatorError.message
11853
- });
11854
- }
11855
- });
11856
- await Promise.allSettled(cleanupPromises);
11857
- }
11858
- this.replicators = [];
11859
- this.database = null;
11860
- this.eventListenersInstalled.clear();
11861
- this.removeAllListeners();
11862
- });
11863
- if (!ok) {
13580
+ async createDeadLetterResource() {
13581
+ const [ok, err] = await tryFn(
13582
+ () => this.database.createResource({
13583
+ name: this.config.deadLetterResource,
13584
+ attributes: {
13585
+ id: "string|required",
13586
+ originalId: "string|required",
13587
+ queueId: "string|required",
13588
+ data: "json|required",
13589
+ error: "string|required",
13590
+ attempts: "number|required",
13591
+ createdAt: "string|required"
13592
+ },
13593
+ behavior: "body-overflow",
13594
+ timestamps: true
13595
+ })
13596
+ );
13597
+ if (ok || this.database.resources[this.config.deadLetterResource]) {
13598
+ this.deadLetterResourceObj = this.database.resources[this.config.deadLetterResource];
11864
13599
  if (this.config.verbose) {
11865
- console.warn(`[ReplicatorPlugin] Failed to cleanup plugin: ${error.message}`);
13600
+ console.log(`[S3QueuePlugin] Dead letter queue created: ${this.config.deadLetterResource}`);
11866
13601
  }
11867
- this.emit("replicator_plugin_cleanup_error", {
11868
- error: error.message
11869
- });
11870
13602
  }
11871
13603
  }
11872
13604
  }
@@ -11880,7 +13612,7 @@ class SchedulerPlugin extends Plugin {
11880
13612
  defaultTimeout: options.defaultTimeout || 3e5,
11881
13613
  // 5 minutes
11882
13614
  defaultRetries: options.defaultRetries || 1,
11883
- jobHistoryResource: options.jobHistoryResource || "job_executions",
13615
+ jobHistoryResource: options.jobHistoryResource || "plg_job_executions",
11884
13616
  persistJobs: options.persistJobs !== false,
11885
13617
  verbose: options.verbose || false,
11886
13618
  onJobStart: options.onJobStart || null,
@@ -11889,12 +13621,20 @@ class SchedulerPlugin extends Plugin {
11889
13621
  ...options
11890
13622
  };
11891
13623
  this.database = null;
13624
+ this.lockResource = null;
11892
13625
  this.jobs = /* @__PURE__ */ new Map();
11893
13626
  this.activeJobs = /* @__PURE__ */ new Map();
11894
13627
  this.timers = /* @__PURE__ */ new Map();
11895
13628
  this.statistics = /* @__PURE__ */ new Map();
11896
13629
  this._validateConfiguration();
11897
13630
  }
13631
+ /**
13632
+ * Helper to detect test environment
13633
+ * @private
13634
+ */
13635
+ _isTestEnvironment() {
13636
+ return process.env.NODE_ENV === "test" || process.env.JEST_WORKER_ID !== void 0 || global.expect !== void 0;
13637
+ }
11898
13638
  _validateConfiguration() {
11899
13639
  if (Object.keys(this.config.jobs).length === 0) {
11900
13640
  throw new Error("SchedulerPlugin: At least one job must be defined");
@@ -11921,6 +13661,7 @@ class SchedulerPlugin extends Plugin {
11921
13661
  }
11922
13662
  async setup(database) {
11923
13663
  this.database = database;
13664
+ await this._createLockResource();
11924
13665
  if (this.config.persistJobs) {
11925
13666
  await this._createJobHistoryResource();
11926
13667
  }
@@ -11949,6 +13690,25 @@ class SchedulerPlugin extends Plugin {
11949
13690
  await this._startScheduling();
11950
13691
  this.emit("initialized", { jobs: this.jobs.size });
11951
13692
  }
13693
+ async _createLockResource() {
13694
+ const [ok, err, lockResource] = await tryFn(
13695
+ () => this.database.createResource({
13696
+ name: "plg_scheduler_job_locks",
13697
+ attributes: {
13698
+ id: "string|required",
13699
+ jobName: "string|required",
13700
+ lockedAt: "number|required",
13701
+ instanceId: "string|optional"
13702
+ },
13703
+ behavior: "body-only",
13704
+ timestamps: false
13705
+ })
13706
+ );
13707
+ if (!ok && !this.database.resources.plg_scheduler_job_locks) {
13708
+ throw new Error(`Failed to create lock resource: ${err?.message}`);
13709
+ }
13710
+ this.lockResource = ok ? lockResource : this.database.resources.plg_scheduler_job_locks;
13711
+ }
11952
13712
  async _createJobHistoryResource() {
11953
13713
  const [ok] = await tryFn(() => this.database.createResource({
11954
13714
  name: this.config.jobHistoryResource,
@@ -12042,18 +13802,37 @@ class SchedulerPlugin extends Plugin {
12042
13802
  next.setHours(next.getHours() + 1);
12043
13803
  }
12044
13804
  }
12045
- const isTestEnvironment = process.env.NODE_ENV === "test" || process.env.JEST_WORKER_ID !== void 0 || global.expect !== void 0;
12046
- if (isTestEnvironment) {
13805
+ if (this._isTestEnvironment()) {
12047
13806
  next.setTime(next.getTime() + 1e3);
12048
13807
  }
12049
13808
  return next;
12050
13809
  }
12051
13810
  async _executeJob(jobName) {
12052
13811
  const job = this.jobs.get(jobName);
12053
- if (!job || this.activeJobs.has(jobName)) {
13812
+ if (!job) {
13813
+ return;
13814
+ }
13815
+ if (this.activeJobs.has(jobName)) {
13816
+ return;
13817
+ }
13818
+ this.activeJobs.set(jobName, "acquiring-lock");
13819
+ const lockId = `lock-${jobName}`;
13820
+ const [lockAcquired, lockErr] = await tryFn(
13821
+ () => this.lockResource.insert({
13822
+ id: lockId,
13823
+ jobName,
13824
+ lockedAt: Date.now(),
13825
+ instanceId: process.pid ? String(process.pid) : "unknown"
13826
+ })
13827
+ );
13828
+ if (!lockAcquired) {
13829
+ if (this.config.verbose) {
13830
+ console.log(`[SchedulerPlugin] Job '${jobName}' already running on another instance`);
13831
+ }
13832
+ this.activeJobs.delete(jobName);
12054
13833
  return;
12055
13834
  }
12056
- const executionId = `${jobName}_${Date.now()}`;
13835
+ const executionId = `${jobName}_${idGenerator()}`;
12057
13836
  const startTime = Date.now();
12058
13837
  const context = {
12059
13838
  jobName,
@@ -12062,91 +13841,95 @@ class SchedulerPlugin extends Plugin {
12062
13841
  database: this.database
12063
13842
  };
12064
13843
  this.activeJobs.set(jobName, executionId);
12065
- if (this.config.onJobStart) {
12066
- await this._executeHook(this.config.onJobStart, jobName, context);
12067
- }
12068
- this.emit("job_start", { jobName, executionId, startTime });
12069
- let attempt = 0;
12070
- let lastError = null;
12071
- let result = null;
12072
- let status = "success";
12073
- const isTestEnvironment = process.env.NODE_ENV === "test" || process.env.JEST_WORKER_ID !== void 0 || global.expect !== void 0;
12074
- while (attempt <= job.retries) {
12075
- try {
12076
- const actualTimeout = isTestEnvironment ? Math.min(job.timeout, 1e3) : job.timeout;
12077
- let timeoutId;
12078
- const timeoutPromise = new Promise((_, reject) => {
12079
- timeoutId = setTimeout(() => reject(new Error("Job execution timeout")), actualTimeout);
12080
- });
12081
- const jobPromise = job.action(this.database, context, this);
13844
+ try {
13845
+ if (this.config.onJobStart) {
13846
+ await this._executeHook(this.config.onJobStart, jobName, context);
13847
+ }
13848
+ this.emit("job_start", { jobName, executionId, startTime });
13849
+ let attempt = 0;
13850
+ let lastError = null;
13851
+ let result = null;
13852
+ let status = "success";
13853
+ const isTestEnvironment = this._isTestEnvironment();
13854
+ while (attempt <= job.retries) {
12082
13855
  try {
12083
- result = await Promise.race([jobPromise, timeoutPromise]);
12084
- clearTimeout(timeoutId);
12085
- } catch (raceError) {
12086
- clearTimeout(timeoutId);
12087
- throw raceError;
12088
- }
12089
- status = "success";
12090
- break;
12091
- } catch (error) {
12092
- lastError = error;
12093
- attempt++;
12094
- if (attempt <= job.retries) {
12095
- if (this.config.verbose) {
12096
- console.warn(`[SchedulerPlugin] Job '${jobName}' failed (attempt ${attempt + 1}):`, error.message);
13856
+ const actualTimeout = isTestEnvironment ? Math.min(job.timeout, 1e3) : job.timeout;
13857
+ let timeoutId;
13858
+ const timeoutPromise = new Promise((_, reject) => {
13859
+ timeoutId = setTimeout(() => reject(new Error("Job execution timeout")), actualTimeout);
13860
+ });
13861
+ const jobPromise = job.action(this.database, context, this);
13862
+ try {
13863
+ result = await Promise.race([jobPromise, timeoutPromise]);
13864
+ clearTimeout(timeoutId);
13865
+ } catch (raceError) {
13866
+ clearTimeout(timeoutId);
13867
+ throw raceError;
13868
+ }
13869
+ status = "success";
13870
+ break;
13871
+ } catch (error) {
13872
+ lastError = error;
13873
+ attempt++;
13874
+ if (attempt <= job.retries) {
13875
+ if (this.config.verbose) {
13876
+ console.warn(`[SchedulerPlugin] Job '${jobName}' failed (attempt ${attempt + 1}):`, error.message);
13877
+ }
13878
+ const baseDelay = Math.min(Math.pow(2, attempt) * 1e3, 5e3);
13879
+ const delay = isTestEnvironment ? 1 : baseDelay;
13880
+ await new Promise((resolve) => setTimeout(resolve, delay));
12097
13881
  }
12098
- const baseDelay = Math.min(Math.pow(2, attempt) * 1e3, 5e3);
12099
- const delay = isTestEnvironment ? 1 : baseDelay;
12100
- await new Promise((resolve) => setTimeout(resolve, delay));
12101
13882
  }
12102
13883
  }
12103
- }
12104
- const endTime = Date.now();
12105
- const duration = Math.max(1, endTime - startTime);
12106
- if (lastError && attempt > job.retries) {
12107
- status = lastError.message.includes("timeout") ? "timeout" : "error";
12108
- }
12109
- job.lastRun = new Date(endTime);
12110
- job.runCount++;
12111
- if (status === "success") {
12112
- job.successCount++;
12113
- } else {
12114
- job.errorCount++;
12115
- }
12116
- const stats = this.statistics.get(jobName);
12117
- stats.totalRuns++;
12118
- stats.lastRun = new Date(endTime);
12119
- if (status === "success") {
12120
- stats.totalSuccesses++;
12121
- stats.lastSuccess = new Date(endTime);
12122
- } else {
12123
- stats.totalErrors++;
12124
- stats.lastError = { time: new Date(endTime), message: lastError?.message };
12125
- }
12126
- stats.avgDuration = (stats.avgDuration * (stats.totalRuns - 1) + duration) / stats.totalRuns;
12127
- if (this.config.persistJobs) {
12128
- await this._persistJobExecution(jobName, executionId, startTime, endTime, duration, status, result, lastError, attempt);
12129
- }
12130
- if (status === "success" && this.config.onJobComplete) {
12131
- await this._executeHook(this.config.onJobComplete, jobName, result, duration);
12132
- } else if (status !== "success" && this.config.onJobError) {
12133
- await this._executeHook(this.config.onJobError, jobName, lastError, attempt);
12134
- }
12135
- this.emit("job_complete", {
12136
- jobName,
12137
- executionId,
12138
- status,
12139
- duration,
12140
- result,
12141
- error: lastError?.message,
12142
- retryCount: attempt
12143
- });
12144
- this.activeJobs.delete(jobName);
12145
- if (job.enabled) {
12146
- this._scheduleNextExecution(jobName);
12147
- }
12148
- if (lastError && status !== "success") {
12149
- throw lastError;
13884
+ const endTime = Date.now();
13885
+ const duration = Math.max(1, endTime - startTime);
13886
+ if (lastError && attempt > job.retries) {
13887
+ status = lastError.message.includes("timeout") ? "timeout" : "error";
13888
+ }
13889
+ job.lastRun = new Date(endTime);
13890
+ job.runCount++;
13891
+ if (status === "success") {
13892
+ job.successCount++;
13893
+ } else {
13894
+ job.errorCount++;
13895
+ }
13896
+ const stats = this.statistics.get(jobName);
13897
+ stats.totalRuns++;
13898
+ stats.lastRun = new Date(endTime);
13899
+ if (status === "success") {
13900
+ stats.totalSuccesses++;
13901
+ stats.lastSuccess = new Date(endTime);
13902
+ } else {
13903
+ stats.totalErrors++;
13904
+ stats.lastError = { time: new Date(endTime), message: lastError?.message };
13905
+ }
13906
+ stats.avgDuration = (stats.avgDuration * (stats.totalRuns - 1) + duration) / stats.totalRuns;
13907
+ if (this.config.persistJobs) {
13908
+ await this._persistJobExecution(jobName, executionId, startTime, endTime, duration, status, result, lastError, attempt);
13909
+ }
13910
+ if (status === "success" && this.config.onJobComplete) {
13911
+ await this._executeHook(this.config.onJobComplete, jobName, result, duration);
13912
+ } else if (status !== "success" && this.config.onJobError) {
13913
+ await this._executeHook(this.config.onJobError, jobName, lastError, attempt);
13914
+ }
13915
+ this.emit("job_complete", {
13916
+ jobName,
13917
+ executionId,
13918
+ status,
13919
+ duration,
13920
+ result,
13921
+ error: lastError?.message,
13922
+ retryCount: attempt
13923
+ });
13924
+ this.activeJobs.delete(jobName);
13925
+ if (job.enabled) {
13926
+ this._scheduleNextExecution(jobName);
13927
+ }
13928
+ if (lastError && status !== "success") {
13929
+ throw lastError;
13930
+ }
13931
+ } finally {
13932
+ await tryFn(() => this.lockResource.delete(lockId));
12150
13933
  }
12151
13934
  }
12152
13935
  async _persistJobExecution(jobName, executionId, startTime, endTime, duration, status, result, error, retryCount) {
@@ -12178,6 +13961,7 @@ class SchedulerPlugin extends Plugin {
12178
13961
  }
12179
13962
  /**
12180
13963
  * Manually trigger a job execution
13964
+ * Note: Race conditions are prevented by distributed locking in _executeJob()
12181
13965
  */
12182
13966
  async runJob(jobName, context = {}) {
12183
13967
  const job = this.jobs.get(jobName);
@@ -12263,12 +14047,15 @@ class SchedulerPlugin extends Plugin {
12263
14047
  return [];
12264
14048
  }
12265
14049
  const { limit = 50, status = null } = options;
12266
- const [ok, err, allHistory] = await tryFn(
12267
- () => this.database.resource(this.config.jobHistoryResource).list({
12268
- orderBy: { startTime: "desc" },
12269
- limit: limit * 2
12270
- // Get more to allow for filtering
12271
- })
14050
+ const queryParams = {
14051
+ jobName
14052
+ // Uses byJob partition for efficient lookup
14053
+ };
14054
+ if (status) {
14055
+ queryParams.status = status;
14056
+ }
14057
+ const [ok, err, history] = await tryFn(
14058
+ () => this.database.resource(this.config.jobHistoryResource).query(queryParams)
12272
14059
  );
12273
14060
  if (!ok) {
12274
14061
  if (this.config.verbose) {
@@ -12276,11 +14063,7 @@ class SchedulerPlugin extends Plugin {
12276
14063
  }
12277
14064
  return [];
12278
14065
  }
12279
- let filtered = allHistory.filter((h) => h.jobName === jobName);
12280
- if (status) {
12281
- filtered = filtered.filter((h) => h.status === status);
12282
- }
12283
- filtered = filtered.sort((a, b) => b.startTime - a.startTime).slice(0, limit);
14066
+ let filtered = history.sort((a, b) => b.startTime - a.startTime).slice(0, limit);
12284
14067
  return filtered.map((h) => {
12285
14068
  let result = null;
12286
14069
  if (h.result) {
@@ -12375,8 +14158,7 @@ class SchedulerPlugin extends Plugin {
12375
14158
  clearTimeout(timer);
12376
14159
  }
12377
14160
  this.timers.clear();
12378
- const isTestEnvironment = process.env.NODE_ENV === "test" || process.env.JEST_WORKER_ID !== void 0 || global.expect !== void 0;
12379
- if (!isTestEnvironment && this.activeJobs.size > 0) {
14161
+ if (!this._isTestEnvironment() && this.activeJobs.size > 0) {
12380
14162
  if (this.config.verbose) {
12381
14163
  console.log(`[SchedulerPlugin] Waiting for ${this.activeJobs.size} active jobs to complete...`);
12382
14164
  }
@@ -12389,7 +14171,7 @@ class SchedulerPlugin extends Plugin {
12389
14171
  console.warn(`[SchedulerPlugin] ${this.activeJobs.size} jobs still running after timeout`);
12390
14172
  }
12391
14173
  }
12392
- if (isTestEnvironment) {
14174
+ if (this._isTestEnvironment()) {
12393
14175
  this.activeJobs.clear();
12394
14176
  }
12395
14177
  }
@@ -12410,14 +14192,14 @@ class StateMachinePlugin extends Plugin {
12410
14192
  actions: options.actions || {},
12411
14193
  guards: options.guards || {},
12412
14194
  persistTransitions: options.persistTransitions !== false,
12413
- transitionLogResource: options.transitionLogResource || "state_transitions",
12414
- stateResource: options.stateResource || "entity_states",
12415
- verbose: options.verbose || false,
12416
- ...options
14195
+ transitionLogResource: options.transitionLogResource || "plg_state_transitions",
14196
+ stateResource: options.stateResource || "plg_entity_states",
14197
+ retryAttempts: options.retryAttempts || 3,
14198
+ retryDelay: options.retryDelay || 100,
14199
+ verbose: options.verbose || false
12417
14200
  };
12418
14201
  this.database = null;
12419
14202
  this.machines = /* @__PURE__ */ new Map();
12420
- this.stateStorage = /* @__PURE__ */ new Map();
12421
14203
  this._validateConfiguration();
12422
14204
  }
12423
14205
  _validateConfiguration() {
@@ -12558,43 +14340,55 @@ class StateMachinePlugin extends Plugin {
12558
14340
  machine.currentStates.set(entityId, toState);
12559
14341
  if (this.config.persistTransitions) {
12560
14342
  const transitionId = `${machineId}_${entityId}_${timestamp}`;
12561
- const [logOk, logErr] = await tryFn(
12562
- () => this.database.resource(this.config.transitionLogResource).insert({
12563
- id: transitionId,
12564
- machineId,
12565
- entityId,
12566
- fromState,
12567
- toState,
12568
- event,
12569
- context,
12570
- timestamp,
12571
- createdAt: now.slice(0, 10)
12572
- // YYYY-MM-DD for partitioning
12573
- })
12574
- );
14343
+ let logOk = false;
14344
+ let lastLogErr;
14345
+ for (let attempt = 0; attempt < this.config.retryAttempts; attempt++) {
14346
+ const [ok, err] = await tryFn(
14347
+ () => this.database.resource(this.config.transitionLogResource).insert({
14348
+ id: transitionId,
14349
+ machineId,
14350
+ entityId,
14351
+ fromState,
14352
+ toState,
14353
+ event,
14354
+ context,
14355
+ timestamp,
14356
+ createdAt: now.slice(0, 10)
14357
+ // YYYY-MM-DD for partitioning
14358
+ })
14359
+ );
14360
+ if (ok) {
14361
+ logOk = true;
14362
+ break;
14363
+ }
14364
+ lastLogErr = err;
14365
+ if (attempt < this.config.retryAttempts - 1) {
14366
+ const delay = this.config.retryDelay * Math.pow(2, attempt);
14367
+ await new Promise((resolve) => setTimeout(resolve, delay));
14368
+ }
14369
+ }
12575
14370
  if (!logOk && this.config.verbose) {
12576
- console.warn(`[StateMachinePlugin] Failed to log transition:`, logErr.message);
14371
+ console.warn(`[StateMachinePlugin] Failed to log transition after ${this.config.retryAttempts} attempts:`, lastLogErr.message);
12577
14372
  }
12578
14373
  const stateId = `${machineId}_${entityId}`;
12579
- const [stateOk, stateErr] = await tryFn(async () => {
12580
- const exists = await this.database.resource(this.config.stateResource).exists(stateId);
12581
- const stateData = {
12582
- id: stateId,
12583
- machineId,
12584
- entityId,
12585
- currentState: toState,
12586
- context,
12587
- lastTransition: transitionId,
12588
- updatedAt: now
12589
- };
12590
- if (exists) {
12591
- await this.database.resource(this.config.stateResource).update(stateId, stateData);
12592
- } else {
12593
- await this.database.resource(this.config.stateResource).insert(stateData);
14374
+ const stateData = {
14375
+ machineId,
14376
+ entityId,
14377
+ currentState: toState,
14378
+ context,
14379
+ lastTransition: transitionId,
14380
+ updatedAt: now
14381
+ };
14382
+ const [updateOk] = await tryFn(
14383
+ () => this.database.resource(this.config.stateResource).update(stateId, stateData)
14384
+ );
14385
+ if (!updateOk) {
14386
+ const [insertOk, insertErr] = await tryFn(
14387
+ () => this.database.resource(this.config.stateResource).insert({ id: stateId, ...stateData })
14388
+ );
14389
+ if (!insertOk && this.config.verbose) {
14390
+ console.warn(`[StateMachinePlugin] Failed to upsert state:`, insertErr.message);
12594
14391
  }
12595
- });
12596
- if (!stateOk && this.config.verbose) {
12597
- console.warn(`[StateMachinePlugin] Failed to update state:`, stateErr.message);
12598
14392
  }
12599
14393
  }
12600
14394
  }
@@ -12625,8 +14419,9 @@ class StateMachinePlugin extends Plugin {
12625
14419
  }
12626
14420
  /**
12627
14421
  * Get valid events for current state
14422
+ * Can accept either a state name (sync) or entityId (async to fetch latest state)
12628
14423
  */
12629
- getValidEvents(machineId, stateOrEntityId) {
14424
+ async getValidEvents(machineId, stateOrEntityId) {
12630
14425
  const machine = this.machines.get(machineId);
12631
14426
  if (!machine) {
12632
14427
  throw new Error(`State machine '${machineId}' not found`);
@@ -12635,7 +14430,7 @@ class StateMachinePlugin extends Plugin {
12635
14430
  if (machine.config.states[stateOrEntityId]) {
12636
14431
  state = stateOrEntityId;
12637
14432
  } else {
12638
- state = machine.currentStates.get(stateOrEntityId) || machine.config.initialState;
14433
+ state = await this.getState(machineId, stateOrEntityId);
12639
14434
  }
12640
14435
  const stateConfig = machine.config.states[state];
12641
14436
  return stateConfig && stateConfig.on ? Object.keys(stateConfig.on) : [];
@@ -12649,9 +14444,10 @@ class StateMachinePlugin extends Plugin {
12649
14444
  }
12650
14445
  const { limit = 50, offset = 0 } = options;
12651
14446
  const [ok, err, transitions] = await tryFn(
12652
- () => this.database.resource(this.config.transitionLogResource).list({
12653
- where: { machineId, entityId },
12654
- orderBy: { timestamp: "desc" },
14447
+ () => this.database.resource(this.config.transitionLogResource).query({
14448
+ machineId,
14449
+ entityId
14450
+ }, {
12655
14451
  limit,
12656
14452
  offset
12657
14453
  })
@@ -12662,8 +14458,8 @@ class StateMachinePlugin extends Plugin {
12662
14458
  }
12663
14459
  return [];
12664
14460
  }
12665
- const sortedTransitions = transitions.sort((a, b) => b.timestamp - a.timestamp);
12666
- return sortedTransitions.map((t) => ({
14461
+ const sorted = (transitions || []).sort((a, b) => b.timestamp - a.timestamp);
14462
+ return sorted.map((t) => ({
12667
14463
  from: t.fromState,
12668
14464
  to: t.toState,
12669
14465
  event: t.event,
@@ -12684,15 +14480,20 @@ class StateMachinePlugin extends Plugin {
12684
14480
  if (this.config.persistTransitions) {
12685
14481
  const now = (/* @__PURE__ */ new Date()).toISOString();
12686
14482
  const stateId = `${machineId}_${entityId}`;
12687
- await this.database.resource(this.config.stateResource).insert({
12688
- id: stateId,
12689
- machineId,
12690
- entityId,
12691
- currentState: initialState,
12692
- context,
12693
- lastTransition: null,
12694
- updatedAt: now
12695
- });
14483
+ const [ok, err] = await tryFn(
14484
+ () => this.database.resource(this.config.stateResource).insert({
14485
+ id: stateId,
14486
+ machineId,
14487
+ entityId,
14488
+ currentState: initialState,
14489
+ context,
14490
+ lastTransition: null,
14491
+ updatedAt: now
14492
+ })
14493
+ );
14494
+ if (!ok && err && !err.message?.includes("already exists")) {
14495
+ throw new Error(`Failed to initialize entity state: ${err.message}`);
14496
+ }
12696
14497
  }
12697
14498
  const initialStateConfig = machine.config.states[initialState];
12698
14499
  if (initialStateConfig && initialStateConfig.entry) {
@@ -12757,7 +14558,6 @@ class StateMachinePlugin extends Plugin {
12757
14558
  }
12758
14559
  async stop() {
12759
14560
  this.machines.clear();
12760
- this.stateStorage.clear();
12761
14561
  }
12762
14562
  async cleanup() {
12763
14563
  await this.stop();
@@ -12765,5 +14565,5 @@ class StateMachinePlugin extends Plugin {
12765
14565
  }
12766
14566
  }
12767
14567
 
12768
- export { AVAILABLE_BEHAVIORS, AuditPlugin, AuthenticationError, BackupPlugin, BaseError, CachePlugin, Client, ConnectionString, ConnectionStringError, CostsPlugin, CryptoError, DEFAULT_BEHAVIOR, Database, DatabaseError, EncryptionError, ErrorMap, FullTextPlugin, InvalidResourceItem, MetricsPlugin, MissingMetadata, NoSuchBucket, NoSuchKey, NotFound, PartitionError, PermissionError, Plugin, PluginObject, ReplicatorPlugin, Resource, ResourceError, ResourceIdsPageReader, ResourceIdsReader, ResourceNotFound, ResourceReader, ResourceWriter, Database as S3db, S3dbError, SchedulerPlugin, Schema, SchemaError, StateMachinePlugin, UnknownError, ValidationError, Validator, behaviors, calculateAttributeNamesSize, calculateAttributeSizes, calculateEffectiveLimit, calculateSystemOverhead, calculateTotalSize, calculateUTF8Bytes, clearUTF8Cache, clearUTF8Memo, clearUTF8Memory, decode, decodeDecimal, decrypt, S3db as default, encode, encodeDecimal, encrypt, getBehavior, getSizeBreakdown, idGenerator, mapAwsError, md5, passwordGenerator, sha256, streamToString, transformValue, tryFn, tryFnSync };
14568
+ export { AVAILABLE_BEHAVIORS, AuditPlugin, AuthenticationError, BackupPlugin, BaseError, CachePlugin, Client, ConnectionString, ConnectionStringError, CostsPlugin, CryptoError, DEFAULT_BEHAVIOR, Database, DatabaseError, EncryptionError, ErrorMap, EventualConsistencyPlugin, FullTextPlugin, InvalidResourceItem, MetricsPlugin, MissingMetadata, NoSuchBucket, NoSuchKey, NotFound, PartitionError, PermissionError, Plugin, PluginObject, ReplicatorPlugin, Resource, ResourceError, ResourceIdsPageReader, ResourceIdsReader, ResourceNotFound, ResourceReader, ResourceWriter, S3QueuePlugin, Database as S3db, S3dbError, SchedulerPlugin, Schema, SchemaError, StateMachinePlugin, UnknownError, ValidationError, Validator, behaviors, calculateAttributeNamesSize, calculateAttributeSizes, calculateEffectiveLimit, calculateSystemOverhead, calculateTotalSize, calculateUTF8Bytes, clearUTF8Cache, clearUTF8Memo, clearUTF8Memory, decode, decodeDecimal, decrypt, S3db as default, encode, encodeDecimal, encrypt, getBehavior, getSizeBreakdown, idGenerator, mapAwsError, md5, passwordGenerator, sha256, streamToString, transformValue, tryFn, tryFnSync };
12769
14569
  //# sourceMappingURL=s3db.es.js.map