s3db.js 10.0.0 → 10.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/s3db.cjs.js CHANGED
@@ -10,11 +10,12 @@ var promises$1 = require('stream/promises');
10
10
  var path = require('path');
11
11
  var crypto = require('crypto');
12
12
  var zlib = require('node:zlib');
13
+ var os = require('os');
14
+ var jsonStableStringify = require('json-stable-stringify');
13
15
  var stream = require('stream');
14
16
  var promisePool = require('@supercharge/promise-pool');
15
17
  var web = require('node:stream/web');
16
18
  var lodashEs = require('lodash-es');
17
- var jsonStableStringify = require('json-stable-stringify');
18
19
  var http = require('http');
19
20
  var https = require('https');
20
21
  var nodeHttpHandler = require('@smithy/node-http-handler');
@@ -842,7 +843,7 @@ class AuditPlugin extends Plugin {
842
843
  }
843
844
  async onSetup() {
844
845
  const [ok, err, auditResource] = await tryFn(() => this.database.createResource({
845
- name: "audits",
846
+ name: "plg_audits",
846
847
  attributes: {
847
848
  id: "string|required",
848
849
  resourceName: "string|required",
@@ -858,15 +859,15 @@ class AuditPlugin extends Plugin {
858
859
  },
859
860
  behavior: "body-overflow"
860
861
  }));
861
- this.auditResource = ok ? auditResource : this.database.resources.audits || null;
862
+ this.auditResource = ok ? auditResource : this.database.resources.plg_audits || null;
862
863
  if (!ok && !this.auditResource) return;
863
864
  this.database.addHook("afterCreateResource", (context) => {
864
- if (context.resource.name !== "audits") {
865
+ if (context.resource.name !== "plg_audits") {
865
866
  this.setupResourceAuditing(context.resource);
866
867
  }
867
868
  });
868
869
  for (const resource of Object.values(this.database.resources)) {
869
- if (resource.name !== "audits") {
870
+ if (resource.name !== "plg_audits") {
870
871
  this.setupResourceAuditing(resource);
871
872
  }
872
873
  }
@@ -1886,11 +1887,10 @@ function validateBackupConfig(driver, config = {}) {
1886
1887
  class BackupPlugin extends Plugin {
1887
1888
  constructor(options = {}) {
1888
1889
  super();
1889
- this.driverName = options.driver || "filesystem";
1890
- this.driverConfig = options.config || {};
1891
1890
  this.config = {
1892
- // Legacy destinations support (will be converted to multi driver)
1893
- destinations: options.destinations || null,
1891
+ // Driver configuration
1892
+ driver: options.driver || "filesystem",
1893
+ driverConfig: options.config || {},
1894
1894
  // Scheduling configuration
1895
1895
  schedule: options.schedule || {},
1896
1896
  // Retention policy (Grandfather-Father-Son)
@@ -1908,8 +1908,8 @@ class BackupPlugin extends Plugin {
1908
1908
  parallelism: options.parallelism || 4,
1909
1909
  include: options.include || null,
1910
1910
  exclude: options.exclude || [],
1911
- backupMetadataResource: options.backupMetadataResource || "backup_metadata",
1912
- tempDir: options.tempDir || "/tmp/s3db/backups",
1911
+ backupMetadataResource: options.backupMetadataResource || "plg_backup_metadata",
1912
+ tempDir: options.tempDir || path.join(os.tmpdir(), "s3db", "backups"),
1913
1913
  verbose: options.verbose || false,
1914
1914
  // Hooks
1915
1915
  onBackupStart: options.onBackupStart || null,
@@ -1921,32 +1921,9 @@ class BackupPlugin extends Plugin {
1921
1921
  };
1922
1922
  this.driver = null;
1923
1923
  this.activeBackups = /* @__PURE__ */ new Set();
1924
- this._handleLegacyDestinations();
1925
- validateBackupConfig(this.driverName, this.driverConfig);
1924
+ validateBackupConfig(this.config.driver, this.config.driverConfig);
1926
1925
  this._validateConfiguration();
1927
1926
  }
1928
- /**
1929
- * Convert legacy destinations format to multi driver format
1930
- */
1931
- _handleLegacyDestinations() {
1932
- if (this.config.destinations && Array.isArray(this.config.destinations)) {
1933
- this.driverName = "multi";
1934
- this.driverConfig = {
1935
- strategy: "all",
1936
- destinations: this.config.destinations.map((dest) => {
1937
- const { type, ...config } = dest;
1938
- return {
1939
- driver: type,
1940
- config
1941
- };
1942
- })
1943
- };
1944
- this.config.destinations = null;
1945
- if (this.config.verbose) {
1946
- console.log("[BackupPlugin] Converted legacy destinations format to multi driver");
1947
- }
1948
- }
1949
- }
1950
1927
  _validateConfiguration() {
1951
1928
  if (this.config.encryption && (!this.config.encryption.key || !this.config.encryption.algorithm)) {
1952
1929
  throw new Error("BackupPlugin: Encryption requires both key and algorithm");
@@ -1956,7 +1933,7 @@ class BackupPlugin extends Plugin {
1956
1933
  }
1957
1934
  }
1958
1935
  async onSetup() {
1959
- this.driver = createBackupDriver(this.driverName, this.driverConfig);
1936
+ this.driver = createBackupDriver(this.config.driver, this.config.driverConfig);
1960
1937
  await this.driver.setup(this.database);
1961
1938
  await promises.mkdir(this.config.tempDir, { recursive: true });
1962
1939
  await this._createBackupMetadataResource();
@@ -2004,6 +1981,9 @@ class BackupPlugin extends Plugin {
2004
1981
  async backup(type = "full", options = {}) {
2005
1982
  const backupId = this._generateBackupId(type);
2006
1983
  const startTime = Date.now();
1984
+ if (this.activeBackups.has(backupId)) {
1985
+ throw new Error(`Backup '${backupId}' is already in progress`);
1986
+ }
2007
1987
  try {
2008
1988
  this.activeBackups.add(backupId);
2009
1989
  if (this.config.onBackupStart) {
@@ -2019,16 +1999,9 @@ class BackupPlugin extends Plugin {
2019
1999
  if (exportedFiles.length === 0) {
2020
2000
  throw new Error("No resources were exported for backup");
2021
2001
  }
2022
- let finalPath;
2023
- let totalSize = 0;
2024
- if (this.config.compression !== "none") {
2025
- finalPath = path.join(tempBackupDir, `${backupId}.tar.gz`);
2026
- totalSize = await this._createCompressedArchive(exportedFiles, finalPath);
2027
- } else {
2028
- finalPath = exportedFiles[0];
2029
- const [statOk, , stats] = await tryFn(() => promises.stat(finalPath));
2030
- totalSize = statOk ? stats.size : 0;
2031
- }
2002
+ const archiveExtension = this.config.compression !== "none" ? ".tar.gz" : ".json";
2003
+ const finalPath = path.join(tempBackupDir, `${backupId}${archiveExtension}`);
2004
+ const totalSize = await this._createArchive(exportedFiles, finalPath, this.config.compression);
2032
2005
  const checksum = await this._generateChecksum(finalPath);
2033
2006
  const uploadResult = await this.driver.upload(finalPath, backupId, manifest);
2034
2007
  if (this.config.verification) {
@@ -2137,15 +2110,35 @@ class BackupPlugin extends Plugin {
2137
2110
  for (const resourceName of resourceNames) {
2138
2111
  const resource = this.database.resources[resourceName];
2139
2112
  if (!resource) {
2140
- console.warn(`[BackupPlugin] Resource '${resourceName}' not found, skipping`);
2113
+ if (this.config.verbose) {
2114
+ console.warn(`[BackupPlugin] Resource '${resourceName}' not found, skipping`);
2115
+ }
2141
2116
  continue;
2142
2117
  }
2143
2118
  const exportPath = path.join(tempDir, `${resourceName}.json`);
2144
2119
  let records;
2145
2120
  if (type === "incremental") {
2146
- const yesterday = new Date(Date.now() - 24 * 60 * 60 * 1e3);
2121
+ const [lastBackupOk, , lastBackups] = await tryFn(
2122
+ () => this.database.resource(this.config.backupMetadataResource).list({
2123
+ filter: {
2124
+ status: "completed",
2125
+ type: { $in: ["full", "incremental"] }
2126
+ },
2127
+ sort: { timestamp: -1 },
2128
+ limit: 1
2129
+ })
2130
+ );
2131
+ let sinceTimestamp;
2132
+ if (lastBackupOk && lastBackups && lastBackups.length > 0) {
2133
+ sinceTimestamp = new Date(lastBackups[0].timestamp);
2134
+ } else {
2135
+ sinceTimestamp = new Date(Date.now() - 24 * 60 * 60 * 1e3);
2136
+ }
2137
+ if (this.config.verbose) {
2138
+ console.log(`[BackupPlugin] Incremental backup for '${resourceName}' since ${sinceTimestamp.toISOString()}`);
2139
+ }
2147
2140
  records = await resource.list({
2148
- filter: { updatedAt: { ">": yesterday.toISOString() } }
2141
+ filter: { updatedAt: { ">": sinceTimestamp.toISOString() } }
2149
2142
  });
2150
2143
  } else {
2151
2144
  records = await resource.list();
@@ -2165,29 +2158,57 @@ class BackupPlugin extends Plugin {
2165
2158
  }
2166
2159
  return exportedFiles;
2167
2160
  }
2168
- async _createCompressedArchive(files, targetPath) {
2169
- const output = fs.createWriteStream(targetPath);
2170
- const gzip = zlib.createGzip({ level: 6 });
2161
+ async _createArchive(files, targetPath, compressionType) {
2162
+ const archive = {
2163
+ version: "1.0",
2164
+ created: (/* @__PURE__ */ new Date()).toISOString(),
2165
+ files: []
2166
+ };
2171
2167
  let totalSize = 0;
2172
- await promises$1.pipeline(
2173
- async function* () {
2174
- for (const filePath of files) {
2175
- const content = await promises.readFile(filePath);
2176
- totalSize += content.length;
2177
- yield content;
2168
+ for (const filePath of files) {
2169
+ const [readOk, readErr, content] = await tryFn(() => promises.readFile(filePath, "utf8"));
2170
+ if (!readOk) {
2171
+ if (this.config.verbose) {
2172
+ console.warn(`[BackupPlugin] Failed to read ${filePath}: ${readErr?.message}`);
2178
2173
  }
2179
- },
2180
- gzip,
2181
- output
2182
- );
2174
+ continue;
2175
+ }
2176
+ const fileName = path.basename(filePath);
2177
+ totalSize += content.length;
2178
+ archive.files.push({
2179
+ name: fileName,
2180
+ size: content.length,
2181
+ content
2182
+ });
2183
+ }
2184
+ const archiveJson = JSON.stringify(archive);
2185
+ if (compressionType === "none") {
2186
+ await promises.writeFile(targetPath, archiveJson, "utf8");
2187
+ } else {
2188
+ const output = fs.createWriteStream(targetPath);
2189
+ const gzip = zlib.createGzip({ level: 6 });
2190
+ await promises$1.pipeline(
2191
+ async function* () {
2192
+ yield Buffer.from(archiveJson, "utf8");
2193
+ },
2194
+ gzip,
2195
+ output
2196
+ );
2197
+ }
2183
2198
  const [statOk, , stats] = await tryFn(() => promises.stat(targetPath));
2184
2199
  return statOk ? stats.size : totalSize;
2185
2200
  }
2186
2201
  async _generateChecksum(filePath) {
2187
- const hash = crypto.createHash("sha256");
2188
- const stream = fs.createReadStream(filePath);
2189
- await promises$1.pipeline(stream, hash);
2190
- return hash.digest("hex");
2202
+ const [ok, err, result] = await tryFn(async () => {
2203
+ const hash = crypto.createHash("sha256");
2204
+ const stream = fs.createReadStream(filePath);
2205
+ await promises$1.pipeline(stream, hash);
2206
+ return hash.digest("hex");
2207
+ });
2208
+ if (!ok) {
2209
+ throw new Error(`Failed to generate checksum for ${filePath}: ${err?.message}`);
2210
+ }
2211
+ return result;
2191
2212
  }
2192
2213
  async _cleanupTempFiles(tempDir) {
2193
2214
  const [ok] = await tryFn(
@@ -2249,7 +2270,109 @@ class BackupPlugin extends Plugin {
2249
2270
  }
2250
2271
  async _restoreFromBackup(backupPath, options) {
2251
2272
  const restoredResources = [];
2252
- return restoredResources;
2273
+ try {
2274
+ let archiveData = "";
2275
+ if (this.config.compression !== "none") {
2276
+ const input = fs.createReadStream(backupPath);
2277
+ const gunzip = zlib.createGunzip();
2278
+ const chunks = [];
2279
+ await new Promise((resolve, reject) => {
2280
+ input.pipe(gunzip).on("data", (chunk) => chunks.push(chunk)).on("end", resolve).on("error", reject);
2281
+ });
2282
+ archiveData = Buffer.concat(chunks).toString("utf8");
2283
+ } else {
2284
+ archiveData = await promises.readFile(backupPath, "utf8");
2285
+ }
2286
+ let archive;
2287
+ try {
2288
+ archive = JSON.parse(archiveData);
2289
+ } catch (parseError) {
2290
+ throw new Error(`Failed to parse backup archive: ${parseError.message}`);
2291
+ }
2292
+ if (!archive || typeof archive !== "object") {
2293
+ throw new Error("Invalid backup archive: not a valid JSON object");
2294
+ }
2295
+ if (!archive.version || !archive.files) {
2296
+ throw new Error("Invalid backup archive format: missing version or files array");
2297
+ }
2298
+ if (this.config.verbose) {
2299
+ console.log(`[BackupPlugin] Restoring ${archive.files.length} files from backup`);
2300
+ }
2301
+ for (const file of archive.files) {
2302
+ try {
2303
+ const resourceData = JSON.parse(file.content);
2304
+ if (!resourceData.resourceName || !resourceData.definition) {
2305
+ if (this.config.verbose) {
2306
+ console.warn(`[BackupPlugin] Skipping invalid file: ${file.name}`);
2307
+ }
2308
+ continue;
2309
+ }
2310
+ const resourceName = resourceData.resourceName;
2311
+ if (options.resources && !options.resources.includes(resourceName)) {
2312
+ continue;
2313
+ }
2314
+ let resource = this.database.resources[resourceName];
2315
+ if (!resource) {
2316
+ if (this.config.verbose) {
2317
+ console.log(`[BackupPlugin] Creating resource '${resourceName}'`);
2318
+ }
2319
+ const [createOk, createErr] = await tryFn(
2320
+ () => this.database.createResource(resourceData.definition)
2321
+ );
2322
+ if (!createOk) {
2323
+ if (this.config.verbose) {
2324
+ console.warn(`[BackupPlugin] Failed to create resource '${resourceName}': ${createErr?.message}`);
2325
+ }
2326
+ continue;
2327
+ }
2328
+ resource = this.database.resources[resourceName];
2329
+ }
2330
+ if (resourceData.records && Array.isArray(resourceData.records)) {
2331
+ const mode = options.mode || "merge";
2332
+ if (mode === "replace") {
2333
+ const ids = await resource.listIds();
2334
+ for (const id of ids) {
2335
+ await resource.delete(id);
2336
+ }
2337
+ }
2338
+ let insertedCount = 0;
2339
+ for (const record of resourceData.records) {
2340
+ const [insertOk] = await tryFn(async () => {
2341
+ if (mode === "skip") {
2342
+ const existing = await resource.get(record.id);
2343
+ if (existing) {
2344
+ return false;
2345
+ }
2346
+ }
2347
+ await resource.insert(record);
2348
+ return true;
2349
+ });
2350
+ if (insertOk) {
2351
+ insertedCount++;
2352
+ }
2353
+ }
2354
+ restoredResources.push({
2355
+ name: resourceName,
2356
+ recordsRestored: insertedCount,
2357
+ totalRecords: resourceData.records.length
2358
+ });
2359
+ if (this.config.verbose) {
2360
+ console.log(`[BackupPlugin] Restored ${insertedCount}/${resourceData.records.length} records to '${resourceName}'`);
2361
+ }
2362
+ }
2363
+ } catch (fileError) {
2364
+ if (this.config.verbose) {
2365
+ console.warn(`[BackupPlugin] Error processing file ${file.name}: ${fileError.message}`);
2366
+ }
2367
+ }
2368
+ }
2369
+ return restoredResources;
2370
+ } catch (error) {
2371
+ if (this.config.verbose) {
2372
+ console.error(`[BackupPlugin] Error restoring backup: ${error.message}`);
2373
+ }
2374
+ throw new Error(`Failed to restore backup: ${error.message}`);
2375
+ }
2253
2376
  }
2254
2377
  /**
2255
2378
  * List available backups
@@ -2293,6 +2416,90 @@ class BackupPlugin extends Plugin {
2293
2416
  return ok ? backup : null;
2294
2417
  }
2295
2418
  async _cleanupOldBackups() {
2419
+ try {
2420
+ const [listOk, , allBackups] = await tryFn(
2421
+ () => this.database.resource(this.config.backupMetadataResource).list({
2422
+ filter: { status: "completed" },
2423
+ sort: { timestamp: -1 }
2424
+ })
2425
+ );
2426
+ if (!listOk || !allBackups || allBackups.length === 0) {
2427
+ return;
2428
+ }
2429
+ const now = Date.now();
2430
+ const msPerDay = 24 * 60 * 60 * 1e3;
2431
+ const msPerWeek = 7 * msPerDay;
2432
+ const msPerMonth = 30 * msPerDay;
2433
+ const msPerYear = 365 * msPerDay;
2434
+ const categorized = {
2435
+ daily: [],
2436
+ weekly: [],
2437
+ monthly: [],
2438
+ yearly: []
2439
+ };
2440
+ for (const backup of allBackups) {
2441
+ const age = now - backup.timestamp;
2442
+ if (age <= msPerDay * this.config.retention.daily) {
2443
+ categorized.daily.push(backup);
2444
+ } else if (age <= msPerWeek * this.config.retention.weekly) {
2445
+ categorized.weekly.push(backup);
2446
+ } else if (age <= msPerMonth * this.config.retention.monthly) {
2447
+ categorized.monthly.push(backup);
2448
+ } else if (age <= msPerYear * this.config.retention.yearly) {
2449
+ categorized.yearly.push(backup);
2450
+ }
2451
+ }
2452
+ const toKeep = /* @__PURE__ */ new Set();
2453
+ categorized.daily.forEach((b) => toKeep.add(b.id));
2454
+ const weeklyByWeek = /* @__PURE__ */ new Map();
2455
+ for (const backup of categorized.weekly) {
2456
+ const weekNum = Math.floor((now - backup.timestamp) / msPerWeek);
2457
+ if (!weeklyByWeek.has(weekNum)) {
2458
+ weeklyByWeek.set(weekNum, backup);
2459
+ toKeep.add(backup.id);
2460
+ }
2461
+ }
2462
+ const monthlyByMonth = /* @__PURE__ */ new Map();
2463
+ for (const backup of categorized.monthly) {
2464
+ const monthNum = Math.floor((now - backup.timestamp) / msPerMonth);
2465
+ if (!monthlyByMonth.has(monthNum)) {
2466
+ monthlyByMonth.set(monthNum, backup);
2467
+ toKeep.add(backup.id);
2468
+ }
2469
+ }
2470
+ const yearlyByYear = /* @__PURE__ */ new Map();
2471
+ for (const backup of categorized.yearly) {
2472
+ const yearNum = Math.floor((now - backup.timestamp) / msPerYear);
2473
+ if (!yearlyByYear.has(yearNum)) {
2474
+ yearlyByYear.set(yearNum, backup);
2475
+ toKeep.add(backup.id);
2476
+ }
2477
+ }
2478
+ const backupsToDelete = allBackups.filter((b) => !toKeep.has(b.id));
2479
+ if (backupsToDelete.length === 0) {
2480
+ return;
2481
+ }
2482
+ if (this.config.verbose) {
2483
+ console.log(`[BackupPlugin] Cleaning up ${backupsToDelete.length} old backups (keeping ${toKeep.size})`);
2484
+ }
2485
+ for (const backup of backupsToDelete) {
2486
+ try {
2487
+ await this.driver.delete(backup.id, backup.driverInfo);
2488
+ await this.database.resource(this.config.backupMetadataResource).delete(backup.id);
2489
+ if (this.config.verbose) {
2490
+ console.log(`[BackupPlugin] Deleted old backup: ${backup.id}`);
2491
+ }
2492
+ } catch (deleteError) {
2493
+ if (this.config.verbose) {
2494
+ console.warn(`[BackupPlugin] Failed to delete backup ${backup.id}: ${deleteError.message}`);
2495
+ }
2496
+ }
2497
+ }
2498
+ } catch (error) {
2499
+ if (this.config.verbose) {
2500
+ console.warn(`[BackupPlugin] Error during cleanup: ${error.message}`);
2501
+ }
2502
+ }
2296
2503
  }
2297
2504
  async _executeHook(hook, ...args) {
2298
2505
  if (typeof hook === "function") {
@@ -3582,81 +3789,58 @@ class PartitionAwareFilesystemCache extends FilesystemCache {
3582
3789
  class CachePlugin extends Plugin {
3583
3790
  constructor(options = {}) {
3584
3791
  super(options);
3585
- this.driverName = options.driver || "s3";
3586
- this.ttl = options.ttl;
3587
- this.maxSize = options.maxSize;
3588
- this.config = options.config || {};
3589
- this.includePartitions = options.includePartitions !== false;
3590
- this.partitionStrategy = options.partitionStrategy || "hierarchical";
3591
- this.partitionAware = options.partitionAware !== false;
3592
- this.trackUsage = options.trackUsage !== false;
3593
- this.preloadRelated = options.preloadRelated !== false;
3594
- this.legacyConfig = {
3595
- memoryOptions: options.memoryOptions,
3596
- filesystemOptions: options.filesystemOptions,
3597
- s3Options: options.s3Options,
3598
- driver: options.driver
3792
+ this.config = {
3793
+ // Driver configuration
3794
+ driver: options.driver || "s3",
3795
+ config: {
3796
+ ttl: options.ttl,
3797
+ maxSize: options.maxSize,
3798
+ ...options.config
3799
+ // Driver-specific config (can override ttl/maxSize)
3800
+ },
3801
+ // Resource filtering
3802
+ include: options.include || null,
3803
+ // Array of resource names to cache (null = all)
3804
+ exclude: options.exclude || [],
3805
+ // Array of resource names to exclude
3806
+ // Partition settings
3807
+ includePartitions: options.includePartitions !== false,
3808
+ partitionStrategy: options.partitionStrategy || "hierarchical",
3809
+ partitionAware: options.partitionAware !== false,
3810
+ trackUsage: options.trackUsage !== false,
3811
+ preloadRelated: options.preloadRelated !== false,
3812
+ // Retry configuration
3813
+ retryAttempts: options.retryAttempts || 3,
3814
+ retryDelay: options.retryDelay || 100,
3815
+ // ms
3816
+ // Logging
3817
+ verbose: options.verbose || false
3599
3818
  };
3600
3819
  }
3601
3820
  async setup(database) {
3602
3821
  await super.setup(database);
3603
3822
  }
3604
3823
  async onSetup() {
3605
- if (this.driverName && typeof this.driverName === "object") {
3606
- this.driver = this.driverName;
3607
- } else if (this.driverName === "memory") {
3608
- const driverConfig = {
3609
- ...this.legacyConfig.memoryOptions,
3610
- // Legacy support (lowest priority)
3611
- ...this.config
3612
- // New config format (medium priority)
3613
- };
3614
- if (this.ttl !== void 0) {
3615
- driverConfig.ttl = this.ttl;
3616
- }
3617
- if (this.maxSize !== void 0) {
3618
- driverConfig.maxSize = this.maxSize;
3619
- }
3620
- this.driver = new MemoryCache(driverConfig);
3621
- } else if (this.driverName === "filesystem") {
3622
- const driverConfig = {
3623
- ...this.legacyConfig.filesystemOptions,
3624
- // Legacy support (lowest priority)
3625
- ...this.config
3626
- // New config format (medium priority)
3627
- };
3628
- if (this.ttl !== void 0) {
3629
- driverConfig.ttl = this.ttl;
3630
- }
3631
- if (this.maxSize !== void 0) {
3632
- driverConfig.maxSize = this.maxSize;
3633
- }
3634
- if (this.partitionAware) {
3824
+ if (this.config.driver && typeof this.config.driver === "object") {
3825
+ this.driver = this.config.driver;
3826
+ } else if (this.config.driver === "memory") {
3827
+ this.driver = new MemoryCache(this.config.config);
3828
+ } else if (this.config.driver === "filesystem") {
3829
+ if (this.config.partitionAware) {
3635
3830
  this.driver = new PartitionAwareFilesystemCache({
3636
- partitionStrategy: this.partitionStrategy,
3637
- trackUsage: this.trackUsage,
3638
- preloadRelated: this.preloadRelated,
3639
- ...driverConfig
3831
+ partitionStrategy: this.config.partitionStrategy,
3832
+ trackUsage: this.config.trackUsage,
3833
+ preloadRelated: this.config.preloadRelated,
3834
+ ...this.config.config
3640
3835
  });
3641
3836
  } else {
3642
- this.driver = new FilesystemCache(driverConfig);
3837
+ this.driver = new FilesystemCache(this.config.config);
3643
3838
  }
3644
3839
  } else {
3645
- const driverConfig = {
3840
+ this.driver = new S3Cache({
3646
3841
  client: this.database.client,
3647
- // Required for S3Cache
3648
- ...this.legacyConfig.s3Options,
3649
- // Legacy support (lowest priority)
3650
- ...this.config
3651
- // New config format (medium priority)
3652
- };
3653
- if (this.ttl !== void 0) {
3654
- driverConfig.ttl = this.ttl;
3655
- }
3656
- if (this.maxSize !== void 0) {
3657
- driverConfig.maxSize = this.maxSize;
3658
- }
3659
- this.driver = new S3Cache(driverConfig);
3842
+ ...this.config.config
3843
+ });
3660
3844
  }
3661
3845
  this.installDatabaseHooks();
3662
3846
  this.installResourceHooks();
@@ -3666,7 +3850,9 @@ class CachePlugin extends Plugin {
3666
3850
  */
3667
3851
  installDatabaseHooks() {
3668
3852
  this.database.addHook("afterCreateResource", async ({ resource }) => {
3669
- this.installResourceHooksForResource(resource);
3853
+ if (this.shouldCacheResource(resource.name)) {
3854
+ this.installResourceHooksForResource(resource);
3855
+ }
3670
3856
  });
3671
3857
  }
3672
3858
  async onStart() {
@@ -3676,9 +3862,24 @@ class CachePlugin extends Plugin {
3676
3862
  // Remove the old installDatabaseProxy method
3677
3863
  installResourceHooks() {
3678
3864
  for (const resource of Object.values(this.database.resources)) {
3865
+ if (!this.shouldCacheResource(resource.name)) {
3866
+ continue;
3867
+ }
3679
3868
  this.installResourceHooksForResource(resource);
3680
3869
  }
3681
3870
  }
3871
+ shouldCacheResource(resourceName) {
3872
+ if (resourceName.startsWith("plg_") && !this.config.include) {
3873
+ return false;
3874
+ }
3875
+ if (this.config.exclude.includes(resourceName)) {
3876
+ return false;
3877
+ }
3878
+ if (this.config.include && !this.config.include.includes(resourceName)) {
3879
+ return false;
3880
+ }
3881
+ return true;
3882
+ }
3682
3883
  installResourceHooksForResource(resource) {
3683
3884
  if (!this.driver) return;
3684
3885
  Object.defineProperty(resource, "cache", {
@@ -3827,37 +4028,74 @@ class CachePlugin extends Plugin {
3827
4028
  if (data && data.id) {
3828
4029
  const itemSpecificMethods = ["get", "exists", "content", "hasContent"];
3829
4030
  for (const method of itemSpecificMethods) {
3830
- try {
3831
- const specificKey = await this.generateCacheKey(resource, method, { id: data.id });
3832
- await resource.cache.clear(specificKey.replace(".json.gz", ""));
3833
- } catch (error) {
4031
+ const specificKey = await this.generateCacheKey(resource, method, { id: data.id });
4032
+ const [ok2, err2] = await this.clearCacheWithRetry(resource.cache, specificKey);
4033
+ if (!ok2) {
4034
+ this.emit("cache_clear_error", {
4035
+ resource: resource.name,
4036
+ method,
4037
+ id: data.id,
4038
+ error: err2.message
4039
+ });
4040
+ if (this.config.verbose) {
4041
+ console.warn(`[CachePlugin] Failed to clear ${method} cache for ${resource.name}:${data.id}:`, err2.message);
4042
+ }
3834
4043
  }
3835
4044
  }
3836
4045
  if (this.config.includePartitions === true && resource.config?.partitions && Object.keys(resource.config.partitions).length > 0) {
3837
4046
  const partitionValues = this.getPartitionValues(data, resource);
3838
4047
  for (const [partitionName, values] of Object.entries(partitionValues)) {
3839
4048
  if (values && Object.keys(values).length > 0 && Object.values(values).some((v) => v !== null && v !== void 0)) {
3840
- try {
3841
- const partitionKeyPrefix = path.join(keyPrefix, `partition=${partitionName}`);
3842
- await resource.cache.clear(partitionKeyPrefix);
3843
- } catch (error) {
4049
+ const partitionKeyPrefix = path.join(keyPrefix, `partition=${partitionName}`);
4050
+ const [ok2, err2] = await this.clearCacheWithRetry(resource.cache, partitionKeyPrefix);
4051
+ if (!ok2) {
4052
+ this.emit("cache_clear_error", {
4053
+ resource: resource.name,
4054
+ partition: partitionName,
4055
+ error: err2.message
4056
+ });
4057
+ if (this.config.verbose) {
4058
+ console.warn(`[CachePlugin] Failed to clear partition cache for ${resource.name}/${partitionName}:`, err2.message);
4059
+ }
3844
4060
  }
3845
4061
  }
3846
4062
  }
3847
4063
  }
3848
4064
  }
3849
- try {
3850
- await resource.cache.clear(keyPrefix);
3851
- } catch (error) {
4065
+ const [ok, err] = await this.clearCacheWithRetry(resource.cache, keyPrefix);
4066
+ if (!ok) {
4067
+ this.emit("cache_clear_error", {
4068
+ resource: resource.name,
4069
+ type: "broad",
4070
+ error: err.message
4071
+ });
4072
+ if (this.config.verbose) {
4073
+ console.warn(`[CachePlugin] Failed to clear broad cache for ${resource.name}, trying specific methods:`, err.message);
4074
+ }
3852
4075
  const aggregateMethods = ["count", "list", "listIds", "getAll", "page", "query"];
3853
4076
  for (const method of aggregateMethods) {
3854
- try {
3855
- await resource.cache.clear(`${keyPrefix}/action=${method}`);
3856
- await resource.cache.clear(`resource=${resource.name}/action=${method}`);
3857
- } catch (methodError) {
3858
- }
4077
+ await this.clearCacheWithRetry(resource.cache, `${keyPrefix}/action=${method}`);
4078
+ await this.clearCacheWithRetry(resource.cache, `resource=${resource.name}/action=${method}`);
4079
+ }
4080
+ }
4081
+ }
4082
+ async clearCacheWithRetry(cache, key) {
4083
+ let lastError;
4084
+ for (let attempt = 0; attempt < this.config.retryAttempts; attempt++) {
4085
+ const [ok, err] = await tryFn(() => cache.clear(key));
4086
+ if (ok) {
4087
+ return [true, null];
4088
+ }
4089
+ lastError = err;
4090
+ if (err.name === "NoSuchKey" || err.code === "NoSuchKey") {
4091
+ return [true, null];
4092
+ }
4093
+ if (attempt < this.config.retryAttempts - 1) {
4094
+ const delay = this.config.retryDelay * Math.pow(2, attempt);
4095
+ await new Promise((resolve) => setTimeout(resolve, delay));
3859
4096
  }
3860
4097
  }
4098
+ return [false, lastError];
3861
4099
  }
3862
4100
  async generateCacheKey(resource, action, params = {}, partition = null, partitionValues = null) {
3863
4101
  const keyParts = [
@@ -3873,14 +4111,14 @@ class CachePlugin extends Plugin {
3873
4111
  }
3874
4112
  }
3875
4113
  if (Object.keys(params).length > 0) {
3876
- const paramsHash = await this.hashParams(params);
4114
+ const paramsHash = this.hashParams(params);
3877
4115
  keyParts.push(paramsHash);
3878
4116
  }
3879
4117
  return path.join(...keyParts) + ".json.gz";
3880
4118
  }
3881
- async hashParams(params) {
3882
- const sortedParams = Object.keys(params).sort().map((key) => `${key}:${JSON.stringify(params[key])}`).join("|") || "empty";
3883
- return await sha256(sortedParams);
4119
+ hashParams(params) {
4120
+ const serialized = jsonStableStringify(params) || "empty";
4121
+ return crypto.createHash("md5").update(serialized).digest("hex").substring(0, 16);
3884
4122
  }
3885
4123
  // Utility methods
3886
4124
  async getCacheStats() {
@@ -3905,50 +4143,48 @@ class CachePlugin extends Plugin {
3905
4143
  if (!resource) {
3906
4144
  throw new Error(`Resource '${resourceName}' not found`);
3907
4145
  }
3908
- const { includePartitions = true } = options;
4146
+ const { includePartitions = true, sampleSize = 100 } = options;
3909
4147
  if (this.driver instanceof PartitionAwareFilesystemCache && resource.warmPartitionCache) {
3910
4148
  const partitionNames = resource.config.partitions ? Object.keys(resource.config.partitions) : [];
3911
4149
  return await resource.warmPartitionCache(partitionNames, options);
3912
4150
  }
3913
- await resource.getAll();
3914
- if (includePartitions && resource.config.partitions) {
4151
+ let offset = 0;
4152
+ const pageSize = 100;
4153
+ const sampledRecords = [];
4154
+ while (sampledRecords.length < sampleSize) {
4155
+ const [ok, err, pageResult] = await tryFn(() => resource.page({ offset, size: pageSize }));
4156
+ if (!ok || !pageResult) {
4157
+ break;
4158
+ }
4159
+ const pageItems = Array.isArray(pageResult) ? pageResult : pageResult.items || [];
4160
+ if (pageItems.length === 0) {
4161
+ break;
4162
+ }
4163
+ sampledRecords.push(...pageItems);
4164
+ offset += pageSize;
4165
+ }
4166
+ if (includePartitions && resource.config.partitions && sampledRecords.length > 0) {
3915
4167
  for (const [partitionName, partitionDef] of Object.entries(resource.config.partitions)) {
3916
4168
  if (partitionDef.fields) {
3917
- const allRecords = await resource.getAll();
3918
- const recordsArray = Array.isArray(allRecords) ? allRecords : [];
3919
- const partitionValues = /* @__PURE__ */ new Set();
3920
- for (const record of recordsArray.slice(0, 10)) {
4169
+ const partitionValuesSet = /* @__PURE__ */ new Set();
4170
+ for (const record of sampledRecords) {
3921
4171
  const values = this.getPartitionValues(record, resource);
3922
4172
  if (values[partitionName]) {
3923
- partitionValues.add(JSON.stringify(values[partitionName]));
4173
+ partitionValuesSet.add(JSON.stringify(values[partitionName]));
3924
4174
  }
3925
4175
  }
3926
- for (const partitionValueStr of partitionValues) {
3927
- const partitionValues2 = JSON.parse(partitionValueStr);
3928
- await resource.list({ partition: partitionName, partitionValues: partitionValues2 });
4176
+ for (const partitionValueStr of partitionValuesSet) {
4177
+ const partitionValues = JSON.parse(partitionValueStr);
4178
+ await tryFn(() => resource.list({ partition: partitionName, partitionValues }));
3929
4179
  }
3930
4180
  }
3931
4181
  }
3932
4182
  }
3933
- }
3934
- // Partition-specific methods
3935
- async getPartitionCacheStats(resourceName, partition = null) {
3936
- if (!(this.driver instanceof PartitionAwareFilesystemCache)) {
3937
- throw new Error("Partition cache statistics are only available with PartitionAwareFilesystemCache");
3938
- }
3939
- return await this.driver.getPartitionStats(resourceName, partition);
3940
- }
3941
- async getCacheRecommendations(resourceName) {
3942
- if (!(this.driver instanceof PartitionAwareFilesystemCache)) {
3943
- throw new Error("Cache recommendations are only available with PartitionAwareFilesystemCache");
3944
- }
3945
- return await this.driver.getCacheRecommendations(resourceName);
3946
- }
3947
- async clearPartitionCache(resourceName, partition, partitionValues = {}) {
3948
- if (!(this.driver instanceof PartitionAwareFilesystemCache)) {
3949
- throw new Error("Partition cache clearing is only available with PartitionAwareFilesystemCache");
3950
- }
3951
- return await this.driver.clearPartition(resourceName, partition, partitionValues);
4183
+ return {
4184
+ resourceName,
4185
+ recordsSampled: sampledRecords.length,
4186
+ partitionsWarmed: includePartitions && resource.config.partitions ? Object.keys(resource.config.partitions).length : 0
4187
+ };
3952
4188
  }
3953
4189
  async analyzeCacheUsage() {
3954
4190
  if (!(this.driver instanceof PartitionAwareFilesystemCache)) {
@@ -3965,6 +4201,9 @@ class CachePlugin extends Plugin {
3965
4201
  }
3966
4202
  };
3967
4203
  for (const [resourceName, resource] of Object.entries(this.database.resources)) {
4204
+ if (!this.shouldCacheResource(resourceName)) {
4205
+ continue;
4206
+ }
3968
4207
  try {
3969
4208
  analysis.resourceStats[resourceName] = await this.driver.getPartitionStats(resourceName);
3970
4209
  analysis.recommendations[resourceName] = await this.driver.getCacheRecommendations(resourceName);
@@ -4065,13 +4304,12 @@ class EventualConsistencyPlugin extends Plugin {
4065
4304
  if (!options.field) {
4066
4305
  throw new Error("EventualConsistencyPlugin requires 'field' option");
4067
4306
  }
4307
+ const detectedTimezone = this._detectTimezone();
4068
4308
  this.config = {
4069
4309
  resource: options.resource,
4070
4310
  field: options.field,
4071
4311
  cohort: {
4072
- interval: options.cohort?.interval || "24h",
4073
- timezone: options.cohort?.timezone || "UTC",
4074
- ...options.cohort
4312
+ timezone: options.cohort?.timezone || detectedTimezone
4075
4313
  },
4076
4314
  reducer: options.reducer || ((transactions) => {
4077
4315
  let baseValue = 0;
@@ -4086,19 +4324,42 @@ class EventualConsistencyPlugin extends Plugin {
4086
4324
  }
4087
4325
  return baseValue;
4088
4326
  }),
4089
- consolidationInterval: options.consolidationInterval || 36e5,
4090
- // 1 hour default
4327
+ consolidationInterval: options.consolidationInterval ?? 300,
4328
+ // 5 minutes (in seconds)
4329
+ consolidationConcurrency: options.consolidationConcurrency || 5,
4330
+ consolidationWindow: options.consolidationWindow || 24,
4331
+ // Hours to look back for pending transactions (watermark)
4091
4332
  autoConsolidate: options.autoConsolidate !== false,
4333
+ lateArrivalStrategy: options.lateArrivalStrategy || "warn",
4334
+ // 'ignore', 'warn', 'process'
4092
4335
  batchTransactions: options.batchTransactions || false,
4336
+ // CAUTION: Not safe in distributed environments! Loses data on container crash
4093
4337
  batchSize: options.batchSize || 100,
4094
4338
  mode: options.mode || "async",
4095
4339
  // 'async' or 'sync'
4096
- ...options
4340
+ lockTimeout: options.lockTimeout || 300,
4341
+ // 5 minutes (in seconds, configurable)
4342
+ transactionRetention: options.transactionRetention || 30,
4343
+ // Days to keep applied transactions
4344
+ gcInterval: options.gcInterval || 86400,
4345
+ // 24 hours (in seconds)
4346
+ verbose: options.verbose || false
4097
4347
  };
4098
4348
  this.transactionResource = null;
4099
4349
  this.targetResource = null;
4100
4350
  this.consolidationTimer = null;
4351
+ this.gcTimer = null;
4101
4352
  this.pendingTransactions = /* @__PURE__ */ new Map();
4353
+ if (this.config.batchTransactions && !this.config.verbose) {
4354
+ console.warn(
4355
+ `[EventualConsistency] WARNING: batchTransactions is enabled. This stores transactions in memory and will lose data if container crashes. Not recommended for distributed/production environments. Set verbose: true to suppress this warning.`
4356
+ );
4357
+ }
4358
+ if (this.config.verbose && !options.cohort?.timezone) {
4359
+ console.log(
4360
+ `[EventualConsistency] Auto-detected timezone: ${this.config.cohort.timezone} (from ${process.env.TZ ? "TZ env var" : "system Intl API"})`
4361
+ );
4362
+ }
4102
4363
  }
4103
4364
  async onSetup() {
4104
4365
  this.targetResource = this.database.resources[this.config.resource];
@@ -4135,7 +4396,9 @@ class EventualConsistencyPlugin extends Plugin {
4135
4396
  // 'set', 'add', or 'sub'
4136
4397
  timestamp: "string|required",
4137
4398
  cohortDate: "string|required",
4138
- // For partitioning
4399
+ // For daily partitioning
4400
+ cohortHour: "string|required",
4401
+ // For hourly partitioning
4139
4402
  cohortMonth: "string|optional",
4140
4403
  // For monthly partitioning
4141
4404
  source: "string|optional",
@@ -4153,10 +4416,28 @@ class EventualConsistencyPlugin extends Plugin {
4153
4416
  throw new Error(`Failed to create transaction resource: ${err?.message}`);
4154
4417
  }
4155
4418
  this.transactionResource = ok ? transactionResource : this.database.resources[transactionResourceName];
4419
+ const lockResourceName = `${this.config.resource}_consolidation_locks_${this.config.field}`;
4420
+ const [lockOk, lockErr, lockResource] = await tryFn(
4421
+ () => this.database.createResource({
4422
+ name: lockResourceName,
4423
+ attributes: {
4424
+ id: "string|required",
4425
+ lockedAt: "number|required",
4426
+ workerId: "string|optional"
4427
+ },
4428
+ behavior: "body-only",
4429
+ timestamps: false
4430
+ })
4431
+ );
4432
+ if (!lockOk && !this.database.resources[lockResourceName]) {
4433
+ throw new Error(`Failed to create lock resource: ${lockErr?.message}`);
4434
+ }
4435
+ this.lockResource = lockOk ? lockResource : this.database.resources[lockResourceName];
4156
4436
  this.addHelperMethods();
4157
4437
  if (this.config.autoConsolidate) {
4158
4438
  this.startConsolidationTimer();
4159
4439
  }
4440
+ this.startGarbageCollectionTimer();
4160
4441
  }
4161
4442
  async onStart() {
4162
4443
  if (this.deferredSetup) {
@@ -4173,6 +4454,10 @@ class EventualConsistencyPlugin extends Plugin {
4173
4454
  clearInterval(this.consolidationTimer);
4174
4455
  this.consolidationTimer = null;
4175
4456
  }
4457
+ if (this.gcTimer) {
4458
+ clearInterval(this.gcTimer);
4459
+ this.gcTimer = null;
4460
+ }
4176
4461
  await this.flushPendingTransactions();
4177
4462
  this.emit("eventual-consistency.stopped", {
4178
4463
  resource: this.config.resource,
@@ -4181,6 +4466,11 @@ class EventualConsistencyPlugin extends Plugin {
4181
4466
  }
4182
4467
  createPartitionConfig() {
4183
4468
  const partitions = {
4469
+ byHour: {
4470
+ fields: {
4471
+ cohortHour: "string"
4472
+ }
4473
+ },
4184
4474
  byDay: {
4185
4475
  fields: {
4186
4476
  cohortDate: "string"
@@ -4194,6 +4484,65 @@ class EventualConsistencyPlugin extends Plugin {
4194
4484
  };
4195
4485
  return partitions;
4196
4486
  }
4487
+ /**
4488
+ * Auto-detect timezone from environment or system
4489
+ * @private
4490
+ */
4491
+ _detectTimezone() {
4492
+ if (process.env.TZ) {
4493
+ return process.env.TZ;
4494
+ }
4495
+ try {
4496
+ const systemTimezone = Intl.DateTimeFormat().resolvedOptions().timeZone;
4497
+ if (systemTimezone) {
4498
+ return systemTimezone;
4499
+ }
4500
+ } catch (err) {
4501
+ }
4502
+ return "UTC";
4503
+ }
4504
+ /**
4505
+ * Helper method to resolve field and plugin from arguments
4506
+ * Supports both single-field (field, value) and multi-field (field, value) signatures
4507
+ * @private
4508
+ */
4509
+ _resolveFieldAndPlugin(resource, fieldOrValue, value) {
4510
+ const hasMultipleFields = Object.keys(resource._eventualConsistencyPlugins).length > 1;
4511
+ if (hasMultipleFields && value === void 0) {
4512
+ throw new Error(`Multiple fields have eventual consistency. Please specify the field explicitly.`);
4513
+ }
4514
+ const field = value !== void 0 ? fieldOrValue : this.config.field;
4515
+ const actualValue = value !== void 0 ? value : fieldOrValue;
4516
+ const fieldPlugin = resource._eventualConsistencyPlugins[field];
4517
+ if (!fieldPlugin) {
4518
+ throw new Error(`No eventual consistency plugin found for field "${field}"`);
4519
+ }
4520
+ return { field, value: actualValue, plugin: fieldPlugin };
4521
+ }
4522
+ /**
4523
+ * Helper method to perform atomic consolidation in sync mode
4524
+ * @private
4525
+ */
4526
+ async _syncModeConsolidate(id, field) {
4527
+ const consolidatedValue = await this.consolidateRecord(id);
4528
+ await this.targetResource.update(id, {
4529
+ [field]: consolidatedValue
4530
+ });
4531
+ return consolidatedValue;
4532
+ }
4533
+ /**
4534
+ * Create synthetic 'set' transaction from current value
4535
+ * @private
4536
+ */
4537
+ _createSyntheticSetTransaction(currentValue) {
4538
+ return {
4539
+ id: "__synthetic__",
4540
+ operation: "set",
4541
+ value: currentValue,
4542
+ timestamp: (/* @__PURE__ */ new Date(0)).toISOString(),
4543
+ synthetic: true
4544
+ };
4545
+ }
4197
4546
  addHelperMethods() {
4198
4547
  const resource = this.targetResource;
4199
4548
  const defaultField = this.config.field;
@@ -4203,16 +4552,7 @@ class EventualConsistencyPlugin extends Plugin {
4203
4552
  }
4204
4553
  resource._eventualConsistencyPlugins[defaultField] = plugin;
4205
4554
  resource.set = async (id, fieldOrValue, value) => {
4206
- const hasMultipleFields = Object.keys(resource._eventualConsistencyPlugins).length > 1;
4207
- if (hasMultipleFields && value === void 0) {
4208
- throw new Error(`Multiple fields have eventual consistency. Please specify the field: set(id, field, value)`);
4209
- }
4210
- const field = value !== void 0 ? fieldOrValue : defaultField;
4211
- const actualValue = value !== void 0 ? value : fieldOrValue;
4212
- const fieldPlugin = resource._eventualConsistencyPlugins[field];
4213
- if (!fieldPlugin) {
4214
- throw new Error(`No eventual consistency plugin found for field "${field}"`);
4215
- }
4555
+ const { field, value: actualValue, plugin: fieldPlugin } = plugin._resolveFieldAndPlugin(resource, fieldOrValue, value);
4216
4556
  await fieldPlugin.createTransaction({
4217
4557
  originalId: id,
4218
4558
  operation: "set",
@@ -4220,25 +4560,12 @@ class EventualConsistencyPlugin extends Plugin {
4220
4560
  source: "set"
4221
4561
  });
4222
4562
  if (fieldPlugin.config.mode === "sync") {
4223
- const consolidatedValue = await fieldPlugin.consolidateRecord(id);
4224
- await resource.update(id, {
4225
- [field]: consolidatedValue
4226
- });
4227
- return consolidatedValue;
4563
+ return await fieldPlugin._syncModeConsolidate(id, field);
4228
4564
  }
4229
4565
  return actualValue;
4230
4566
  };
4231
4567
  resource.add = async (id, fieldOrAmount, amount) => {
4232
- const hasMultipleFields = Object.keys(resource._eventualConsistencyPlugins).length > 1;
4233
- if (hasMultipleFields && amount === void 0) {
4234
- throw new Error(`Multiple fields have eventual consistency. Please specify the field: add(id, field, amount)`);
4235
- }
4236
- const field = amount !== void 0 ? fieldOrAmount : defaultField;
4237
- const actualAmount = amount !== void 0 ? amount : fieldOrAmount;
4238
- const fieldPlugin = resource._eventualConsistencyPlugins[field];
4239
- if (!fieldPlugin) {
4240
- throw new Error(`No eventual consistency plugin found for field "${field}"`);
4241
- }
4568
+ const { field, value: actualAmount, plugin: fieldPlugin } = plugin._resolveFieldAndPlugin(resource, fieldOrAmount, amount);
4242
4569
  await fieldPlugin.createTransaction({
4243
4570
  originalId: id,
4244
4571
  operation: "add",
@@ -4246,26 +4573,13 @@ class EventualConsistencyPlugin extends Plugin {
4246
4573
  source: "add"
4247
4574
  });
4248
4575
  if (fieldPlugin.config.mode === "sync") {
4249
- const consolidatedValue = await fieldPlugin.consolidateRecord(id);
4250
- await resource.update(id, {
4251
- [field]: consolidatedValue
4252
- });
4253
- return consolidatedValue;
4576
+ return await fieldPlugin._syncModeConsolidate(id, field);
4254
4577
  }
4255
4578
  const currentValue = await fieldPlugin.getConsolidatedValue(id);
4256
4579
  return currentValue + actualAmount;
4257
4580
  };
4258
4581
  resource.sub = async (id, fieldOrAmount, amount) => {
4259
- const hasMultipleFields = Object.keys(resource._eventualConsistencyPlugins).length > 1;
4260
- if (hasMultipleFields && amount === void 0) {
4261
- throw new Error(`Multiple fields have eventual consistency. Please specify the field: sub(id, field, amount)`);
4262
- }
4263
- const field = amount !== void 0 ? fieldOrAmount : defaultField;
4264
- const actualAmount = amount !== void 0 ? amount : fieldOrAmount;
4265
- const fieldPlugin = resource._eventualConsistencyPlugins[field];
4266
- if (!fieldPlugin) {
4267
- throw new Error(`No eventual consistency plugin found for field "${field}"`);
4268
- }
4582
+ const { field, value: actualAmount, plugin: fieldPlugin } = plugin._resolveFieldAndPlugin(resource, fieldOrAmount, amount);
4269
4583
  await fieldPlugin.createTransaction({
4270
4584
  originalId: id,
4271
4585
  operation: "sub",
@@ -4273,11 +4587,7 @@ class EventualConsistencyPlugin extends Plugin {
4273
4587
  source: "sub"
4274
4588
  });
4275
4589
  if (fieldPlugin.config.mode === "sync") {
4276
- const consolidatedValue = await fieldPlugin.consolidateRecord(id);
4277
- await resource.update(id, {
4278
- [field]: consolidatedValue
4279
- });
4280
- return consolidatedValue;
4590
+ return await fieldPlugin._syncModeConsolidate(id, field);
4281
4591
  }
4282
4592
  const currentValue = await fieldPlugin.getConsolidatedValue(id);
4283
4593
  return currentValue - actualAmount;
@@ -4307,14 +4617,34 @@ class EventualConsistencyPlugin extends Plugin {
4307
4617
  async createTransaction(data) {
4308
4618
  const now = /* @__PURE__ */ new Date();
4309
4619
  const cohortInfo = this.getCohortInfo(now);
4620
+ const watermarkMs = this.config.consolidationWindow * 60 * 60 * 1e3;
4621
+ const watermarkTime = now.getTime() - watermarkMs;
4622
+ const cohortHourDate = /* @__PURE__ */ new Date(cohortInfo.hour + ":00:00Z");
4623
+ if (cohortHourDate.getTime() < watermarkTime) {
4624
+ const hoursLate = Math.floor((now.getTime() - cohortHourDate.getTime()) / (60 * 60 * 1e3));
4625
+ if (this.config.lateArrivalStrategy === "ignore") {
4626
+ if (this.config.verbose) {
4627
+ console.warn(
4628
+ `[EventualConsistency] Late arrival ignored: transaction for ${cohortInfo.hour} is ${hoursLate}h late (watermark: ${this.config.consolidationWindow}h)`
4629
+ );
4630
+ }
4631
+ return null;
4632
+ } else if (this.config.lateArrivalStrategy === "warn") {
4633
+ console.warn(
4634
+ `[EventualConsistency] Late arrival detected: transaction for ${cohortInfo.hour} is ${hoursLate}h late (watermark: ${this.config.consolidationWindow}h). Processing anyway, but consolidation may not pick it up.`
4635
+ );
4636
+ }
4637
+ }
4310
4638
  const transaction = {
4311
- id: `txn-${Date.now()}-${Math.random().toString(36).substring(2, 11)}`,
4639
+ id: idGenerator(),
4640
+ // Use nanoid for guaranteed uniqueness
4312
4641
  originalId: data.originalId,
4313
4642
  field: this.config.field,
4314
4643
  value: data.value || 0,
4315
4644
  operation: data.operation || "set",
4316
4645
  timestamp: now.toISOString(),
4317
4646
  cohortDate: cohortInfo.date,
4647
+ cohortHour: cohortInfo.hour,
4318
4648
  cohortMonth: cohortInfo.month,
4319
4649
  source: data.source || "unknown",
4320
4650
  applied: false
@@ -4332,9 +4662,16 @@ class EventualConsistencyPlugin extends Plugin {
4332
4662
  async flushPendingTransactions() {
4333
4663
  if (this.pendingTransactions.size === 0) return;
4334
4664
  const transactions = Array.from(this.pendingTransactions.values());
4335
- this.pendingTransactions.clear();
4336
- for (const transaction of transactions) {
4337
- await this.transactionResource.insert(transaction);
4665
+ try {
4666
+ await Promise.all(
4667
+ transactions.map(
4668
+ (transaction) => this.transactionResource.insert(transaction)
4669
+ )
4670
+ );
4671
+ this.pendingTransactions.clear();
4672
+ } catch (error) {
4673
+ console.error("Failed to flush pending transactions:", error);
4674
+ throw error;
4338
4675
  }
4339
4676
  }
4340
4677
  getCohortInfo(date) {
@@ -4344,53 +4681,90 @@ class EventualConsistencyPlugin extends Plugin {
4344
4681
  const year = localDate.getFullYear();
4345
4682
  const month = String(localDate.getMonth() + 1).padStart(2, "0");
4346
4683
  const day = String(localDate.getDate()).padStart(2, "0");
4684
+ const hour = String(localDate.getHours()).padStart(2, "0");
4347
4685
  return {
4348
4686
  date: `${year}-${month}-${day}`,
4687
+ hour: `${year}-${month}-${day}T${hour}`,
4688
+ // ISO-like format for hour partition
4349
4689
  month: `${year}-${month}`
4350
4690
  };
4351
4691
  }
4352
4692
  getTimezoneOffset(timezone) {
4353
- const offsets = {
4354
- "UTC": 0,
4355
- "America/New_York": -5 * 36e5,
4356
- "America/Chicago": -6 * 36e5,
4357
- "America/Denver": -7 * 36e5,
4358
- "America/Los_Angeles": -8 * 36e5,
4359
- "America/Sao_Paulo": -3 * 36e5,
4360
- "Europe/London": 0,
4361
- "Europe/Paris": 1 * 36e5,
4362
- "Europe/Berlin": 1 * 36e5,
4363
- "Asia/Tokyo": 9 * 36e5,
4364
- "Asia/Shanghai": 8 * 36e5,
4365
- "Australia/Sydney": 10 * 36e5
4366
- };
4367
- return offsets[timezone] || 0;
4368
- }
4369
- startConsolidationTimer() {
4370
- const interval = this.config.consolidationInterval;
4371
- this.consolidationTimer = setInterval(async () => {
4372
- await this.runConsolidation();
4373
- }, interval);
4693
+ try {
4694
+ const now = /* @__PURE__ */ new Date();
4695
+ const utcDate = new Date(now.toLocaleString("en-US", { timeZone: "UTC" }));
4696
+ const tzDate = new Date(now.toLocaleString("en-US", { timeZone: timezone }));
4697
+ return tzDate.getTime() - utcDate.getTime();
4698
+ } catch (err) {
4699
+ const offsets = {
4700
+ "UTC": 0,
4701
+ "America/New_York": -5 * 36e5,
4702
+ "America/Chicago": -6 * 36e5,
4703
+ "America/Denver": -7 * 36e5,
4704
+ "America/Los_Angeles": -8 * 36e5,
4705
+ "America/Sao_Paulo": -3 * 36e5,
4706
+ "Europe/London": 0,
4707
+ "Europe/Paris": 1 * 36e5,
4708
+ "Europe/Berlin": 1 * 36e5,
4709
+ "Asia/Tokyo": 9 * 36e5,
4710
+ "Asia/Shanghai": 8 * 36e5,
4711
+ "Australia/Sydney": 10 * 36e5
4712
+ };
4713
+ if (this.config.verbose && !offsets[timezone]) {
4714
+ console.warn(
4715
+ `[EventualConsistency] Unknown timezone '${timezone}', using UTC. Consider using a valid IANA timezone (e.g., 'America/New_York')`
4716
+ );
4717
+ }
4718
+ return offsets[timezone] || 0;
4719
+ }
4720
+ }
4721
+ startConsolidationTimer() {
4722
+ const intervalMs = this.config.consolidationInterval * 1e3;
4723
+ this.consolidationTimer = setInterval(async () => {
4724
+ await this.runConsolidation();
4725
+ }, intervalMs);
4374
4726
  }
4375
4727
  async runConsolidation() {
4376
4728
  try {
4377
- const [ok, err, transactions] = await tryFn(
4378
- () => this.transactionResource.query({
4379
- applied: false
4729
+ const now = /* @__PURE__ */ new Date();
4730
+ const hoursToCheck = this.config.consolidationWindow || 24;
4731
+ const cohortHours = [];
4732
+ for (let i = 0; i < hoursToCheck; i++) {
4733
+ const date = new Date(now.getTime() - i * 60 * 60 * 1e3);
4734
+ const cohortInfo = this.getCohortInfo(date);
4735
+ cohortHours.push(cohortInfo.hour);
4736
+ }
4737
+ const transactionsByHour = await Promise.all(
4738
+ cohortHours.map(async (cohortHour) => {
4739
+ const [ok, err, txns] = await tryFn(
4740
+ () => this.transactionResource.query({
4741
+ cohortHour,
4742
+ applied: false
4743
+ })
4744
+ );
4745
+ return ok ? txns : [];
4380
4746
  })
4381
4747
  );
4382
- if (!ok) {
4383
- console.error("Consolidation failed to query transactions:", err);
4748
+ const transactions = transactionsByHour.flat();
4749
+ if (transactions.length === 0) {
4750
+ if (this.config.verbose) {
4751
+ console.log(`[EventualConsistency] No pending transactions to consolidate`);
4752
+ }
4384
4753
  return;
4385
4754
  }
4386
4755
  const uniqueIds = [...new Set(transactions.map((t) => t.originalId))];
4387
- for (const id of uniqueIds) {
4388
- await this.consolidateRecord(id);
4756
+ const { results, errors } = await promisePool.PromisePool.for(uniqueIds).withConcurrency(this.config.consolidationConcurrency).process(async (id) => {
4757
+ return await this.consolidateRecord(id);
4758
+ });
4759
+ if (errors && errors.length > 0) {
4760
+ console.error(`Consolidation completed with ${errors.length} errors:`, errors);
4389
4761
  }
4390
4762
  this.emit("eventual-consistency.consolidated", {
4391
4763
  resource: this.config.resource,
4392
4764
  field: this.config.field,
4393
- recordCount: uniqueIds.length
4765
+ recordCount: uniqueIds.length,
4766
+ successCount: results.length,
4767
+ errorCount: errors.length
4394
4768
  });
4395
4769
  } catch (error) {
4396
4770
  console.error("Consolidation error:", error);
@@ -4398,49 +4772,73 @@ class EventualConsistencyPlugin extends Plugin {
4398
4772
  }
4399
4773
  }
4400
4774
  async consolidateRecord(originalId) {
4401
- const [recordOk, recordErr, record] = await tryFn(
4402
- () => this.targetResource.get(originalId)
4403
- );
4404
- const currentValue = recordOk && record ? record[this.config.field] || 0 : 0;
4405
- const [ok, err, transactions] = await tryFn(
4406
- () => this.transactionResource.query({
4407
- originalId,
4408
- applied: false
4775
+ await this.cleanupStaleLocks();
4776
+ const lockId = `lock-${originalId}`;
4777
+ const [lockAcquired, lockErr, lock] = await tryFn(
4778
+ () => this.lockResource.insert({
4779
+ id: lockId,
4780
+ lockedAt: Date.now(),
4781
+ workerId: process.pid ? String(process.pid) : "unknown"
4409
4782
  })
4410
4783
  );
4411
- if (!ok || !transactions || transactions.length === 0) {
4412
- return currentValue;
4413
- }
4414
- transactions.sort(
4415
- (a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()
4416
- );
4417
- const hasSetOperation = transactions.some((t) => t.operation === "set");
4418
- if (currentValue !== 0 && !hasSetOperation) {
4419
- transactions.unshift({
4420
- id: "__synthetic__",
4421
- // Synthetic ID that we'll skip when marking as applied
4422
- operation: "set",
4423
- value: currentValue,
4424
- timestamp: (/* @__PURE__ */ new Date(0)).toISOString()
4425
- // Very old timestamp to ensure it's first
4426
- });
4784
+ if (!lockAcquired) {
4785
+ if (this.config.verbose) {
4786
+ console.log(`[EventualConsistency] Lock for ${originalId} already held, skipping`);
4787
+ }
4788
+ const [recordOk, recordErr, record] = await tryFn(
4789
+ () => this.targetResource.get(originalId)
4790
+ );
4791
+ return recordOk && record ? record[this.config.field] || 0 : 0;
4427
4792
  }
4428
- const consolidatedValue = this.config.reducer(transactions);
4429
- const [updateOk, updateErr] = await tryFn(
4430
- () => this.targetResource.update(originalId, {
4431
- [this.config.field]: consolidatedValue
4432
- })
4433
- );
4434
- if (updateOk) {
4435
- for (const txn of transactions) {
4436
- if (txn.id !== "__synthetic__") {
4437
- await this.transactionResource.update(txn.id, {
4438
- applied: true
4439
- });
4793
+ try {
4794
+ const [recordOk, recordErr, record] = await tryFn(
4795
+ () => this.targetResource.get(originalId)
4796
+ );
4797
+ const currentValue = recordOk && record ? record[this.config.field] || 0 : 0;
4798
+ const [ok, err, transactions] = await tryFn(
4799
+ () => this.transactionResource.query({
4800
+ originalId,
4801
+ applied: false
4802
+ })
4803
+ );
4804
+ if (!ok || !transactions || transactions.length === 0) {
4805
+ return currentValue;
4806
+ }
4807
+ transactions.sort(
4808
+ (a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()
4809
+ );
4810
+ const hasSetOperation = transactions.some((t) => t.operation === "set");
4811
+ if (currentValue !== 0 && !hasSetOperation) {
4812
+ transactions.unshift(this._createSyntheticSetTransaction(currentValue));
4813
+ }
4814
+ const consolidatedValue = this.config.reducer(transactions);
4815
+ const [updateOk, updateErr] = await tryFn(
4816
+ () => this.targetResource.update(originalId, {
4817
+ [this.config.field]: consolidatedValue
4818
+ })
4819
+ );
4820
+ if (updateOk) {
4821
+ const transactionsToUpdate = transactions.filter((txn) => txn.id !== "__synthetic__");
4822
+ const { results, errors } = await promisePool.PromisePool.for(transactionsToUpdate).withConcurrency(10).process(async (txn) => {
4823
+ const [ok2, err2] = await tryFn(
4824
+ () => this.transactionResource.update(txn.id, { applied: true })
4825
+ );
4826
+ if (!ok2 && this.config.verbose) {
4827
+ console.warn(`[EventualConsistency] Failed to mark transaction ${txn.id} as applied:`, err2?.message);
4828
+ }
4829
+ return ok2;
4830
+ });
4831
+ if (errors && errors.length > 0 && this.config.verbose) {
4832
+ console.warn(`[EventualConsistency] ${errors.length} transactions failed to mark as applied`);
4440
4833
  }
4441
4834
  }
4835
+ return consolidatedValue;
4836
+ } finally {
4837
+ const [lockReleased, lockReleaseErr] = await tryFn(() => this.lockResource.delete(lockId));
4838
+ if (!lockReleased && this.config.verbose) {
4839
+ console.warn(`[EventualConsistency] Failed to release lock ${lockId}:`, lockReleaseErr?.message);
4840
+ }
4442
4841
  }
4443
- return consolidatedValue;
4444
4842
  }
4445
4843
  async getConsolidatedValue(originalId, options = {}) {
4446
4844
  const includeApplied = options.includeApplied || false;
@@ -4454,11 +4852,11 @@ class EventualConsistencyPlugin extends Plugin {
4454
4852
  () => this.transactionResource.query(query)
4455
4853
  );
4456
4854
  if (!ok || !transactions || transactions.length === 0) {
4457
- const [recordOk, recordErr, record] = await tryFn(
4855
+ const [recordOk2, recordErr2, record2] = await tryFn(
4458
4856
  () => this.targetResource.get(originalId)
4459
4857
  );
4460
- if (recordOk && record) {
4461
- return record[this.config.field] || 0;
4858
+ if (recordOk2 && record2) {
4859
+ return record2[this.config.field] || 0;
4462
4860
  }
4463
4861
  return 0;
4464
4862
  }
@@ -4471,6 +4869,14 @@ class EventualConsistencyPlugin extends Plugin {
4471
4869
  return true;
4472
4870
  });
4473
4871
  }
4872
+ const [recordOk, recordErr, record] = await tryFn(
4873
+ () => this.targetResource.get(originalId)
4874
+ );
4875
+ const currentValue = recordOk && record ? record[this.config.field] || 0 : 0;
4876
+ const hasSetOperation = filtered.some((t) => t.operation === "set");
4877
+ if (currentValue !== 0 && !hasSetOperation) {
4878
+ filtered.unshift(this._createSyntheticSetTransaction(currentValue));
4879
+ }
4474
4880
  filtered.sort(
4475
4881
  (a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()
4476
4882
  );
@@ -4505,6 +4911,133 @@ class EventualConsistencyPlugin extends Plugin {
4505
4911
  }
4506
4912
  return stats;
4507
4913
  }
4914
+ /**
4915
+ * Clean up stale locks that exceed the configured timeout
4916
+ * Uses distributed locking to prevent multiple containers from cleaning simultaneously
4917
+ */
4918
+ async cleanupStaleLocks() {
4919
+ const now = Date.now();
4920
+ const lockTimeoutMs = this.config.lockTimeout * 1e3;
4921
+ const cutoffTime = now - lockTimeoutMs;
4922
+ const cleanupLockId = `lock-cleanup-${this.config.resource}-${this.config.field}`;
4923
+ const [lockAcquired] = await tryFn(
4924
+ () => this.lockResource.insert({
4925
+ id: cleanupLockId,
4926
+ lockedAt: Date.now(),
4927
+ workerId: process.pid ? String(process.pid) : "unknown"
4928
+ })
4929
+ );
4930
+ if (!lockAcquired) {
4931
+ if (this.config.verbose) {
4932
+ console.log(`[EventualConsistency] Lock cleanup already running in another container`);
4933
+ }
4934
+ return;
4935
+ }
4936
+ try {
4937
+ const [ok, err, locks] = await tryFn(() => this.lockResource.list());
4938
+ if (!ok || !locks || locks.length === 0) return;
4939
+ const staleLocks = locks.filter(
4940
+ (lock) => lock.id !== cleanupLockId && lock.lockedAt < cutoffTime
4941
+ );
4942
+ if (staleLocks.length === 0) return;
4943
+ if (this.config.verbose) {
4944
+ console.log(`[EventualConsistency] Cleaning up ${staleLocks.length} stale locks`);
4945
+ }
4946
+ const { results, errors } = await promisePool.PromisePool.for(staleLocks).withConcurrency(5).process(async (lock) => {
4947
+ const [deleted] = await tryFn(() => this.lockResource.delete(lock.id));
4948
+ return deleted;
4949
+ });
4950
+ if (errors && errors.length > 0 && this.config.verbose) {
4951
+ console.warn(`[EventualConsistency] ${errors.length} stale locks failed to delete`);
4952
+ }
4953
+ } catch (error) {
4954
+ if (this.config.verbose) {
4955
+ console.warn(`[EventualConsistency] Error cleaning up stale locks:`, error.message);
4956
+ }
4957
+ } finally {
4958
+ await tryFn(() => this.lockResource.delete(cleanupLockId));
4959
+ }
4960
+ }
4961
+ /**
4962
+ * Start garbage collection timer for old applied transactions
4963
+ */
4964
+ startGarbageCollectionTimer() {
4965
+ const gcIntervalMs = this.config.gcInterval * 1e3;
4966
+ this.gcTimer = setInterval(async () => {
4967
+ await this.runGarbageCollection();
4968
+ }, gcIntervalMs);
4969
+ }
4970
+ /**
4971
+ * Delete old applied transactions based on retention policy
4972
+ * Uses distributed locking to prevent multiple containers from running GC simultaneously
4973
+ */
4974
+ async runGarbageCollection() {
4975
+ const gcLockId = `lock-gc-${this.config.resource}-${this.config.field}`;
4976
+ const [lockAcquired] = await tryFn(
4977
+ () => this.lockResource.insert({
4978
+ id: gcLockId,
4979
+ lockedAt: Date.now(),
4980
+ workerId: process.pid ? String(process.pid) : "unknown"
4981
+ })
4982
+ );
4983
+ if (!lockAcquired) {
4984
+ if (this.config.verbose) {
4985
+ console.log(`[EventualConsistency] GC already running in another container`);
4986
+ }
4987
+ return;
4988
+ }
4989
+ try {
4990
+ const now = Date.now();
4991
+ const retentionMs = this.config.transactionRetention * 24 * 60 * 60 * 1e3;
4992
+ const cutoffDate = new Date(now - retentionMs);
4993
+ const cutoffIso = cutoffDate.toISOString();
4994
+ if (this.config.verbose) {
4995
+ console.log(`[EventualConsistency] Running GC for transactions older than ${cutoffIso} (${this.config.transactionRetention} days)`);
4996
+ }
4997
+ const cutoffMonth = cutoffDate.toISOString().substring(0, 7);
4998
+ const [ok, err, oldTransactions] = await tryFn(
4999
+ () => this.transactionResource.query({
5000
+ applied: true,
5001
+ timestamp: { "<": cutoffIso }
5002
+ })
5003
+ );
5004
+ if (!ok) {
5005
+ if (this.config.verbose) {
5006
+ console.warn(`[EventualConsistency] GC failed to query transactions:`, err?.message);
5007
+ }
5008
+ return;
5009
+ }
5010
+ if (!oldTransactions || oldTransactions.length === 0) {
5011
+ if (this.config.verbose) {
5012
+ console.log(`[EventualConsistency] No old transactions to clean up`);
5013
+ }
5014
+ return;
5015
+ }
5016
+ if (this.config.verbose) {
5017
+ console.log(`[EventualConsistency] Deleting ${oldTransactions.length} old transactions`);
5018
+ }
5019
+ const { results, errors } = await promisePool.PromisePool.for(oldTransactions).withConcurrency(10).process(async (txn) => {
5020
+ const [deleted] = await tryFn(() => this.transactionResource.delete(txn.id));
5021
+ return deleted;
5022
+ });
5023
+ if (this.config.verbose) {
5024
+ console.log(`[EventualConsistency] GC completed: ${results.length} deleted, ${errors.length} errors`);
5025
+ }
5026
+ this.emit("eventual-consistency.gc-completed", {
5027
+ resource: this.config.resource,
5028
+ field: this.config.field,
5029
+ deletedCount: results.length,
5030
+ errorCount: errors.length
5031
+ });
5032
+ } catch (error) {
5033
+ if (this.config.verbose) {
5034
+ console.warn(`[EventualConsistency] GC error:`, error.message);
5035
+ }
5036
+ this.emit("eventual-consistency.gc-error", error);
5037
+ } finally {
5038
+ await tryFn(() => this.lockResource.delete(gcLockId));
5039
+ }
5040
+ }
4508
5041
  }
4509
5042
 
4510
5043
  class FullTextPlugin extends Plugin {
@@ -4521,7 +5054,7 @@ class FullTextPlugin extends Plugin {
4521
5054
  async setup(database) {
4522
5055
  this.database = database;
4523
5056
  const [ok, err, indexResource] = await tryFn(() => database.createResource({
4524
- name: "fulltext_indexes",
5057
+ name: "plg_fulltext_indexes",
4525
5058
  attributes: {
4526
5059
  id: "string|required",
4527
5060
  resourceName: "string|required",
@@ -4580,7 +5113,7 @@ class FullTextPlugin extends Plugin {
4580
5113
  }
4581
5114
  installDatabaseHooks() {
4582
5115
  this.database.addHook("afterCreateResource", (resource) => {
4583
- if (resource.name !== "fulltext_indexes") {
5116
+ if (resource.name !== "plg_fulltext_indexes") {
4584
5117
  this.installResourceHooks(resource);
4585
5118
  }
4586
5119
  });
@@ -4594,14 +5127,14 @@ class FullTextPlugin extends Plugin {
4594
5127
  }
4595
5128
  this.database.plugins.fulltext = this;
4596
5129
  for (const resource of Object.values(this.database.resources)) {
4597
- if (resource.name === "fulltext_indexes") continue;
5130
+ if (resource.name === "plg_fulltext_indexes") continue;
4598
5131
  this.installResourceHooks(resource);
4599
5132
  }
4600
5133
  if (!this.database._fulltextProxyInstalled) {
4601
5134
  this.database._previousCreateResourceForFullText = this.database.createResource;
4602
5135
  this.database.createResource = async function(...args) {
4603
5136
  const resource = await this._previousCreateResourceForFullText(...args);
4604
- if (this.plugins?.fulltext && resource.name !== "fulltext_indexes") {
5137
+ if (this.plugins?.fulltext && resource.name !== "plg_fulltext_indexes") {
4605
5138
  this.plugins.fulltext.installResourceHooks(resource);
4606
5139
  }
4607
5140
  return resource;
@@ -4609,7 +5142,7 @@ class FullTextPlugin extends Plugin {
4609
5142
  this.database._fulltextProxyInstalled = true;
4610
5143
  }
4611
5144
  for (const resource of Object.values(this.database.resources)) {
4612
- if (resource.name !== "fulltext_indexes") {
5145
+ if (resource.name !== "plg_fulltext_indexes") {
4613
5146
  this.installResourceHooks(resource);
4614
5147
  }
4615
5148
  }
@@ -4852,7 +5385,7 @@ class FullTextPlugin extends Plugin {
4852
5385
  return this._rebuildAllIndexesInternal();
4853
5386
  }
4854
5387
  async _rebuildAllIndexesInternal() {
4855
- const resourceNames = Object.keys(this.database.resources).filter((name) => name !== "fulltext_indexes");
5388
+ const resourceNames = Object.keys(this.database.resources).filter((name) => name !== "plg_fulltext_indexes");
4856
5389
  for (const resourceName of resourceNames) {
4857
5390
  const [ok, err] = await tryFn(() => this.rebuildIndex(resourceName));
4858
5391
  }
@@ -4904,7 +5437,7 @@ class MetricsPlugin extends Plugin {
4904
5437
  if (typeof process !== "undefined" && process.env.NODE_ENV === "test") return;
4905
5438
  const [ok, err] = await tryFn(async () => {
4906
5439
  const [ok1, err1, metricsResource] = await tryFn(() => database.createResource({
4907
- name: "metrics",
5440
+ name: "plg_metrics",
4908
5441
  attributes: {
4909
5442
  id: "string|required",
4910
5443
  type: "string|required",
@@ -4919,9 +5452,9 @@ class MetricsPlugin extends Plugin {
4919
5452
  metadata: "json"
4920
5453
  }
4921
5454
  }));
4922
- this.metricsResource = ok1 ? metricsResource : database.resources.metrics;
5455
+ this.metricsResource = ok1 ? metricsResource : database.resources.plg_metrics;
4923
5456
  const [ok2, err2, errorsResource] = await tryFn(() => database.createResource({
4924
- name: "error_logs",
5457
+ name: "plg_error_logs",
4925
5458
  attributes: {
4926
5459
  id: "string|required",
4927
5460
  resourceName: "string|required",
@@ -4931,9 +5464,9 @@ class MetricsPlugin extends Plugin {
4931
5464
  metadata: "json"
4932
5465
  }
4933
5466
  }));
4934
- this.errorsResource = ok2 ? errorsResource : database.resources.error_logs;
5467
+ this.errorsResource = ok2 ? errorsResource : database.resources.plg_error_logs;
4935
5468
  const [ok3, err3, performanceResource] = await tryFn(() => database.createResource({
4936
- name: "performance_logs",
5469
+ name: "plg_performance_logs",
4937
5470
  attributes: {
4938
5471
  id: "string|required",
4939
5472
  resourceName: "string|required",
@@ -4943,12 +5476,12 @@ class MetricsPlugin extends Plugin {
4943
5476
  metadata: "json"
4944
5477
  }
4945
5478
  }));
4946
- this.performanceResource = ok3 ? performanceResource : database.resources.performance_logs;
5479
+ this.performanceResource = ok3 ? performanceResource : database.resources.plg_performance_logs;
4947
5480
  });
4948
5481
  if (!ok) {
4949
- this.metricsResource = database.resources.metrics;
4950
- this.errorsResource = database.resources.error_logs;
4951
- this.performanceResource = database.resources.performance_logs;
5482
+ this.metricsResource = database.resources.plg_metrics;
5483
+ this.errorsResource = database.resources.plg_error_logs;
5484
+ this.performanceResource = database.resources.plg_performance_logs;
4952
5485
  }
4953
5486
  this.installDatabaseHooks();
4954
5487
  this.installMetricsHooks();
@@ -4967,7 +5500,7 @@ class MetricsPlugin extends Plugin {
4967
5500
  }
4968
5501
  installDatabaseHooks() {
4969
5502
  this.database.addHook("afterCreateResource", (resource) => {
4970
- if (resource.name !== "metrics" && resource.name !== "error_logs" && resource.name !== "performance_logs") {
5503
+ if (resource.name !== "plg_metrics" && resource.name !== "plg_error_logs" && resource.name !== "plg_performance_logs") {
4971
5504
  this.installResourceHooks(resource);
4972
5505
  }
4973
5506
  });
@@ -4977,7 +5510,7 @@ class MetricsPlugin extends Plugin {
4977
5510
  }
4978
5511
  installMetricsHooks() {
4979
5512
  for (const resource of Object.values(this.database.resources)) {
4980
- if (["metrics", "error_logs", "performance_logs"].includes(resource.name)) {
5513
+ if (["plg_metrics", "plg_error_logs", "plg_performance_logs"].includes(resource.name)) {
4981
5514
  continue;
4982
5515
  }
4983
5516
  this.installResourceHooks(resource);
@@ -4985,7 +5518,7 @@ class MetricsPlugin extends Plugin {
4985
5518
  this.database._createResource = this.database.createResource;
4986
5519
  this.database.createResource = async function(...args) {
4987
5520
  const resource = await this._createResource(...args);
4988
- if (this.plugins?.metrics && !["metrics", "error_logs", "performance_logs"].includes(resource.name)) {
5521
+ if (this.plugins?.metrics && !["plg_metrics", "plg_error_logs", "plg_performance_logs"].includes(resource.name)) {
4989
5522
  this.plugins.metrics.installResourceHooks(resource);
4990
5523
  }
4991
5524
  return resource;
@@ -5369,6 +5902,253 @@ class MetricsPlugin extends Plugin {
5369
5902
  }
5370
5903
  }
5371
5904
 
5905
+ class SqsConsumer {
5906
+ constructor({ queueUrl, onMessage, onError, poolingInterval = 5e3, maxMessages = 10, region = "us-east-1", credentials, endpoint, driver = "sqs" }) {
5907
+ this.driver = driver;
5908
+ this.queueUrl = queueUrl;
5909
+ this.onMessage = onMessage;
5910
+ this.onError = onError;
5911
+ this.poolingInterval = poolingInterval;
5912
+ this.maxMessages = maxMessages;
5913
+ this.region = region;
5914
+ this.credentials = credentials;
5915
+ this.endpoint = endpoint;
5916
+ this.sqs = null;
5917
+ this._stopped = false;
5918
+ this._timer = null;
5919
+ this._pollPromise = null;
5920
+ this._pollResolve = null;
5921
+ this._SQSClient = null;
5922
+ this._ReceiveMessageCommand = null;
5923
+ this._DeleteMessageCommand = null;
5924
+ }
5925
+ async start() {
5926
+ const [ok, err, sdk] = await tryFn(() => import('@aws-sdk/client-sqs'));
5927
+ if (!ok) throw new Error("SqsConsumer: @aws-sdk/client-sqs is not installed. Please install it to use the SQS consumer.");
5928
+ const { SQSClient, ReceiveMessageCommand, DeleteMessageCommand } = sdk;
5929
+ this._SQSClient = SQSClient;
5930
+ this._ReceiveMessageCommand = ReceiveMessageCommand;
5931
+ this._DeleteMessageCommand = DeleteMessageCommand;
5932
+ this.sqs = new SQSClient({ region: this.region, credentials: this.credentials, endpoint: this.endpoint });
5933
+ this._stopped = false;
5934
+ this._pollPromise = new Promise((resolve) => {
5935
+ this._pollResolve = resolve;
5936
+ });
5937
+ this._poll();
5938
+ }
5939
+ async stop() {
5940
+ this._stopped = true;
5941
+ if (this._timer) {
5942
+ clearTimeout(this._timer);
5943
+ this._timer = null;
5944
+ }
5945
+ if (this._pollResolve) {
5946
+ this._pollResolve();
5947
+ }
5948
+ }
5949
+ async _poll() {
5950
+ if (this._stopped) {
5951
+ if (this._pollResolve) this._pollResolve();
5952
+ return;
5953
+ }
5954
+ const [ok, err, result] = await tryFn(async () => {
5955
+ const cmd = new this._ReceiveMessageCommand({
5956
+ QueueUrl: this.queueUrl,
5957
+ MaxNumberOfMessages: this.maxMessages,
5958
+ WaitTimeSeconds: 10,
5959
+ MessageAttributeNames: ["All"]
5960
+ });
5961
+ const { Messages } = await this.sqs.send(cmd);
5962
+ if (Messages && Messages.length > 0) {
5963
+ for (const msg of Messages) {
5964
+ const [okMsg, errMsg] = await tryFn(async () => {
5965
+ const parsedMsg = this._parseMessage(msg);
5966
+ await this.onMessage(parsedMsg, msg);
5967
+ await this.sqs.send(new this._DeleteMessageCommand({
5968
+ QueueUrl: this.queueUrl,
5969
+ ReceiptHandle: msg.ReceiptHandle
5970
+ }));
5971
+ });
5972
+ if (!okMsg && this.onError) {
5973
+ this.onError(errMsg, msg);
5974
+ }
5975
+ }
5976
+ }
5977
+ });
5978
+ if (!ok && this.onError) {
5979
+ this.onError(err);
5980
+ }
5981
+ this._timer = setTimeout(() => this._poll(), this.poolingInterval);
5982
+ }
5983
+ _parseMessage(msg) {
5984
+ let body;
5985
+ const [ok, err, parsed] = tryFn(() => JSON.parse(msg.Body));
5986
+ body = ok ? parsed : msg.Body;
5987
+ const attributes = {};
5988
+ if (msg.MessageAttributes) {
5989
+ for (const [k, v] of Object.entries(msg.MessageAttributes)) {
5990
+ attributes[k] = v.StringValue;
5991
+ }
5992
+ }
5993
+ return { $body: body, $attributes: attributes, $raw: msg };
5994
+ }
5995
+ }
5996
+
5997
+ class RabbitMqConsumer {
5998
+ constructor({ amqpUrl, queue, prefetch = 10, reconnectInterval = 2e3, onMessage, onError, driver = "rabbitmq" }) {
5999
+ this.amqpUrl = amqpUrl;
6000
+ this.queue = queue;
6001
+ this.prefetch = prefetch;
6002
+ this.reconnectInterval = reconnectInterval;
6003
+ this.onMessage = onMessage;
6004
+ this.onError = onError;
6005
+ this.driver = driver;
6006
+ this.connection = null;
6007
+ this.channel = null;
6008
+ this._stopped = false;
6009
+ }
6010
+ async start() {
6011
+ this._stopped = false;
6012
+ await this._connect();
6013
+ }
6014
+ async stop() {
6015
+ this._stopped = true;
6016
+ if (this.channel) await this.channel.close();
6017
+ if (this.connection) await this.connection.close();
6018
+ }
6019
+ async _connect() {
6020
+ const [ok, err] = await tryFn(async () => {
6021
+ const amqp = (await import('amqplib')).default;
6022
+ this.connection = await amqp.connect(this.amqpUrl);
6023
+ this.channel = await this.connection.createChannel();
6024
+ await this.channel.assertQueue(this.queue, { durable: true });
6025
+ this.channel.prefetch(this.prefetch);
6026
+ this.channel.consume(this.queue, async (msg) => {
6027
+ if (msg !== null) {
6028
+ const [okMsg, errMsg] = await tryFn(async () => {
6029
+ const content = JSON.parse(msg.content.toString());
6030
+ await this.onMessage({ $body: content, $raw: msg });
6031
+ this.channel.ack(msg);
6032
+ });
6033
+ if (!okMsg) {
6034
+ if (this.onError) this.onError(errMsg, msg);
6035
+ this.channel.nack(msg, false, false);
6036
+ }
6037
+ }
6038
+ });
6039
+ });
6040
+ if (!ok) {
6041
+ if (this.onError) this.onError(err);
6042
+ if (!this._stopped) {
6043
+ setTimeout(() => this._connect(), this.reconnectInterval);
6044
+ }
6045
+ }
6046
+ }
6047
+ }
6048
+
6049
+ const CONSUMER_DRIVERS = {
6050
+ sqs: SqsConsumer,
6051
+ rabbitmq: RabbitMqConsumer
6052
+ // kafka: KafkaConsumer, // futuro
6053
+ };
6054
+ function createConsumer(driver, config) {
6055
+ const ConsumerClass = CONSUMER_DRIVERS[driver];
6056
+ if (!ConsumerClass) {
6057
+ throw new Error(`Unknown consumer driver: ${driver}. Available: ${Object.keys(CONSUMER_DRIVERS).join(", ")}`);
6058
+ }
6059
+ return new ConsumerClass(config);
6060
+ }
6061
+
6062
+ class QueueConsumerPlugin {
6063
+ constructor(options = {}) {
6064
+ this.options = options;
6065
+ this.driversConfig = Array.isArray(options.consumers) ? options.consumers : [];
6066
+ this.consumers = [];
6067
+ }
6068
+ async setup(database) {
6069
+ this.database = database;
6070
+ for (const driverDef of this.driversConfig) {
6071
+ const { driver, config: driverConfig = {}, consumers: consumerDefs = [] } = driverDef;
6072
+ if (consumerDefs.length === 0 && driverDef.resources) {
6073
+ const { resources, driver: defDriver, config: nestedConfig, ...directConfig } = driverDef;
6074
+ const resourceList = Array.isArray(resources) ? resources : [resources];
6075
+ const flatConfig = nestedConfig ? { ...directConfig, ...nestedConfig } : directConfig;
6076
+ for (const resource of resourceList) {
6077
+ const consumer = createConsumer(driver, {
6078
+ ...flatConfig,
6079
+ onMessage: (msg) => this._handleMessage(msg, resource),
6080
+ onError: (err, raw) => this._handleError(err, raw, resource)
6081
+ });
6082
+ await consumer.start();
6083
+ this.consumers.push(consumer);
6084
+ }
6085
+ } else {
6086
+ for (const consumerDef of consumerDefs) {
6087
+ const { resources, ...consumerConfig } = consumerDef;
6088
+ const resourceList = Array.isArray(resources) ? resources : [resources];
6089
+ for (const resource of resourceList) {
6090
+ const mergedConfig = { ...driverConfig, ...consumerConfig };
6091
+ const consumer = createConsumer(driver, {
6092
+ ...mergedConfig,
6093
+ onMessage: (msg) => this._handleMessage(msg, resource),
6094
+ onError: (err, raw) => this._handleError(err, raw, resource)
6095
+ });
6096
+ await consumer.start();
6097
+ this.consumers.push(consumer);
6098
+ }
6099
+ }
6100
+ }
6101
+ }
6102
+ }
6103
+ async stop() {
6104
+ if (!Array.isArray(this.consumers)) this.consumers = [];
6105
+ for (const consumer of this.consumers) {
6106
+ if (consumer && typeof consumer.stop === "function") {
6107
+ await consumer.stop();
6108
+ }
6109
+ }
6110
+ this.consumers = [];
6111
+ }
6112
+ async _handleMessage(msg, configuredResource) {
6113
+ this.options;
6114
+ let body = msg.$body || msg;
6115
+ if (body.$body && !body.resource && !body.action && !body.data) {
6116
+ body = body.$body;
6117
+ }
6118
+ let resource = body.resource || msg.resource;
6119
+ let action = body.action || msg.action;
6120
+ let data = body.data || msg.data;
6121
+ if (!resource) {
6122
+ throw new Error("QueueConsumerPlugin: resource not found in message");
6123
+ }
6124
+ if (!action) {
6125
+ throw new Error("QueueConsumerPlugin: action not found in message");
6126
+ }
6127
+ const resourceObj = this.database.resources[resource];
6128
+ if (!resourceObj) throw new Error(`QueueConsumerPlugin: resource '${resource}' not found`);
6129
+ let result;
6130
+ const [ok, err, res] = await tryFn(async () => {
6131
+ if (action === "insert") {
6132
+ result = await resourceObj.insert(data);
6133
+ } else if (action === "update") {
6134
+ const { id: updateId, ...updateAttributes } = data;
6135
+ result = await resourceObj.update(updateId, updateAttributes);
6136
+ } else if (action === "delete") {
6137
+ result = await resourceObj.delete(data.id);
6138
+ } else {
6139
+ throw new Error(`QueueConsumerPlugin: unsupported action '${action}'`);
6140
+ }
6141
+ return result;
6142
+ });
6143
+ if (!ok) {
6144
+ throw err;
6145
+ }
6146
+ return res;
6147
+ }
6148
+ _handleError(err, raw, resourceName) {
6149
+ }
6150
+ }
6151
+
5372
6152
  class BaseReplicator extends EventEmitter {
5373
6153
  constructor(config = {}) {
5374
6154
  super();
@@ -6356,7 +7136,7 @@ class Client extends EventEmitter {
6356
7136
  this.emit("command.response", command.constructor.name, response, command.input);
6357
7137
  return response;
6358
7138
  }
6359
- async putObject({ key, metadata, contentType, body, contentEncoding, contentLength }) {
7139
+ async putObject({ key, metadata, contentType, body, contentEncoding, contentLength, ifMatch }) {
6360
7140
  const keyPrefix = typeof this.config.keyPrefix === "string" ? this.config.keyPrefix : "";
6361
7141
  keyPrefix ? path.join(keyPrefix, key) : key;
6362
7142
  const stringMetadata = {};
@@ -6376,6 +7156,7 @@ class Client extends EventEmitter {
6376
7156
  if (contentType !== void 0) options.ContentType = contentType;
6377
7157
  if (contentEncoding !== void 0) options.ContentEncoding = contentEncoding;
6378
7158
  if (contentLength !== void 0) options.ContentLength = contentLength;
7159
+ if (ifMatch !== void 0) options.IfMatch = ifMatch;
6379
7160
  let response, error;
6380
7161
  try {
6381
7162
  response = await this.sendCommand(new clientS3.PutObjectCommand(options));
@@ -8539,6 +9320,7 @@ ${errorDetails}`,
8539
9320
  data._lastModified = request.LastModified;
8540
9321
  data._hasContent = request.ContentLength > 0;
8541
9322
  data._mimeType = request.ContentType || null;
9323
+ data._etag = request.ETag;
8542
9324
  data._v = objectVersion;
8543
9325
  if (request.VersionId) data._versionId = request.VersionId;
8544
9326
  if (request.Expiration) data._expiresAt = request.Expiration;
@@ -8751,59 +9533,225 @@ ${errorDetails}`,
8751
9533
  }
8752
9534
  }
8753
9535
  /**
8754
- * Delete a resource object by ID
9536
+ * Update with conditional check (If-Match ETag)
8755
9537
  * @param {string} id - Resource ID
8756
- * @returns {Promise<Object>} S3 delete response
9538
+ * @param {Object} attributes - Attributes to update
9539
+ * @param {Object} options - Options including ifMatch (ETag)
9540
+ * @returns {Promise<Object>} { success: boolean, data?: Object, etag?: string, error?: string }
8757
9541
  * @example
8758
- * await resource.delete('user-123');
9542
+ * const msg = await resource.get('msg-123');
9543
+ * const result = await resource.updateConditional('msg-123', { status: 'processing' }, { ifMatch: msg._etag });
9544
+ * if (!result.success) {
9545
+ * console.log('Update failed - object was modified by another process');
9546
+ * }
8759
9547
  */
8760
- async delete(id) {
9548
+ async updateConditional(id, attributes, options = {}) {
8761
9549
  if (lodashEs.isEmpty(id)) {
8762
9550
  throw new Error("id cannot be empty");
8763
9551
  }
8764
- let objectData;
8765
- let deleteError = null;
8766
- const [ok, err, data] = await tryFn(() => this.get(id));
8767
- if (ok) {
8768
- objectData = data;
8769
- } else {
8770
- objectData = { id };
8771
- deleteError = err;
9552
+ const { ifMatch } = options;
9553
+ if (!ifMatch) {
9554
+ throw new Error("updateConditional requires ifMatch option with ETag value");
8772
9555
  }
8773
- await this.executeHooks("beforeDelete", objectData);
8774
- const key = this.getResourceKey(id);
8775
- const [ok2, err2, response] = await tryFn(() => this.client.deleteObject(key));
8776
- this.emit("delete", {
8777
- ...objectData,
8778
- $before: { ...objectData },
8779
- $after: null
8780
- });
8781
- if (deleteError) {
8782
- throw mapAwsError(deleteError, {
8783
- bucket: this.client.config.bucket,
8784
- key,
8785
- resourceName: this.name,
8786
- operation: "delete",
8787
- id
8788
- });
9556
+ const exists = await this.exists(id);
9557
+ if (!exists) {
9558
+ return {
9559
+ success: false,
9560
+ error: `Resource with id '${id}' does not exist`
9561
+ };
8789
9562
  }
8790
- if (!ok2) throw mapAwsError(err2, {
8791
- key,
8792
- resourceName: this.name,
8793
- operation: "delete",
8794
- id
8795
- });
8796
- if (this.config.asyncPartitions && this.config.partitions && Object.keys(this.config.partitions).length > 0) {
8797
- setImmediate(() => {
8798
- this.deletePartitionReferences(objectData).catch((err3) => {
8799
- this.emit("partitionIndexError", {
8800
- operation: "delete",
8801
- id,
8802
- error: err3,
8803
- message: err3.message
8804
- });
8805
- });
8806
- });
9563
+ const originalData = await this.get(id);
9564
+ const attributesClone = lodashEs.cloneDeep(attributes);
9565
+ let mergedData = lodashEs.cloneDeep(originalData);
9566
+ for (const [key2, value] of Object.entries(attributesClone)) {
9567
+ if (key2.includes(".")) {
9568
+ let ref = mergedData;
9569
+ const parts = key2.split(".");
9570
+ for (let i = 0; i < parts.length - 1; i++) {
9571
+ if (typeof ref[parts[i]] !== "object" || ref[parts[i]] === null) {
9572
+ ref[parts[i]] = {};
9573
+ }
9574
+ ref = ref[parts[i]];
9575
+ }
9576
+ ref[parts[parts.length - 1]] = lodashEs.cloneDeep(value);
9577
+ } else if (typeof value === "object" && value !== null && !Array.isArray(value)) {
9578
+ mergedData[key2] = lodashEs.merge({}, mergedData[key2], value);
9579
+ } else {
9580
+ mergedData[key2] = lodashEs.cloneDeep(value);
9581
+ }
9582
+ }
9583
+ if (this.config.timestamps) {
9584
+ const now = (/* @__PURE__ */ new Date()).toISOString();
9585
+ mergedData.updatedAt = now;
9586
+ if (!mergedData.metadata) mergedData.metadata = {};
9587
+ mergedData.metadata.updatedAt = now;
9588
+ }
9589
+ const preProcessedData = await this.executeHooks("beforeUpdate", lodashEs.cloneDeep(mergedData));
9590
+ const completeData = { ...originalData, ...preProcessedData, id };
9591
+ const { isValid, errors, data } = await this.validate(lodashEs.cloneDeep(completeData));
9592
+ if (!isValid) {
9593
+ return {
9594
+ success: false,
9595
+ error: "Validation failed: " + (errors && errors.length ? JSON.stringify(errors) : "unknown"),
9596
+ validationErrors: errors
9597
+ };
9598
+ }
9599
+ const { id: validatedId, ...validatedAttributes } = data;
9600
+ const mappedData = await this.schema.mapper(validatedAttributes);
9601
+ mappedData._v = String(this.version);
9602
+ const behaviorImpl = getBehavior(this.behavior);
9603
+ const { mappedData: processedMetadata, body } = await behaviorImpl.handleUpdate({
9604
+ resource: this,
9605
+ id,
9606
+ data: validatedAttributes,
9607
+ mappedData,
9608
+ originalData: { ...attributesClone, id }
9609
+ });
9610
+ const key = this.getResourceKey(id);
9611
+ let existingContentType = void 0;
9612
+ let finalBody = body;
9613
+ if (body === "" && this.behavior !== "body-overflow") {
9614
+ const [ok2, err2, existingObject] = await tryFn(() => this.client.getObject(key));
9615
+ if (ok2 && existingObject.ContentLength > 0) {
9616
+ const existingBodyBuffer = Buffer.from(await existingObject.Body.transformToByteArray());
9617
+ const existingBodyString = existingBodyBuffer.toString();
9618
+ const [okParse, errParse] = await tryFn(() => Promise.resolve(JSON.parse(existingBodyString)));
9619
+ if (!okParse) {
9620
+ finalBody = existingBodyBuffer;
9621
+ existingContentType = existingObject.ContentType;
9622
+ }
9623
+ }
9624
+ }
9625
+ let finalContentType = existingContentType;
9626
+ if (finalBody && finalBody !== "" && !finalContentType) {
9627
+ const [okParse, errParse] = await tryFn(() => Promise.resolve(JSON.parse(finalBody)));
9628
+ if (okParse) finalContentType = "application/json";
9629
+ }
9630
+ const [ok, err, response] = await tryFn(() => this.client.putObject({
9631
+ key,
9632
+ body: finalBody,
9633
+ contentType: finalContentType,
9634
+ metadata: processedMetadata,
9635
+ ifMatch
9636
+ // ← Conditional write with ETag
9637
+ }));
9638
+ if (!ok) {
9639
+ if (err.name === "PreconditionFailed" || err.$metadata?.httpStatusCode === 412) {
9640
+ return {
9641
+ success: false,
9642
+ error: "ETag mismatch - object was modified by another process"
9643
+ };
9644
+ }
9645
+ return {
9646
+ success: false,
9647
+ error: err.message || "Update failed"
9648
+ };
9649
+ }
9650
+ const updatedData = await this.composeFullObjectFromWrite({
9651
+ id,
9652
+ metadata: processedMetadata,
9653
+ body: finalBody,
9654
+ behavior: this.behavior
9655
+ });
9656
+ const oldData = { ...originalData, id };
9657
+ const newData = { ...validatedAttributes, id };
9658
+ if (this.config.asyncPartitions && this.config.partitions && Object.keys(this.config.partitions).length > 0) {
9659
+ setImmediate(() => {
9660
+ this.handlePartitionReferenceUpdates(oldData, newData).catch((err2) => {
9661
+ this.emit("partitionIndexError", {
9662
+ operation: "updateConditional",
9663
+ id,
9664
+ error: err2,
9665
+ message: err2.message
9666
+ });
9667
+ });
9668
+ });
9669
+ const nonPartitionHooks = this.hooks.afterUpdate.filter(
9670
+ (hook) => !hook.toString().includes("handlePartitionReferenceUpdates")
9671
+ );
9672
+ let finalResult = updatedData;
9673
+ for (const hook of nonPartitionHooks) {
9674
+ finalResult = await hook(finalResult);
9675
+ }
9676
+ this.emit("update", {
9677
+ ...updatedData,
9678
+ $before: { ...originalData },
9679
+ $after: { ...finalResult }
9680
+ });
9681
+ return {
9682
+ success: true,
9683
+ data: finalResult,
9684
+ etag: response.ETag
9685
+ };
9686
+ } else {
9687
+ await this.handlePartitionReferenceUpdates(oldData, newData);
9688
+ const finalResult = await this.executeHooks("afterUpdate", updatedData);
9689
+ this.emit("update", {
9690
+ ...updatedData,
9691
+ $before: { ...originalData },
9692
+ $after: { ...finalResult }
9693
+ });
9694
+ return {
9695
+ success: true,
9696
+ data: finalResult,
9697
+ etag: response.ETag
9698
+ };
9699
+ }
9700
+ }
9701
+ /**
9702
+ * Delete a resource object by ID
9703
+ * @param {string} id - Resource ID
9704
+ * @returns {Promise<Object>} S3 delete response
9705
+ * @example
9706
+ * await resource.delete('user-123');
9707
+ */
9708
+ async delete(id) {
9709
+ if (lodashEs.isEmpty(id)) {
9710
+ throw new Error("id cannot be empty");
9711
+ }
9712
+ let objectData;
9713
+ let deleteError = null;
9714
+ const [ok, err, data] = await tryFn(() => this.get(id));
9715
+ if (ok) {
9716
+ objectData = data;
9717
+ } else {
9718
+ objectData = { id };
9719
+ deleteError = err;
9720
+ }
9721
+ await this.executeHooks("beforeDelete", objectData);
9722
+ const key = this.getResourceKey(id);
9723
+ const [ok2, err2, response] = await tryFn(() => this.client.deleteObject(key));
9724
+ this.emit("delete", {
9725
+ ...objectData,
9726
+ $before: { ...objectData },
9727
+ $after: null
9728
+ });
9729
+ if (deleteError) {
9730
+ throw mapAwsError(deleteError, {
9731
+ bucket: this.client.config.bucket,
9732
+ key,
9733
+ resourceName: this.name,
9734
+ operation: "delete",
9735
+ id
9736
+ });
9737
+ }
9738
+ if (!ok2) throw mapAwsError(err2, {
9739
+ key,
9740
+ resourceName: this.name,
9741
+ operation: "delete",
9742
+ id
9743
+ });
9744
+ if (this.config.asyncPartitions && this.config.partitions && Object.keys(this.config.partitions).length > 0) {
9745
+ setImmediate(() => {
9746
+ this.deletePartitionReferences(objectData).catch((err3) => {
9747
+ this.emit("partitionIndexError", {
9748
+ operation: "delete",
9749
+ id,
9750
+ error: err3,
9751
+ message: err3.message
9752
+ });
9753
+ });
9754
+ });
8807
9755
  const nonPartitionHooks = this.hooks.afterDelete.filter(
8808
9756
  (hook) => !hook.toString().includes("deletePartitionReferences")
8809
9757
  );
@@ -10157,7 +11105,7 @@ class Database extends EventEmitter {
10157
11105
  this.id = idGenerator(7);
10158
11106
  this.version = "1";
10159
11107
  this.s3dbVersion = (() => {
10160
- const [ok, err, version] = tryFn(() => true ? "10.0.0" : "latest");
11108
+ const [ok, err, version] = tryFn(() => true ? "10.0.3" : "latest");
10161
11109
  return ok ? version : "latest";
10162
11110
  })();
10163
11111
  this.resources = {};
@@ -11410,16 +12358,20 @@ class S3dbReplicator extends BaseReplicator {
11410
12358
  return resource;
11411
12359
  }
11412
12360
  _getDestResourceObj(resource) {
11413
- const available = Object.keys(this.client.resources || {});
12361
+ const db = this.targetDatabase || this.client;
12362
+ const available = Object.keys(db.resources || {});
11414
12363
  const norm = normalizeResourceName$1(resource);
11415
12364
  const found = available.find((r) => normalizeResourceName$1(r) === norm);
11416
12365
  if (!found) {
11417
12366
  throw new Error(`[S3dbReplicator] Destination resource not found: ${resource}. Available: ${available.join(", ")}`);
11418
12367
  }
11419
- return this.client.resources[found];
12368
+ return db.resources[found];
11420
12369
  }
11421
12370
  async replicateBatch(resourceName, records) {
11422
- if (!this.enabled || !this.shouldReplicateResource(resourceName)) {
12371
+ if (this.enabled === false) {
12372
+ return { skipped: true, reason: "replicator_disabled" };
12373
+ }
12374
+ if (!this.shouldReplicateResource(resourceName)) {
11423
12375
  return { skipped: true, reason: "resource_not_included" };
11424
12376
  }
11425
12377
  const results = [];
@@ -11530,11 +12482,12 @@ class SqsReplicator extends BaseReplicator {
11530
12482
  this.client = client;
11531
12483
  this.queueUrl = config.queueUrl;
11532
12484
  this.queues = config.queues || {};
11533
- this.defaultQueue = config.defaultQueue || config.defaultQueueUrl || config.queueUrlDefault;
12485
+ this.defaultQueue = config.defaultQueue || config.defaultQueueUrl || config.queueUrlDefault || null;
11534
12486
  this.region = config.region || "us-east-1";
11535
12487
  this.sqsClient = client || null;
11536
12488
  this.messageGroupId = config.messageGroupId;
11537
12489
  this.deduplicationId = config.deduplicationId;
12490
+ this.resourceQueueMap = config.resourceQueueMap || null;
11538
12491
  if (Array.isArray(resources)) {
11539
12492
  this.resources = {};
11540
12493
  for (const resource of resources) {
@@ -11665,7 +12618,10 @@ class SqsReplicator extends BaseReplicator {
11665
12618
  }
11666
12619
  }
11667
12620
  async replicate(resource, operation, data, id, beforeData = null) {
11668
- if (!this.enabled || !this.shouldReplicateResource(resource)) {
12621
+ if (this.enabled === false) {
12622
+ return { skipped: true, reason: "replicator_disabled" };
12623
+ }
12624
+ if (!this.shouldReplicateResource(resource)) {
11669
12625
  return { skipped: true, reason: "resource_not_included" };
11670
12626
  }
11671
12627
  const [ok, err, result] = await tryFn(async () => {
@@ -11709,7 +12665,10 @@ class SqsReplicator extends BaseReplicator {
11709
12665
  return { success: false, error: err.message };
11710
12666
  }
11711
12667
  async replicateBatch(resource, records) {
11712
- if (!this.enabled || !this.shouldReplicateResource(resource)) {
12668
+ if (this.enabled === false) {
12669
+ return { skipped: true, reason: "replicator_disabled" };
12670
+ }
12671
+ if (!this.shouldReplicateResource(resource)) {
11713
12672
  return { skipped: true, reason: "resource_not_included" };
11714
12673
  }
11715
12674
  const [ok, err, result] = await tryFn(async () => {
@@ -11863,22 +12822,23 @@ class ReplicatorPlugin extends Plugin {
11863
12822
  replicators: options.replicators || [],
11864
12823
  logErrors: options.logErrors !== false,
11865
12824
  replicatorLogResource: options.replicatorLogResource || "replicator_log",
12825
+ persistReplicatorLog: options.persistReplicatorLog || false,
11866
12826
  enabled: options.enabled !== false,
11867
12827
  batchSize: options.batchSize || 100,
11868
12828
  maxRetries: options.maxRetries || 3,
11869
12829
  timeout: options.timeout || 3e4,
11870
- verbose: options.verbose || false,
11871
- ...options
12830
+ verbose: options.verbose || false
11872
12831
  };
11873
12832
  this.replicators = [];
11874
12833
  this.database = null;
11875
12834
  this.eventListenersInstalled = /* @__PURE__ */ new Set();
11876
- }
11877
- /**
11878
- * Decompress data if it was compressed
11879
- */
11880
- async decompressData(data) {
11881
- return data;
12835
+ this.eventHandlers = /* @__PURE__ */ new Map();
12836
+ this.stats = {
12837
+ totalReplications: 0,
12838
+ totalErrors: 0,
12839
+ lastSync: null
12840
+ };
12841
+ this._afterCreateResourceHook = null;
11882
12842
  }
11883
12843
  // Helper to filter out internal S3DB fields
11884
12844
  filterInternalFields(obj) {
@@ -11899,7 +12859,7 @@ class ReplicatorPlugin extends Plugin {
11899
12859
  if (!resource || this.eventListenersInstalled.has(resource.name) || resource.name === this.config.replicatorLogResource) {
11900
12860
  return;
11901
12861
  }
11902
- resource.on("insert", async (data) => {
12862
+ const insertHandler = async (data) => {
11903
12863
  const [ok, error] = await tryFn(async () => {
11904
12864
  const completeData = { ...data, createdAt: (/* @__PURE__ */ new Date()).toISOString() };
11905
12865
  await plugin.processReplicatorEvent("insert", resource.name, completeData.id, completeData);
@@ -11910,8 +12870,8 @@ class ReplicatorPlugin extends Plugin {
11910
12870
  }
11911
12871
  this.emit("error", { operation: "insert", error: error.message, resource: resource.name });
11912
12872
  }
11913
- });
11914
- resource.on("update", async (data, beforeData) => {
12873
+ };
12874
+ const updateHandler = async (data, beforeData) => {
11915
12875
  const [ok, error] = await tryFn(async () => {
11916
12876
  const completeData = await plugin.getCompleteData(resource, data);
11917
12877
  const dataWithTimestamp = { ...completeData, updatedAt: (/* @__PURE__ */ new Date()).toISOString() };
@@ -11923,8 +12883,8 @@ class ReplicatorPlugin extends Plugin {
11923
12883
  }
11924
12884
  this.emit("error", { operation: "update", error: error.message, resource: resource.name });
11925
12885
  }
11926
- });
11927
- resource.on("delete", async (data) => {
12886
+ };
12887
+ const deleteHandler = async (data) => {
11928
12888
  const [ok, error] = await tryFn(async () => {
11929
12889
  await plugin.processReplicatorEvent("delete", resource.name, data.id, data);
11930
12890
  });
@@ -11934,14 +12894,22 @@ class ReplicatorPlugin extends Plugin {
11934
12894
  }
11935
12895
  this.emit("error", { operation: "delete", error: error.message, resource: resource.name });
11936
12896
  }
11937
- });
12897
+ };
12898
+ this.eventHandlers.set(resource.name, {
12899
+ insert: insertHandler,
12900
+ update: updateHandler,
12901
+ delete: deleteHandler
12902
+ });
12903
+ resource.on("insert", insertHandler);
12904
+ resource.on("update", updateHandler);
12905
+ resource.on("delete", deleteHandler);
11938
12906
  this.eventListenersInstalled.add(resource.name);
11939
12907
  }
11940
12908
  async setup(database) {
11941
12909
  this.database = database;
11942
12910
  if (this.config.persistReplicatorLog) {
11943
12911
  const [ok, err, logResource] = await tryFn(() => database.createResource({
11944
- name: this.config.replicatorLogResource || "replicator_logs",
12912
+ name: this.config.replicatorLogResource || "plg_replicator_logs",
11945
12913
  attributes: {
11946
12914
  id: "string|required",
11947
12915
  resource: "string|required",
@@ -11955,13 +12923,13 @@ class ReplicatorPlugin extends Plugin {
11955
12923
  if (ok) {
11956
12924
  this.replicatorLogResource = logResource;
11957
12925
  } else {
11958
- this.replicatorLogResource = database.resources[this.config.replicatorLogResource || "replicator_logs"];
12926
+ this.replicatorLogResource = database.resources[this.config.replicatorLogResource || "plg_replicator_logs"];
11959
12927
  }
11960
12928
  }
11961
12929
  await this.initializeReplicators(database);
11962
12930
  this.installDatabaseHooks();
11963
12931
  for (const resource of Object.values(database.resources)) {
11964
- if (resource.name !== (this.config.replicatorLogResource || "replicator_logs")) {
12932
+ if (resource.name !== (this.config.replicatorLogResource || "plg_replicator_logs")) {
11965
12933
  this.installEventListeners(resource, database, this);
11966
12934
  }
11967
12935
  }
@@ -11977,14 +12945,18 @@ class ReplicatorPlugin extends Plugin {
11977
12945
  this.removeDatabaseHooks();
11978
12946
  }
11979
12947
  installDatabaseHooks() {
11980
- this.database.addHook("afterCreateResource", (resource) => {
11981
- if (resource.name !== (this.config.replicatorLogResource || "replicator_logs")) {
12948
+ this._afterCreateResourceHook = (resource) => {
12949
+ if (resource.name !== (this.config.replicatorLogResource || "plg_replicator_logs")) {
11982
12950
  this.installEventListeners(resource, this.database, this);
11983
12951
  }
11984
- });
12952
+ };
12953
+ this.database.addHook("afterCreateResource", this._afterCreateResourceHook);
11985
12954
  }
11986
12955
  removeDatabaseHooks() {
11987
- this.database.removeHook("afterCreateResource", this.installEventListeners.bind(this));
12956
+ if (this._afterCreateResourceHook) {
12957
+ this.database.removeHook("afterCreateResource", this._afterCreateResourceHook);
12958
+ this._afterCreateResourceHook = null;
12959
+ }
11988
12960
  }
11989
12961
  createReplicator(driver, config, resources, client) {
11990
12962
  return createReplicator(driver, config, resources, client);
@@ -12009,9 +12981,9 @@ class ReplicatorPlugin extends Plugin {
12009
12981
  async retryWithBackoff(operation, maxRetries = 3) {
12010
12982
  let lastError;
12011
12983
  for (let attempt = 1; attempt <= maxRetries; attempt++) {
12012
- const [ok, error] = await tryFn(operation);
12984
+ const [ok, error, result] = await tryFn(operation);
12013
12985
  if (ok) {
12014
- return ok;
12986
+ return result;
12015
12987
  } else {
12016
12988
  lastError = error;
12017
12989
  if (this.config.verbose) {
@@ -12106,7 +13078,7 @@ class ReplicatorPlugin extends Plugin {
12106
13078
  });
12107
13079
  return Promise.allSettled(promises);
12108
13080
  }
12109
- async processreplicatorItem(item) {
13081
+ async processReplicatorItem(item) {
12110
13082
  const applicableReplicators = this.replicators.filter((replicator) => {
12111
13083
  const should = replicator.shouldReplicateResource && replicator.shouldReplicateResource(item.resourceName, item.operation);
12112
13084
  return should;
@@ -12166,12 +13138,9 @@ class ReplicatorPlugin extends Plugin {
12166
13138
  });
12167
13139
  return Promise.allSettled(promises);
12168
13140
  }
12169
- async logreplicator(item) {
13141
+ async logReplicator(item) {
12170
13142
  const logRes = this.replicatorLog || this.database.resources[normalizeResourceName(this.config.replicatorLogResource)];
12171
13143
  if (!logRes) {
12172
- if (this.database) {
12173
- if (this.database.options && this.database.options.connectionString) ;
12174
- }
12175
13144
  this.emit("replicator.log.failed", { error: "replicator log resource not found", item });
12176
13145
  return;
12177
13146
  }
@@ -12193,7 +13162,7 @@ class ReplicatorPlugin extends Plugin {
12193
13162
  this.emit("replicator.log.failed", { error: err, item });
12194
13163
  }
12195
13164
  }
12196
- async updatereplicatorLog(logId, updates) {
13165
+ async updateReplicatorLog(logId, updates) {
12197
13166
  if (!this.replicatorLog) return;
12198
13167
  const [ok, err] = await tryFn(async () => {
12199
13168
  await this.replicatorLog.update(logId, {
@@ -12206,7 +13175,7 @@ class ReplicatorPlugin extends Plugin {
12206
13175
  }
12207
13176
  }
12208
13177
  // Utility methods
12209
- async getreplicatorStats() {
13178
+ async getReplicatorStats() {
12210
13179
  const replicatorStats = await Promise.all(
12211
13180
  this.replicators.map(async (replicator) => {
12212
13181
  const status = await replicator.getStatus();
@@ -12220,15 +13189,11 @@ class ReplicatorPlugin extends Plugin {
12220
13189
  );
12221
13190
  return {
12222
13191
  replicators: replicatorStats,
12223
- queue: {
12224
- length: this.queue.length,
12225
- isProcessing: this.isProcessing
12226
- },
12227
13192
  stats: this.stats,
12228
13193
  lastSync: this.stats.lastSync
12229
13194
  };
12230
13195
  }
12231
- async getreplicatorLogs(options = {}) {
13196
+ async getReplicatorLogs(options = {}) {
12232
13197
  if (!this.replicatorLog) {
12233
13198
  return [];
12234
13199
  }
@@ -12239,32 +13204,32 @@ class ReplicatorPlugin extends Plugin {
12239
13204
  limit = 100,
12240
13205
  offset = 0
12241
13206
  } = options;
12242
- let query = {};
13207
+ const filter = {};
12243
13208
  if (resourceName) {
12244
- query.resourceName = resourceName;
13209
+ filter.resourceName = resourceName;
12245
13210
  }
12246
13211
  if (operation) {
12247
- query.operation = operation;
13212
+ filter.operation = operation;
12248
13213
  }
12249
13214
  if (status) {
12250
- query.status = status;
13215
+ filter.status = status;
12251
13216
  }
12252
- const logs = await this.replicatorLog.list(query);
12253
- return logs.slice(offset, offset + limit);
13217
+ const logs = await this.replicatorLog.query(filter, { limit, offset });
13218
+ return logs || [];
12254
13219
  }
12255
- async retryFailedreplicators() {
13220
+ async retryFailedReplicators() {
12256
13221
  if (!this.replicatorLog) {
12257
13222
  return { retried: 0 };
12258
13223
  }
12259
- const failedLogs = await this.replicatorLog.list({
13224
+ const failedLogs = await this.replicatorLog.query({
12260
13225
  status: "failed"
12261
13226
  });
12262
13227
  let retried = 0;
12263
- for (const log of failedLogs) {
13228
+ for (const log of failedLogs || []) {
12264
13229
  const [ok, err] = await tryFn(async () => {
12265
13230
  await this.processReplicatorEvent(
12266
- log.resourceName,
12267
13231
  log.operation,
13232
+ log.resourceName,
12268
13233
  log.recordId,
12269
13234
  log.data
12270
13235
  );
@@ -12282,13 +13247,21 @@ class ReplicatorPlugin extends Plugin {
12282
13247
  }
12283
13248
  this.stats.lastSync = (/* @__PURE__ */ new Date()).toISOString();
12284
13249
  for (const resourceName in this.database.resources) {
12285
- if (normalizeResourceName(resourceName) === normalizeResourceName("replicator_logs")) continue;
13250
+ if (normalizeResourceName(resourceName) === normalizeResourceName("plg_replicator_logs")) continue;
12286
13251
  if (replicator.shouldReplicateResource(resourceName)) {
12287
13252
  this.emit("replicator.sync.resource", { resourceName, replicatorId });
12288
13253
  const resource = this.database.resources[resourceName];
12289
- const allRecords = await resource.getAll();
12290
- for (const record of allRecords) {
12291
- await replicator.replicate(resourceName, "insert", record, record.id);
13254
+ let offset = 0;
13255
+ const pageSize = this.config.batchSize || 100;
13256
+ while (true) {
13257
+ const [ok, err, page] = await tryFn(() => resource.page({ offset, size: pageSize }));
13258
+ if (!ok || !page) break;
13259
+ const records = Array.isArray(page) ? page : page.items || [];
13260
+ if (records.length === 0) break;
13261
+ for (const record of records) {
13262
+ await replicator.replicate(resourceName, "insert", record, record.id);
13263
+ }
13264
+ offset += pageSize;
12292
13265
  }
12293
13266
  }
12294
13267
  }
@@ -12316,9 +13289,21 @@ class ReplicatorPlugin extends Plugin {
12316
13289
  });
12317
13290
  await Promise.allSettled(cleanupPromises);
12318
13291
  }
13292
+ if (this.database && this.database.resources) {
13293
+ for (const resourceName of this.eventListenersInstalled) {
13294
+ const resource = this.database.resources[resourceName];
13295
+ const handlers = this.eventHandlers.get(resourceName);
13296
+ if (resource && handlers) {
13297
+ resource.off("insert", handlers.insert);
13298
+ resource.off("update", handlers.update);
13299
+ resource.off("delete", handlers.delete);
13300
+ }
13301
+ }
13302
+ }
12319
13303
  this.replicators = [];
12320
13304
  this.database = null;
12321
13305
  this.eventListenersInstalled.clear();
13306
+ this.eventHandlers.clear();
12322
13307
  this.removeAllListeners();
12323
13308
  });
12324
13309
  if (!ok) {
@@ -12332,46 +13317,591 @@ class ReplicatorPlugin extends Plugin {
12332
13317
  }
12333
13318
  }
12334
13319
 
12335
- class SchedulerPlugin extends Plugin {
13320
+ class S3QueuePlugin extends Plugin {
12336
13321
  constructor(options = {}) {
12337
- super();
13322
+ super(options);
13323
+ if (!options.resource) {
13324
+ throw new Error('S3QueuePlugin requires "resource" option');
13325
+ }
12338
13326
  this.config = {
12339
- timezone: options.timezone || "UTC",
12340
- jobs: options.jobs || {},
12341
- defaultTimeout: options.defaultTimeout || 3e5,
12342
- // 5 minutes
12343
- defaultRetries: options.defaultRetries || 1,
12344
- jobHistoryResource: options.jobHistoryResource || "job_executions",
12345
- persistJobs: options.persistJobs !== false,
13327
+ resource: options.resource,
13328
+ visibilityTimeout: options.visibilityTimeout || 3e4,
13329
+ // 30 seconds
13330
+ pollInterval: options.pollInterval || 1e3,
13331
+ // 1 second
13332
+ maxAttempts: options.maxAttempts || 3,
13333
+ concurrency: options.concurrency || 1,
13334
+ deadLetterResource: options.deadLetterResource || null,
13335
+ autoStart: options.autoStart !== false,
13336
+ onMessage: options.onMessage,
13337
+ onError: options.onError,
13338
+ onComplete: options.onComplete,
12346
13339
  verbose: options.verbose || false,
12347
- onJobStart: options.onJobStart || null,
12348
- onJobComplete: options.onJobComplete || null,
12349
- onJobError: options.onJobError || null,
12350
13340
  ...options
12351
13341
  };
12352
- this.database = null;
12353
- this.jobs = /* @__PURE__ */ new Map();
12354
- this.activeJobs = /* @__PURE__ */ new Map();
12355
- this.timers = /* @__PURE__ */ new Map();
12356
- this.statistics = /* @__PURE__ */ new Map();
12357
- this._validateConfiguration();
13342
+ this.queueResource = null;
13343
+ this.targetResource = null;
13344
+ this.deadLetterResourceObj = null;
13345
+ this.workers = [];
13346
+ this.isRunning = false;
13347
+ this.workerId = `worker-${Date.now()}-${Math.random().toString(36).slice(2, 9)}`;
13348
+ this.processedCache = /* @__PURE__ */ new Map();
13349
+ this.cacheCleanupInterval = null;
13350
+ this.lockCleanupInterval = null;
12358
13351
  }
12359
- _validateConfiguration() {
12360
- if (Object.keys(this.config.jobs).length === 0) {
12361
- throw new Error("SchedulerPlugin: At least one job must be defined");
13352
+ async onSetup() {
13353
+ this.targetResource = this.database.resources[this.config.resource];
13354
+ if (!this.targetResource) {
13355
+ throw new Error(`S3QueuePlugin: resource '${this.config.resource}' not found`);
12362
13356
  }
12363
- for (const [jobName, job] of Object.entries(this.config.jobs)) {
12364
- if (!job.schedule) {
12365
- throw new Error(`SchedulerPlugin: Job '${jobName}' must have a schedule`);
12366
- }
12367
- if (!job.action || typeof job.action !== "function") {
12368
- throw new Error(`SchedulerPlugin: Job '${jobName}' must have an action function`);
12369
- }
12370
- if (!this._isValidCronExpression(job.schedule)) {
12371
- throw new Error(`SchedulerPlugin: Job '${jobName}' has invalid cron expression: ${job.schedule}`);
12372
- }
13357
+ const queueName = `${this.config.resource}_queue`;
13358
+ const [ok, err] = await tryFn(
13359
+ () => this.database.createResource({
13360
+ name: queueName,
13361
+ attributes: {
13362
+ id: "string|required",
13363
+ originalId: "string|required",
13364
+ // ID do registro original
13365
+ status: "string|required",
13366
+ // pending/processing/completed/failed/dead
13367
+ visibleAt: "number|required",
13368
+ // Timestamp de visibilidade
13369
+ claimedBy: "string|optional",
13370
+ // Worker que claimed
13371
+ claimedAt: "number|optional",
13372
+ // Timestamp do claim
13373
+ attempts: "number|default:0",
13374
+ maxAttempts: "number|default:3",
13375
+ error: "string|optional",
13376
+ result: "json|optional",
13377
+ createdAt: "string|required",
13378
+ completedAt: "number|optional"
13379
+ },
13380
+ behavior: "body-overflow",
13381
+ timestamps: true,
13382
+ asyncPartitions: true,
13383
+ partitions: {
13384
+ byStatus: { fields: { status: "string" } },
13385
+ byDate: { fields: { createdAt: "string|maxlength:10" } }
13386
+ }
13387
+ })
13388
+ );
13389
+ if (!ok && !this.database.resources[queueName]) {
13390
+ throw new Error(`Failed to create queue resource: ${err?.message}`);
12373
13391
  }
12374
- }
13392
+ this.queueResource = this.database.resources[queueName];
13393
+ const lockName = `${this.config.resource}_locks`;
13394
+ const [okLock, errLock] = await tryFn(
13395
+ () => this.database.createResource({
13396
+ name: lockName,
13397
+ attributes: {
13398
+ id: "string|required",
13399
+ workerId: "string|required",
13400
+ timestamp: "number|required",
13401
+ ttl: "number|default:5000"
13402
+ },
13403
+ behavior: "body-overflow",
13404
+ timestamps: false
13405
+ })
13406
+ );
13407
+ if (okLock || this.database.resources[lockName]) {
13408
+ this.lockResource = this.database.resources[lockName];
13409
+ } else {
13410
+ this.lockResource = null;
13411
+ if (this.config.verbose) {
13412
+ console.log(`[S3QueuePlugin] Lock resource creation failed, locking disabled: ${errLock?.message}`);
13413
+ }
13414
+ }
13415
+ this.addHelperMethods();
13416
+ if (this.config.deadLetterResource) {
13417
+ await this.createDeadLetterResource();
13418
+ }
13419
+ if (this.config.verbose) {
13420
+ console.log(`[S3QueuePlugin] Setup completed for resource '${this.config.resource}'`);
13421
+ }
13422
+ }
13423
+ async onStart() {
13424
+ if (this.config.autoStart && this.config.onMessage) {
13425
+ await this.startProcessing();
13426
+ }
13427
+ }
13428
+ async onStop() {
13429
+ await this.stopProcessing();
13430
+ }
13431
+ addHelperMethods() {
13432
+ const plugin = this;
13433
+ const resource = this.targetResource;
13434
+ resource.enqueue = async function(data, options = {}) {
13435
+ const recordData = {
13436
+ id: data.id || idGenerator(),
13437
+ ...data
13438
+ };
13439
+ const record = await resource.insert(recordData);
13440
+ const queueEntry = {
13441
+ id: idGenerator(),
13442
+ originalId: record.id,
13443
+ status: "pending",
13444
+ visibleAt: Date.now(),
13445
+ attempts: 0,
13446
+ maxAttempts: options.maxAttempts || plugin.config.maxAttempts,
13447
+ createdAt: (/* @__PURE__ */ new Date()).toISOString().slice(0, 10)
13448
+ };
13449
+ await plugin.queueResource.insert(queueEntry);
13450
+ plugin.emit("message.enqueued", { id: record.id, queueId: queueEntry.id });
13451
+ return record;
13452
+ };
13453
+ resource.queueStats = async function() {
13454
+ return await plugin.getStats();
13455
+ };
13456
+ resource.startProcessing = async function(handler, options = {}) {
13457
+ return await plugin.startProcessing(handler, options);
13458
+ };
13459
+ resource.stopProcessing = async function() {
13460
+ return await plugin.stopProcessing();
13461
+ };
13462
+ }
13463
+ async startProcessing(handler = null, options = {}) {
13464
+ if (this.isRunning) {
13465
+ if (this.config.verbose) {
13466
+ console.log("[S3QueuePlugin] Already running");
13467
+ }
13468
+ return;
13469
+ }
13470
+ const messageHandler = handler || this.config.onMessage;
13471
+ if (!messageHandler) {
13472
+ throw new Error("S3QueuePlugin: onMessage handler required");
13473
+ }
13474
+ this.isRunning = true;
13475
+ const concurrency = options.concurrency || this.config.concurrency;
13476
+ this.cacheCleanupInterval = setInterval(() => {
13477
+ const now = Date.now();
13478
+ const maxAge = 3e4;
13479
+ for (const [queueId, timestamp] of this.processedCache.entries()) {
13480
+ if (now - timestamp > maxAge) {
13481
+ this.processedCache.delete(queueId);
13482
+ }
13483
+ }
13484
+ }, 5e3);
13485
+ this.lockCleanupInterval = setInterval(() => {
13486
+ this.cleanupStaleLocks().catch((err) => {
13487
+ if (this.config.verbose) {
13488
+ console.log(`[lockCleanup] Error: ${err.message}`);
13489
+ }
13490
+ });
13491
+ }, 1e4);
13492
+ for (let i = 0; i < concurrency; i++) {
13493
+ const worker = this.createWorker(messageHandler, i);
13494
+ this.workers.push(worker);
13495
+ }
13496
+ if (this.config.verbose) {
13497
+ console.log(`[S3QueuePlugin] Started ${concurrency} workers`);
13498
+ }
13499
+ this.emit("workers.started", { concurrency, workerId: this.workerId });
13500
+ }
13501
+ async stopProcessing() {
13502
+ if (!this.isRunning) return;
13503
+ this.isRunning = false;
13504
+ if (this.cacheCleanupInterval) {
13505
+ clearInterval(this.cacheCleanupInterval);
13506
+ this.cacheCleanupInterval = null;
13507
+ }
13508
+ if (this.lockCleanupInterval) {
13509
+ clearInterval(this.lockCleanupInterval);
13510
+ this.lockCleanupInterval = null;
13511
+ }
13512
+ await Promise.all(this.workers);
13513
+ this.workers = [];
13514
+ this.processedCache.clear();
13515
+ if (this.config.verbose) {
13516
+ console.log("[S3QueuePlugin] Stopped all workers");
13517
+ }
13518
+ this.emit("workers.stopped", { workerId: this.workerId });
13519
+ }
13520
+ createWorker(handler, workerIndex) {
13521
+ return (async () => {
13522
+ while (this.isRunning) {
13523
+ try {
13524
+ const message = await this.claimMessage();
13525
+ if (message) {
13526
+ await this.processMessage(message, handler);
13527
+ } else {
13528
+ await new Promise((resolve) => setTimeout(resolve, this.config.pollInterval));
13529
+ }
13530
+ } catch (error) {
13531
+ if (this.config.verbose) {
13532
+ console.error(`[Worker ${workerIndex}] Error:`, error.message);
13533
+ }
13534
+ await new Promise((resolve) => setTimeout(resolve, 1e3));
13535
+ }
13536
+ }
13537
+ })();
13538
+ }
13539
+ async claimMessage() {
13540
+ const now = Date.now();
13541
+ const [ok, err, messages] = await tryFn(
13542
+ () => this.queueResource.query({
13543
+ status: "pending"
13544
+ })
13545
+ );
13546
+ if (!ok || !messages || messages.length === 0) {
13547
+ return null;
13548
+ }
13549
+ const available = messages.filter((m) => m.visibleAt <= now);
13550
+ if (available.length === 0) {
13551
+ return null;
13552
+ }
13553
+ for (const msg of available) {
13554
+ const claimed = await this.attemptClaim(msg);
13555
+ if (claimed) {
13556
+ return claimed;
13557
+ }
13558
+ }
13559
+ return null;
13560
+ }
13561
+ /**
13562
+ * Acquire a distributed lock using ETag-based conditional updates
13563
+ * This ensures only one worker can claim a message at a time
13564
+ *
13565
+ * Uses a two-step process:
13566
+ * 1. Create lock resource (similar to queue resource) if not exists
13567
+ * 2. Try to claim lock using ETag-based conditional update
13568
+ */
13569
+ async acquireLock(messageId) {
13570
+ if (!this.lockResource) {
13571
+ return true;
13572
+ }
13573
+ const lockId = `lock-${messageId}`;
13574
+ const now = Date.now();
13575
+ try {
13576
+ const [okGet, errGet, existingLock] = await tryFn(
13577
+ () => this.lockResource.get(lockId)
13578
+ );
13579
+ if (existingLock) {
13580
+ const lockAge = now - existingLock.timestamp;
13581
+ if (lockAge < existingLock.ttl) {
13582
+ return false;
13583
+ }
13584
+ const [ok, err, result] = await tryFn(
13585
+ () => this.lockResource.updateConditional(lockId, {
13586
+ workerId: this.workerId,
13587
+ timestamp: now,
13588
+ ttl: 5e3
13589
+ }, {
13590
+ ifMatch: existingLock._etag
13591
+ })
13592
+ );
13593
+ return ok && result.success;
13594
+ }
13595
+ const [okCreate, errCreate] = await tryFn(
13596
+ () => this.lockResource.insert({
13597
+ id: lockId,
13598
+ workerId: this.workerId,
13599
+ timestamp: now,
13600
+ ttl: 5e3
13601
+ })
13602
+ );
13603
+ return okCreate;
13604
+ } catch (error) {
13605
+ if (this.config.verbose) {
13606
+ console.log(`[acquireLock] Error: ${error.message}`);
13607
+ }
13608
+ return false;
13609
+ }
13610
+ }
13611
+ /**
13612
+ * Release a distributed lock by deleting the lock record
13613
+ */
13614
+ async releaseLock(messageId) {
13615
+ if (!this.lockResource) {
13616
+ return;
13617
+ }
13618
+ const lockId = `lock-${messageId}`;
13619
+ try {
13620
+ await this.lockResource.delete(lockId);
13621
+ } catch (error) {
13622
+ if (this.config.verbose) {
13623
+ console.log(`[releaseLock] Failed to release lock for ${messageId}: ${error.message}`);
13624
+ }
13625
+ }
13626
+ }
13627
+ /**
13628
+ * Clean up stale locks (older than TTL)
13629
+ * This prevents deadlocks if a worker crashes while holding a lock
13630
+ */
13631
+ async cleanupStaleLocks() {
13632
+ if (!this.lockResource) {
13633
+ return;
13634
+ }
13635
+ const now = Date.now();
13636
+ try {
13637
+ const locks = await this.lockResource.list();
13638
+ for (const lock of locks) {
13639
+ const lockAge = now - lock.timestamp;
13640
+ if (lockAge > lock.ttl) {
13641
+ await this.lockResource.delete(lock.id);
13642
+ if (this.config.verbose) {
13643
+ console.log(`[cleanupStaleLocks] Removed expired lock: ${lock.id}`);
13644
+ }
13645
+ }
13646
+ }
13647
+ } catch (error) {
13648
+ if (this.config.verbose) {
13649
+ console.log(`[cleanupStaleLocks] Error during cleanup: ${error.message}`);
13650
+ }
13651
+ }
13652
+ }
13653
+ async attemptClaim(msg) {
13654
+ const now = Date.now();
13655
+ const lockAcquired = await this.acquireLock(msg.id);
13656
+ if (!lockAcquired) {
13657
+ return null;
13658
+ }
13659
+ if (this.processedCache.has(msg.id)) {
13660
+ await this.releaseLock(msg.id);
13661
+ if (this.config.verbose) {
13662
+ console.log(`[attemptClaim] Message ${msg.id} already processed (in cache)`);
13663
+ }
13664
+ return null;
13665
+ }
13666
+ this.processedCache.set(msg.id, Date.now());
13667
+ await this.releaseLock(msg.id);
13668
+ const [okGet, errGet, msgWithETag] = await tryFn(
13669
+ () => this.queueResource.get(msg.id)
13670
+ );
13671
+ if (!okGet || !msgWithETag) {
13672
+ this.processedCache.delete(msg.id);
13673
+ if (this.config.verbose) {
13674
+ console.log(`[attemptClaim] Message ${msg.id} not found or error: ${errGet?.message}`);
13675
+ }
13676
+ return null;
13677
+ }
13678
+ if (msgWithETag.status !== "pending" || msgWithETag.visibleAt > now) {
13679
+ this.processedCache.delete(msg.id);
13680
+ if (this.config.verbose) {
13681
+ console.log(`[attemptClaim] Message ${msg.id} not claimable: status=${msgWithETag.status}, visibleAt=${msgWithETag.visibleAt}, now=${now}`);
13682
+ }
13683
+ return null;
13684
+ }
13685
+ if (this.config.verbose) {
13686
+ console.log(`[attemptClaim] Attempting to claim ${msg.id} with ETag: ${msgWithETag._etag}`);
13687
+ }
13688
+ const [ok, err, result] = await tryFn(
13689
+ () => this.queueResource.updateConditional(msgWithETag.id, {
13690
+ status: "processing",
13691
+ claimedBy: this.workerId,
13692
+ claimedAt: now,
13693
+ visibleAt: now + this.config.visibilityTimeout,
13694
+ attempts: msgWithETag.attempts + 1
13695
+ }, {
13696
+ ifMatch: msgWithETag._etag
13697
+ // ← ATOMIC CLAIM using ETag!
13698
+ })
13699
+ );
13700
+ if (!ok || !result.success) {
13701
+ this.processedCache.delete(msg.id);
13702
+ if (this.config.verbose) {
13703
+ console.log(`[attemptClaim] Failed to claim ${msg.id}: ${err?.message || result.error}`);
13704
+ }
13705
+ return null;
13706
+ }
13707
+ if (this.config.verbose) {
13708
+ console.log(`[attemptClaim] Successfully claimed ${msg.id}`);
13709
+ }
13710
+ const [okRecord, errRecord, record] = await tryFn(
13711
+ () => this.targetResource.get(msgWithETag.originalId)
13712
+ );
13713
+ if (!okRecord) {
13714
+ await this.failMessage(msgWithETag.id, "Original record not found");
13715
+ return null;
13716
+ }
13717
+ return {
13718
+ queueId: msgWithETag.id,
13719
+ record,
13720
+ attempts: msgWithETag.attempts + 1,
13721
+ maxAttempts: msgWithETag.maxAttempts
13722
+ };
13723
+ }
13724
+ async processMessage(message, handler) {
13725
+ const startTime = Date.now();
13726
+ try {
13727
+ const result = await handler(message.record, {
13728
+ queueId: message.queueId,
13729
+ attempts: message.attempts,
13730
+ workerId: this.workerId
13731
+ });
13732
+ await this.completeMessage(message.queueId, result);
13733
+ const duration = Date.now() - startTime;
13734
+ this.emit("message.completed", {
13735
+ queueId: message.queueId,
13736
+ originalId: message.record.id,
13737
+ duration,
13738
+ attempts: message.attempts
13739
+ });
13740
+ if (this.config.onComplete) {
13741
+ await this.config.onComplete(message.record, result);
13742
+ }
13743
+ } catch (error) {
13744
+ const shouldRetry = message.attempts < message.maxAttempts;
13745
+ if (shouldRetry) {
13746
+ await this.retryMessage(message.queueId, message.attempts, error.message);
13747
+ this.emit("message.retry", {
13748
+ queueId: message.queueId,
13749
+ originalId: message.record.id,
13750
+ attempts: message.attempts,
13751
+ error: error.message
13752
+ });
13753
+ } else {
13754
+ await this.moveToDeadLetter(message.queueId, message.record, error.message);
13755
+ this.emit("message.dead", {
13756
+ queueId: message.queueId,
13757
+ originalId: message.record.id,
13758
+ error: error.message
13759
+ });
13760
+ }
13761
+ if (this.config.onError) {
13762
+ await this.config.onError(error, message.record);
13763
+ }
13764
+ }
13765
+ }
13766
+ async completeMessage(queueId, result) {
13767
+ await this.queueResource.update(queueId, {
13768
+ status: "completed",
13769
+ completedAt: Date.now(),
13770
+ result
13771
+ });
13772
+ }
13773
+ async failMessage(queueId, error) {
13774
+ await this.queueResource.update(queueId, {
13775
+ status: "failed",
13776
+ error
13777
+ });
13778
+ }
13779
+ async retryMessage(queueId, attempts, error) {
13780
+ const backoff = Math.min(Math.pow(2, attempts) * 1e3, 3e4);
13781
+ await this.queueResource.update(queueId, {
13782
+ status: "pending",
13783
+ visibleAt: Date.now() + backoff,
13784
+ error
13785
+ });
13786
+ this.processedCache.delete(queueId);
13787
+ }
13788
+ async moveToDeadLetter(queueId, record, error) {
13789
+ if (this.config.deadLetterResource && this.deadLetterResourceObj) {
13790
+ const msg = await this.queueResource.get(queueId);
13791
+ await this.deadLetterResourceObj.insert({
13792
+ id: idGenerator(),
13793
+ originalId: record.id,
13794
+ queueId,
13795
+ data: record,
13796
+ error,
13797
+ attempts: msg.attempts,
13798
+ createdAt: (/* @__PURE__ */ new Date()).toISOString()
13799
+ });
13800
+ }
13801
+ await this.queueResource.update(queueId, {
13802
+ status: "dead",
13803
+ error
13804
+ });
13805
+ }
13806
+ async getStats() {
13807
+ const [ok, err, allMessages] = await tryFn(
13808
+ () => this.queueResource.list()
13809
+ );
13810
+ if (!ok) {
13811
+ if (this.config.verbose) {
13812
+ console.warn("[S3QueuePlugin] Failed to get stats:", err.message);
13813
+ }
13814
+ return null;
13815
+ }
13816
+ const stats = {
13817
+ total: allMessages.length,
13818
+ pending: 0,
13819
+ processing: 0,
13820
+ completed: 0,
13821
+ failed: 0,
13822
+ dead: 0
13823
+ };
13824
+ for (const msg of allMessages) {
13825
+ if (stats[msg.status] !== void 0) {
13826
+ stats[msg.status]++;
13827
+ }
13828
+ }
13829
+ return stats;
13830
+ }
13831
+ async createDeadLetterResource() {
13832
+ const [ok, err] = await tryFn(
13833
+ () => this.database.createResource({
13834
+ name: this.config.deadLetterResource,
13835
+ attributes: {
13836
+ id: "string|required",
13837
+ originalId: "string|required",
13838
+ queueId: "string|required",
13839
+ data: "json|required",
13840
+ error: "string|required",
13841
+ attempts: "number|required",
13842
+ createdAt: "string|required"
13843
+ },
13844
+ behavior: "body-overflow",
13845
+ timestamps: true
13846
+ })
13847
+ );
13848
+ if (ok || this.database.resources[this.config.deadLetterResource]) {
13849
+ this.deadLetterResourceObj = this.database.resources[this.config.deadLetterResource];
13850
+ if (this.config.verbose) {
13851
+ console.log(`[S3QueuePlugin] Dead letter queue created: ${this.config.deadLetterResource}`);
13852
+ }
13853
+ }
13854
+ }
13855
+ }
13856
+
13857
+ class SchedulerPlugin extends Plugin {
13858
+ constructor(options = {}) {
13859
+ super();
13860
+ this.config = {
13861
+ timezone: options.timezone || "UTC",
13862
+ jobs: options.jobs || {},
13863
+ defaultTimeout: options.defaultTimeout || 3e5,
13864
+ // 5 minutes
13865
+ defaultRetries: options.defaultRetries || 1,
13866
+ jobHistoryResource: options.jobHistoryResource || "plg_job_executions",
13867
+ persistJobs: options.persistJobs !== false,
13868
+ verbose: options.verbose || false,
13869
+ onJobStart: options.onJobStart || null,
13870
+ onJobComplete: options.onJobComplete || null,
13871
+ onJobError: options.onJobError || null,
13872
+ ...options
13873
+ };
13874
+ this.database = null;
13875
+ this.lockResource = null;
13876
+ this.jobs = /* @__PURE__ */ new Map();
13877
+ this.activeJobs = /* @__PURE__ */ new Map();
13878
+ this.timers = /* @__PURE__ */ new Map();
13879
+ this.statistics = /* @__PURE__ */ new Map();
13880
+ this._validateConfiguration();
13881
+ }
13882
+ /**
13883
+ * Helper to detect test environment
13884
+ * @private
13885
+ */
13886
+ _isTestEnvironment() {
13887
+ return process.env.NODE_ENV === "test" || process.env.JEST_WORKER_ID !== void 0 || global.expect !== void 0;
13888
+ }
13889
+ _validateConfiguration() {
13890
+ if (Object.keys(this.config.jobs).length === 0) {
13891
+ throw new Error("SchedulerPlugin: At least one job must be defined");
13892
+ }
13893
+ for (const [jobName, job] of Object.entries(this.config.jobs)) {
13894
+ if (!job.schedule) {
13895
+ throw new Error(`SchedulerPlugin: Job '${jobName}' must have a schedule`);
13896
+ }
13897
+ if (!job.action || typeof job.action !== "function") {
13898
+ throw new Error(`SchedulerPlugin: Job '${jobName}' must have an action function`);
13899
+ }
13900
+ if (!this._isValidCronExpression(job.schedule)) {
13901
+ throw new Error(`SchedulerPlugin: Job '${jobName}' has invalid cron expression: ${job.schedule}`);
13902
+ }
13903
+ }
13904
+ }
12375
13905
  _isValidCronExpression(expr) {
12376
13906
  if (typeof expr !== "string") return false;
12377
13907
  const shortcuts = ["@yearly", "@annually", "@monthly", "@weekly", "@daily", "@hourly"];
@@ -12382,6 +13912,7 @@ class SchedulerPlugin extends Plugin {
12382
13912
  }
12383
13913
  async setup(database) {
12384
13914
  this.database = database;
13915
+ await this._createLockResource();
12385
13916
  if (this.config.persistJobs) {
12386
13917
  await this._createJobHistoryResource();
12387
13918
  }
@@ -12410,6 +13941,25 @@ class SchedulerPlugin extends Plugin {
12410
13941
  await this._startScheduling();
12411
13942
  this.emit("initialized", { jobs: this.jobs.size });
12412
13943
  }
13944
+ async _createLockResource() {
13945
+ const [ok, err, lockResource] = await tryFn(
13946
+ () => this.database.createResource({
13947
+ name: "plg_scheduler_job_locks",
13948
+ attributes: {
13949
+ id: "string|required",
13950
+ jobName: "string|required",
13951
+ lockedAt: "number|required",
13952
+ instanceId: "string|optional"
13953
+ },
13954
+ behavior: "body-only",
13955
+ timestamps: false
13956
+ })
13957
+ );
13958
+ if (!ok && !this.database.resources.plg_scheduler_job_locks) {
13959
+ throw new Error(`Failed to create lock resource: ${err?.message}`);
13960
+ }
13961
+ this.lockResource = ok ? lockResource : this.database.resources.plg_scheduler_job_locks;
13962
+ }
12413
13963
  async _createJobHistoryResource() {
12414
13964
  const [ok] = await tryFn(() => this.database.createResource({
12415
13965
  name: this.config.jobHistoryResource,
@@ -12503,18 +14053,37 @@ class SchedulerPlugin extends Plugin {
12503
14053
  next.setHours(next.getHours() + 1);
12504
14054
  }
12505
14055
  }
12506
- const isTestEnvironment = process.env.NODE_ENV === "test" || process.env.JEST_WORKER_ID !== void 0 || global.expect !== void 0;
12507
- if (isTestEnvironment) {
14056
+ if (this._isTestEnvironment()) {
12508
14057
  next.setTime(next.getTime() + 1e3);
12509
14058
  }
12510
14059
  return next;
12511
14060
  }
12512
14061
  async _executeJob(jobName) {
12513
14062
  const job = this.jobs.get(jobName);
12514
- if (!job || this.activeJobs.has(jobName)) {
14063
+ if (!job) {
14064
+ return;
14065
+ }
14066
+ if (this.activeJobs.has(jobName)) {
14067
+ return;
14068
+ }
14069
+ this.activeJobs.set(jobName, "acquiring-lock");
14070
+ const lockId = `lock-${jobName}`;
14071
+ const [lockAcquired, lockErr] = await tryFn(
14072
+ () => this.lockResource.insert({
14073
+ id: lockId,
14074
+ jobName,
14075
+ lockedAt: Date.now(),
14076
+ instanceId: process.pid ? String(process.pid) : "unknown"
14077
+ })
14078
+ );
14079
+ if (!lockAcquired) {
14080
+ if (this.config.verbose) {
14081
+ console.log(`[SchedulerPlugin] Job '${jobName}' already running on another instance`);
14082
+ }
14083
+ this.activeJobs.delete(jobName);
12515
14084
  return;
12516
14085
  }
12517
- const executionId = `${jobName}_${Date.now()}`;
14086
+ const executionId = `${jobName}_${idGenerator()}`;
12518
14087
  const startTime = Date.now();
12519
14088
  const context = {
12520
14089
  jobName,
@@ -12523,91 +14092,95 @@ class SchedulerPlugin extends Plugin {
12523
14092
  database: this.database
12524
14093
  };
12525
14094
  this.activeJobs.set(jobName, executionId);
12526
- if (this.config.onJobStart) {
12527
- await this._executeHook(this.config.onJobStart, jobName, context);
12528
- }
12529
- this.emit("job_start", { jobName, executionId, startTime });
12530
- let attempt = 0;
12531
- let lastError = null;
12532
- let result = null;
12533
- let status = "success";
12534
- const isTestEnvironment = process.env.NODE_ENV === "test" || process.env.JEST_WORKER_ID !== void 0 || global.expect !== void 0;
12535
- while (attempt <= job.retries) {
12536
- try {
12537
- const actualTimeout = isTestEnvironment ? Math.min(job.timeout, 1e3) : job.timeout;
12538
- let timeoutId;
12539
- const timeoutPromise = new Promise((_, reject) => {
12540
- timeoutId = setTimeout(() => reject(new Error("Job execution timeout")), actualTimeout);
12541
- });
12542
- const jobPromise = job.action(this.database, context, this);
14095
+ try {
14096
+ if (this.config.onJobStart) {
14097
+ await this._executeHook(this.config.onJobStart, jobName, context);
14098
+ }
14099
+ this.emit("job_start", { jobName, executionId, startTime });
14100
+ let attempt = 0;
14101
+ let lastError = null;
14102
+ let result = null;
14103
+ let status = "success";
14104
+ const isTestEnvironment = this._isTestEnvironment();
14105
+ while (attempt <= job.retries) {
12543
14106
  try {
12544
- result = await Promise.race([jobPromise, timeoutPromise]);
12545
- clearTimeout(timeoutId);
12546
- } catch (raceError) {
12547
- clearTimeout(timeoutId);
12548
- throw raceError;
12549
- }
12550
- status = "success";
12551
- break;
12552
- } catch (error) {
12553
- lastError = error;
12554
- attempt++;
12555
- if (attempt <= job.retries) {
12556
- if (this.config.verbose) {
12557
- console.warn(`[SchedulerPlugin] Job '${jobName}' failed (attempt ${attempt + 1}):`, error.message);
14107
+ const actualTimeout = isTestEnvironment ? Math.min(job.timeout, 1e3) : job.timeout;
14108
+ let timeoutId;
14109
+ const timeoutPromise = new Promise((_, reject) => {
14110
+ timeoutId = setTimeout(() => reject(new Error("Job execution timeout")), actualTimeout);
14111
+ });
14112
+ const jobPromise = job.action(this.database, context, this);
14113
+ try {
14114
+ result = await Promise.race([jobPromise, timeoutPromise]);
14115
+ clearTimeout(timeoutId);
14116
+ } catch (raceError) {
14117
+ clearTimeout(timeoutId);
14118
+ throw raceError;
14119
+ }
14120
+ status = "success";
14121
+ break;
14122
+ } catch (error) {
14123
+ lastError = error;
14124
+ attempt++;
14125
+ if (attempt <= job.retries) {
14126
+ if (this.config.verbose) {
14127
+ console.warn(`[SchedulerPlugin] Job '${jobName}' failed (attempt ${attempt + 1}):`, error.message);
14128
+ }
14129
+ const baseDelay = Math.min(Math.pow(2, attempt) * 1e3, 5e3);
14130
+ const delay = isTestEnvironment ? 1 : baseDelay;
14131
+ await new Promise((resolve) => setTimeout(resolve, delay));
12558
14132
  }
12559
- const baseDelay = Math.min(Math.pow(2, attempt) * 1e3, 5e3);
12560
- const delay = isTestEnvironment ? 1 : baseDelay;
12561
- await new Promise((resolve) => setTimeout(resolve, delay));
12562
14133
  }
12563
14134
  }
12564
- }
12565
- const endTime = Date.now();
12566
- const duration = Math.max(1, endTime - startTime);
12567
- if (lastError && attempt > job.retries) {
12568
- status = lastError.message.includes("timeout") ? "timeout" : "error";
12569
- }
12570
- job.lastRun = new Date(endTime);
12571
- job.runCount++;
12572
- if (status === "success") {
12573
- job.successCount++;
12574
- } else {
12575
- job.errorCount++;
12576
- }
12577
- const stats = this.statistics.get(jobName);
12578
- stats.totalRuns++;
12579
- stats.lastRun = new Date(endTime);
12580
- if (status === "success") {
12581
- stats.totalSuccesses++;
12582
- stats.lastSuccess = new Date(endTime);
12583
- } else {
12584
- stats.totalErrors++;
12585
- stats.lastError = { time: new Date(endTime), message: lastError?.message };
12586
- }
12587
- stats.avgDuration = (stats.avgDuration * (stats.totalRuns - 1) + duration) / stats.totalRuns;
12588
- if (this.config.persistJobs) {
12589
- await this._persistJobExecution(jobName, executionId, startTime, endTime, duration, status, result, lastError, attempt);
12590
- }
12591
- if (status === "success" && this.config.onJobComplete) {
12592
- await this._executeHook(this.config.onJobComplete, jobName, result, duration);
12593
- } else if (status !== "success" && this.config.onJobError) {
12594
- await this._executeHook(this.config.onJobError, jobName, lastError, attempt);
12595
- }
12596
- this.emit("job_complete", {
12597
- jobName,
12598
- executionId,
12599
- status,
12600
- duration,
12601
- result,
12602
- error: lastError?.message,
12603
- retryCount: attempt
12604
- });
12605
- this.activeJobs.delete(jobName);
12606
- if (job.enabled) {
12607
- this._scheduleNextExecution(jobName);
12608
- }
12609
- if (lastError && status !== "success") {
12610
- throw lastError;
14135
+ const endTime = Date.now();
14136
+ const duration = Math.max(1, endTime - startTime);
14137
+ if (lastError && attempt > job.retries) {
14138
+ status = lastError.message.includes("timeout") ? "timeout" : "error";
14139
+ }
14140
+ job.lastRun = new Date(endTime);
14141
+ job.runCount++;
14142
+ if (status === "success") {
14143
+ job.successCount++;
14144
+ } else {
14145
+ job.errorCount++;
14146
+ }
14147
+ const stats = this.statistics.get(jobName);
14148
+ stats.totalRuns++;
14149
+ stats.lastRun = new Date(endTime);
14150
+ if (status === "success") {
14151
+ stats.totalSuccesses++;
14152
+ stats.lastSuccess = new Date(endTime);
14153
+ } else {
14154
+ stats.totalErrors++;
14155
+ stats.lastError = { time: new Date(endTime), message: lastError?.message };
14156
+ }
14157
+ stats.avgDuration = (stats.avgDuration * (stats.totalRuns - 1) + duration) / stats.totalRuns;
14158
+ if (this.config.persistJobs) {
14159
+ await this._persistJobExecution(jobName, executionId, startTime, endTime, duration, status, result, lastError, attempt);
14160
+ }
14161
+ if (status === "success" && this.config.onJobComplete) {
14162
+ await this._executeHook(this.config.onJobComplete, jobName, result, duration);
14163
+ } else if (status !== "success" && this.config.onJobError) {
14164
+ await this._executeHook(this.config.onJobError, jobName, lastError, attempt);
14165
+ }
14166
+ this.emit("job_complete", {
14167
+ jobName,
14168
+ executionId,
14169
+ status,
14170
+ duration,
14171
+ result,
14172
+ error: lastError?.message,
14173
+ retryCount: attempt
14174
+ });
14175
+ this.activeJobs.delete(jobName);
14176
+ if (job.enabled) {
14177
+ this._scheduleNextExecution(jobName);
14178
+ }
14179
+ if (lastError && status !== "success") {
14180
+ throw lastError;
14181
+ }
14182
+ } finally {
14183
+ await tryFn(() => this.lockResource.delete(lockId));
12611
14184
  }
12612
14185
  }
12613
14186
  async _persistJobExecution(jobName, executionId, startTime, endTime, duration, status, result, error, retryCount) {
@@ -12639,6 +14212,7 @@ class SchedulerPlugin extends Plugin {
12639
14212
  }
12640
14213
  /**
12641
14214
  * Manually trigger a job execution
14215
+ * Note: Race conditions are prevented by distributed locking in _executeJob()
12642
14216
  */
12643
14217
  async runJob(jobName, context = {}) {
12644
14218
  const job = this.jobs.get(jobName);
@@ -12724,12 +14298,15 @@ class SchedulerPlugin extends Plugin {
12724
14298
  return [];
12725
14299
  }
12726
14300
  const { limit = 50, status = null } = options;
12727
- const [ok, err, allHistory] = await tryFn(
12728
- () => this.database.resource(this.config.jobHistoryResource).list({
12729
- orderBy: { startTime: "desc" },
12730
- limit: limit * 2
12731
- // Get more to allow for filtering
12732
- })
14301
+ const queryParams = {
14302
+ jobName
14303
+ // Uses byJob partition for efficient lookup
14304
+ };
14305
+ if (status) {
14306
+ queryParams.status = status;
14307
+ }
14308
+ const [ok, err, history] = await tryFn(
14309
+ () => this.database.resource(this.config.jobHistoryResource).query(queryParams)
12733
14310
  );
12734
14311
  if (!ok) {
12735
14312
  if (this.config.verbose) {
@@ -12737,11 +14314,7 @@ class SchedulerPlugin extends Plugin {
12737
14314
  }
12738
14315
  return [];
12739
14316
  }
12740
- let filtered = allHistory.filter((h) => h.jobName === jobName);
12741
- if (status) {
12742
- filtered = filtered.filter((h) => h.status === status);
12743
- }
12744
- filtered = filtered.sort((a, b) => b.startTime - a.startTime).slice(0, limit);
14317
+ let filtered = history.sort((a, b) => b.startTime - a.startTime).slice(0, limit);
12745
14318
  return filtered.map((h) => {
12746
14319
  let result = null;
12747
14320
  if (h.result) {
@@ -12836,8 +14409,7 @@ class SchedulerPlugin extends Plugin {
12836
14409
  clearTimeout(timer);
12837
14410
  }
12838
14411
  this.timers.clear();
12839
- const isTestEnvironment = process.env.NODE_ENV === "test" || process.env.JEST_WORKER_ID !== void 0 || global.expect !== void 0;
12840
- if (!isTestEnvironment && this.activeJobs.size > 0) {
14412
+ if (!this._isTestEnvironment() && this.activeJobs.size > 0) {
12841
14413
  if (this.config.verbose) {
12842
14414
  console.log(`[SchedulerPlugin] Waiting for ${this.activeJobs.size} active jobs to complete...`);
12843
14415
  }
@@ -12850,7 +14422,7 @@ class SchedulerPlugin extends Plugin {
12850
14422
  console.warn(`[SchedulerPlugin] ${this.activeJobs.size} jobs still running after timeout`);
12851
14423
  }
12852
14424
  }
12853
- if (isTestEnvironment) {
14425
+ if (this._isTestEnvironment()) {
12854
14426
  this.activeJobs.clear();
12855
14427
  }
12856
14428
  }
@@ -12871,14 +14443,14 @@ class StateMachinePlugin extends Plugin {
12871
14443
  actions: options.actions || {},
12872
14444
  guards: options.guards || {},
12873
14445
  persistTransitions: options.persistTransitions !== false,
12874
- transitionLogResource: options.transitionLogResource || "state_transitions",
12875
- stateResource: options.stateResource || "entity_states",
12876
- verbose: options.verbose || false,
12877
- ...options
14446
+ transitionLogResource: options.transitionLogResource || "plg_state_transitions",
14447
+ stateResource: options.stateResource || "plg_entity_states",
14448
+ retryAttempts: options.retryAttempts || 3,
14449
+ retryDelay: options.retryDelay || 100,
14450
+ verbose: options.verbose || false
12878
14451
  };
12879
14452
  this.database = null;
12880
14453
  this.machines = /* @__PURE__ */ new Map();
12881
- this.stateStorage = /* @__PURE__ */ new Map();
12882
14454
  this._validateConfiguration();
12883
14455
  }
12884
14456
  _validateConfiguration() {
@@ -13019,43 +14591,55 @@ class StateMachinePlugin extends Plugin {
13019
14591
  machine.currentStates.set(entityId, toState);
13020
14592
  if (this.config.persistTransitions) {
13021
14593
  const transitionId = `${machineId}_${entityId}_${timestamp}`;
13022
- const [logOk, logErr] = await tryFn(
13023
- () => this.database.resource(this.config.transitionLogResource).insert({
13024
- id: transitionId,
13025
- machineId,
13026
- entityId,
13027
- fromState,
13028
- toState,
13029
- event,
13030
- context,
13031
- timestamp,
13032
- createdAt: now.slice(0, 10)
13033
- // YYYY-MM-DD for partitioning
13034
- })
13035
- );
14594
+ let logOk = false;
14595
+ let lastLogErr;
14596
+ for (let attempt = 0; attempt < this.config.retryAttempts; attempt++) {
14597
+ const [ok, err] = await tryFn(
14598
+ () => this.database.resource(this.config.transitionLogResource).insert({
14599
+ id: transitionId,
14600
+ machineId,
14601
+ entityId,
14602
+ fromState,
14603
+ toState,
14604
+ event,
14605
+ context,
14606
+ timestamp,
14607
+ createdAt: now.slice(0, 10)
14608
+ // YYYY-MM-DD for partitioning
14609
+ })
14610
+ );
14611
+ if (ok) {
14612
+ logOk = true;
14613
+ break;
14614
+ }
14615
+ lastLogErr = err;
14616
+ if (attempt < this.config.retryAttempts - 1) {
14617
+ const delay = this.config.retryDelay * Math.pow(2, attempt);
14618
+ await new Promise((resolve) => setTimeout(resolve, delay));
14619
+ }
14620
+ }
13036
14621
  if (!logOk && this.config.verbose) {
13037
- console.warn(`[StateMachinePlugin] Failed to log transition:`, logErr.message);
14622
+ console.warn(`[StateMachinePlugin] Failed to log transition after ${this.config.retryAttempts} attempts:`, lastLogErr.message);
13038
14623
  }
13039
14624
  const stateId = `${machineId}_${entityId}`;
13040
- const [stateOk, stateErr] = await tryFn(async () => {
13041
- const exists = await this.database.resource(this.config.stateResource).exists(stateId);
13042
- const stateData = {
13043
- id: stateId,
13044
- machineId,
13045
- entityId,
13046
- currentState: toState,
13047
- context,
13048
- lastTransition: transitionId,
13049
- updatedAt: now
13050
- };
13051
- if (exists) {
13052
- await this.database.resource(this.config.stateResource).update(stateId, stateData);
13053
- } else {
13054
- await this.database.resource(this.config.stateResource).insert(stateData);
14625
+ const stateData = {
14626
+ machineId,
14627
+ entityId,
14628
+ currentState: toState,
14629
+ context,
14630
+ lastTransition: transitionId,
14631
+ updatedAt: now
14632
+ };
14633
+ const [updateOk] = await tryFn(
14634
+ () => this.database.resource(this.config.stateResource).update(stateId, stateData)
14635
+ );
14636
+ if (!updateOk) {
14637
+ const [insertOk, insertErr] = await tryFn(
14638
+ () => this.database.resource(this.config.stateResource).insert({ id: stateId, ...stateData })
14639
+ );
14640
+ if (!insertOk && this.config.verbose) {
14641
+ console.warn(`[StateMachinePlugin] Failed to upsert state:`, insertErr.message);
13055
14642
  }
13056
- });
13057
- if (!stateOk && this.config.verbose) {
13058
- console.warn(`[StateMachinePlugin] Failed to update state:`, stateErr.message);
13059
14643
  }
13060
14644
  }
13061
14645
  }
@@ -13086,8 +14670,9 @@ class StateMachinePlugin extends Plugin {
13086
14670
  }
13087
14671
  /**
13088
14672
  * Get valid events for current state
14673
+ * Can accept either a state name (sync) or entityId (async to fetch latest state)
13089
14674
  */
13090
- getValidEvents(machineId, stateOrEntityId) {
14675
+ async getValidEvents(machineId, stateOrEntityId) {
13091
14676
  const machine = this.machines.get(machineId);
13092
14677
  if (!machine) {
13093
14678
  throw new Error(`State machine '${machineId}' not found`);
@@ -13096,7 +14681,7 @@ class StateMachinePlugin extends Plugin {
13096
14681
  if (machine.config.states[stateOrEntityId]) {
13097
14682
  state = stateOrEntityId;
13098
14683
  } else {
13099
- state = machine.currentStates.get(stateOrEntityId) || machine.config.initialState;
14684
+ state = await this.getState(machineId, stateOrEntityId);
13100
14685
  }
13101
14686
  const stateConfig = machine.config.states[state];
13102
14687
  return stateConfig && stateConfig.on ? Object.keys(stateConfig.on) : [];
@@ -13110,9 +14695,10 @@ class StateMachinePlugin extends Plugin {
13110
14695
  }
13111
14696
  const { limit = 50, offset = 0 } = options;
13112
14697
  const [ok, err, transitions] = await tryFn(
13113
- () => this.database.resource(this.config.transitionLogResource).list({
13114
- where: { machineId, entityId },
13115
- orderBy: { timestamp: "desc" },
14698
+ () => this.database.resource(this.config.transitionLogResource).query({
14699
+ machineId,
14700
+ entityId
14701
+ }, {
13116
14702
  limit,
13117
14703
  offset
13118
14704
  })
@@ -13123,8 +14709,8 @@ class StateMachinePlugin extends Plugin {
13123
14709
  }
13124
14710
  return [];
13125
14711
  }
13126
- const sortedTransitions = transitions.sort((a, b) => b.timestamp - a.timestamp);
13127
- return sortedTransitions.map((t) => ({
14712
+ const sorted = (transitions || []).sort((a, b) => b.timestamp - a.timestamp);
14713
+ return sorted.map((t) => ({
13128
14714
  from: t.fromState,
13129
14715
  to: t.toState,
13130
14716
  event: t.event,
@@ -13145,15 +14731,20 @@ class StateMachinePlugin extends Plugin {
13145
14731
  if (this.config.persistTransitions) {
13146
14732
  const now = (/* @__PURE__ */ new Date()).toISOString();
13147
14733
  const stateId = `${machineId}_${entityId}`;
13148
- await this.database.resource(this.config.stateResource).insert({
13149
- id: stateId,
13150
- machineId,
13151
- entityId,
13152
- currentState: initialState,
13153
- context,
13154
- lastTransition: null,
13155
- updatedAt: now
13156
- });
14734
+ const [ok, err] = await tryFn(
14735
+ () => this.database.resource(this.config.stateResource).insert({
14736
+ id: stateId,
14737
+ machineId,
14738
+ entityId,
14739
+ currentState: initialState,
14740
+ context,
14741
+ lastTransition: null,
14742
+ updatedAt: now
14743
+ })
14744
+ );
14745
+ if (!ok && err && !err.message?.includes("already exists")) {
14746
+ throw new Error(`Failed to initialize entity state: ${err.message}`);
14747
+ }
13157
14748
  }
13158
14749
  const initialStateConfig = machine.config.states[initialState];
13159
14750
  if (initialStateConfig && initialStateConfig.entry) {
@@ -13218,7 +14809,6 @@ class StateMachinePlugin extends Plugin {
13218
14809
  }
13219
14810
  async stop() {
13220
14811
  this.machines.clear();
13221
- this.stateStorage.clear();
13222
14812
  }
13223
14813
  async cleanup() {
13224
14814
  await this.stop();
@@ -13254,6 +14844,7 @@ exports.PartitionError = PartitionError;
13254
14844
  exports.PermissionError = PermissionError;
13255
14845
  exports.Plugin = Plugin;
13256
14846
  exports.PluginObject = PluginObject;
14847
+ exports.QueueConsumerPlugin = QueueConsumerPlugin;
13257
14848
  exports.ReplicatorPlugin = ReplicatorPlugin;
13258
14849
  exports.Resource = Resource;
13259
14850
  exports.ResourceError = ResourceError;
@@ -13262,6 +14853,7 @@ exports.ResourceIdsReader = ResourceIdsReader;
13262
14853
  exports.ResourceNotFound = ResourceNotFound;
13263
14854
  exports.ResourceReader = ResourceReader;
13264
14855
  exports.ResourceWriter = ResourceWriter;
14856
+ exports.S3QueuePlugin = S3QueuePlugin;
13265
14857
  exports.S3db = Database;
13266
14858
  exports.S3dbError = S3dbError;
13267
14859
  exports.SchedulerPlugin = SchedulerPlugin;