s3db.js 11.0.2 → 11.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/s3db.es.js CHANGED
@@ -833,18 +833,23 @@ class PluginStorage {
833
833
  return `plugin=${this.pluginSlug}/${parts.join("/")}`;
834
834
  }
835
835
  /**
836
- * Save data with metadata encoding and behavior support
836
+ * Save data with metadata encoding, behavior support, and optional TTL
837
837
  *
838
838
  * @param {string} key - S3 key
839
839
  * @param {Object} data - Data to save
840
840
  * @param {Object} options - Options
841
+ * @param {number} options.ttl - Time-to-live in seconds (optional)
841
842
  * @param {string} options.behavior - 'body-overflow' | 'body-only' | 'enforce-limits'
842
843
  * @param {string} options.contentType - Content type (default: application/json)
843
844
  * @returns {Promise<void>}
844
845
  */
845
- async put(key, data, options = {}) {
846
- const { behavior = "body-overflow", contentType = "application/json" } = options;
847
- const { metadata, body } = this._applyBehavior(data, behavior);
846
+ async set(key, data, options = {}) {
847
+ const { ttl, behavior = "body-overflow", contentType = "application/json" } = options;
848
+ const dataToSave = { ...data };
849
+ if (ttl && typeof ttl === "number" && ttl > 0) {
850
+ dataToSave._expiresAt = Date.now() + ttl * 1e3;
851
+ }
852
+ const { metadata, body } = this._applyBehavior(dataToSave, behavior);
848
853
  const putParams = {
849
854
  key,
850
855
  metadata,
@@ -855,14 +860,21 @@ class PluginStorage {
855
860
  }
856
861
  const [ok, err] = await tryFn(() => this.client.putObject(putParams));
857
862
  if (!ok) {
858
- throw new Error(`PluginStorage.put failed for key ${key}: ${err.message}`);
863
+ throw new Error(`PluginStorage.set failed for key ${key}: ${err.message}`);
859
864
  }
860
865
  }
861
866
  /**
862
- * Get data with automatic metadata decoding
867
+ * Alias for set() to maintain backward compatibility
868
+ * @deprecated Use set() instead
869
+ */
870
+ async put(key, data, options = {}) {
871
+ return this.set(key, data, options);
872
+ }
873
+ /**
874
+ * Get data with automatic metadata decoding and TTL check
863
875
  *
864
876
  * @param {string} key - S3 key
865
- * @returns {Promise<Object|null>} Data or null if not found
877
+ * @returns {Promise<Object|null>} Data or null if not found/expired
866
878
  */
867
879
  async get(key) {
868
880
  const [ok, err, response] = await tryFn(() => this.client.getObject(key));
@@ -874,18 +886,28 @@ class PluginStorage {
874
886
  }
875
887
  const metadata = response.Metadata || {};
876
888
  const parsedMetadata = this._parseMetadataValues(metadata);
889
+ let data = parsedMetadata;
877
890
  if (response.Body) {
878
891
  try {
879
892
  const bodyContent = await response.Body.transformToString();
880
893
  if (bodyContent && bodyContent.trim()) {
881
894
  const body = JSON.parse(bodyContent);
882
- return { ...parsedMetadata, ...body };
895
+ data = { ...parsedMetadata, ...body };
883
896
  }
884
897
  } catch (parseErr) {
885
898
  throw new Error(`PluginStorage.get failed to parse body for key ${key}: ${parseErr.message}`);
886
899
  }
887
900
  }
888
- return parsedMetadata;
901
+ const expiresAt = data._expiresat || data._expiresAt;
902
+ if (expiresAt) {
903
+ if (Date.now() > expiresAt) {
904
+ await this.delete(key);
905
+ return null;
906
+ }
907
+ delete data._expiresat;
908
+ delete data._expiresAt;
909
+ }
910
+ return data;
889
911
  }
890
912
  /**
891
913
  * Parse metadata values back to their original types
@@ -968,6 +990,123 @@ class PluginStorage {
968
990
  if (!keyPrefix) return keys;
969
991
  return keys.map((key) => key.replace(keyPrefix, "")).map((key) => key.startsWith("/") ? key.replace("/", "") : key);
970
992
  }
993
+ /**
994
+ * Check if a key exists (not expired)
995
+ *
996
+ * @param {string} key - S3 key
997
+ * @returns {Promise<boolean>} True if exists and not expired
998
+ */
999
+ async has(key) {
1000
+ const data = await this.get(key);
1001
+ return data !== null;
1002
+ }
1003
+ /**
1004
+ * Check if a key is expired
1005
+ *
1006
+ * @param {string} key - S3 key
1007
+ * @returns {Promise<boolean>} True if expired or not found
1008
+ */
1009
+ async isExpired(key) {
1010
+ const [ok, err, response] = await tryFn(() => this.client.getObject(key));
1011
+ if (!ok) {
1012
+ return true;
1013
+ }
1014
+ const metadata = response.Metadata || {};
1015
+ const parsedMetadata = this._parseMetadataValues(metadata);
1016
+ let data = parsedMetadata;
1017
+ if (response.Body) {
1018
+ try {
1019
+ const bodyContent = await response.Body.transformToString();
1020
+ if (bodyContent && bodyContent.trim()) {
1021
+ const body = JSON.parse(bodyContent);
1022
+ data = { ...parsedMetadata, ...body };
1023
+ }
1024
+ } catch {
1025
+ return true;
1026
+ }
1027
+ }
1028
+ const expiresAt = data._expiresat || data._expiresAt;
1029
+ if (!expiresAt) {
1030
+ return false;
1031
+ }
1032
+ return Date.now() > expiresAt;
1033
+ }
1034
+ /**
1035
+ * Get remaining TTL in seconds
1036
+ *
1037
+ * @param {string} key - S3 key
1038
+ * @returns {Promise<number|null>} Remaining seconds or null if no TTL/not found
1039
+ */
1040
+ async getTTL(key) {
1041
+ const [ok, err, response] = await tryFn(() => this.client.getObject(key));
1042
+ if (!ok) {
1043
+ return null;
1044
+ }
1045
+ const metadata = response.Metadata || {};
1046
+ const parsedMetadata = this._parseMetadataValues(metadata);
1047
+ let data = parsedMetadata;
1048
+ if (response.Body) {
1049
+ try {
1050
+ const bodyContent = await response.Body.transformToString();
1051
+ if (bodyContent && bodyContent.trim()) {
1052
+ const body = JSON.parse(bodyContent);
1053
+ data = { ...parsedMetadata, ...body };
1054
+ }
1055
+ } catch {
1056
+ return null;
1057
+ }
1058
+ }
1059
+ const expiresAt = data._expiresat || data._expiresAt;
1060
+ if (!expiresAt) {
1061
+ return null;
1062
+ }
1063
+ const remaining = Math.max(0, expiresAt - Date.now());
1064
+ return Math.floor(remaining / 1e3);
1065
+ }
1066
+ /**
1067
+ * Extend TTL by adding additional seconds
1068
+ *
1069
+ * @param {string} key - S3 key
1070
+ * @param {number} additionalSeconds - Seconds to add to current TTL
1071
+ * @returns {Promise<boolean>} True if extended, false if not found or no TTL
1072
+ */
1073
+ async touch(key, additionalSeconds) {
1074
+ const [ok, err, response] = await tryFn(() => this.client.getObject(key));
1075
+ if (!ok) {
1076
+ return false;
1077
+ }
1078
+ const metadata = response.Metadata || {};
1079
+ const parsedMetadata = this._parseMetadataValues(metadata);
1080
+ let data = parsedMetadata;
1081
+ if (response.Body) {
1082
+ try {
1083
+ const bodyContent = await response.Body.transformToString();
1084
+ if (bodyContent && bodyContent.trim()) {
1085
+ const body = JSON.parse(bodyContent);
1086
+ data = { ...parsedMetadata, ...body };
1087
+ }
1088
+ } catch {
1089
+ return false;
1090
+ }
1091
+ }
1092
+ const expiresAt = data._expiresat || data._expiresAt;
1093
+ if (!expiresAt) {
1094
+ return false;
1095
+ }
1096
+ data._expiresAt = expiresAt + additionalSeconds * 1e3;
1097
+ delete data._expiresat;
1098
+ const { metadata: newMetadata, body: newBody } = this._applyBehavior(data, "body-overflow");
1099
+ const putParams = {
1100
+ key,
1101
+ metadata: newMetadata,
1102
+ contentType: "application/json"
1103
+ };
1104
+ if (newBody !== null) {
1105
+ putParams.body = JSON.stringify(newBody);
1106
+ }
1107
+ const [putOk] = await tryFn(() => this.client.putObject(putParams));
1108
+ return putOk;
1109
+ }
971
1110
  /**
972
1111
  * Delete a single object
973
1112
  *
@@ -1045,6 +1184,78 @@ class PluginStorage {
1045
1184
  }
1046
1185
  return results;
1047
1186
  }
1187
+ /**
1188
+ * Acquire a distributed lock with TTL and retry logic
1189
+ *
1190
+ * @param {string} lockName - Lock identifier
1191
+ * @param {Object} options - Lock options
1192
+ * @param {number} options.ttl - Lock TTL in seconds (default: 30)
1193
+ * @param {number} options.timeout - Max wait time in ms (default: 0, no wait)
1194
+ * @param {string} options.workerId - Worker identifier (default: 'unknown')
1195
+ * @returns {Promise<Object|null>} Lock object or null if couldn't acquire
1196
+ */
1197
+ async acquireLock(lockName, options = {}) {
1198
+ const { ttl = 30, timeout = 0, workerId = "unknown" } = options;
1199
+ const key = this.getPluginKey(null, "locks", lockName);
1200
+ const startTime = Date.now();
1201
+ while (true) {
1202
+ const existing = await this.get(key);
1203
+ if (!existing) {
1204
+ await this.set(key, { workerId, acquiredAt: Date.now() }, { ttl });
1205
+ return { key, workerId };
1206
+ }
1207
+ if (Date.now() - startTime >= timeout) {
1208
+ return null;
1209
+ }
1210
+ await new Promise((resolve) => setTimeout(resolve, 100));
1211
+ }
1212
+ }
1213
+ /**
1214
+ * Release a distributed lock
1215
+ *
1216
+ * @param {string} lockName - Lock identifier
1217
+ * @returns {Promise<void>}
1218
+ */
1219
+ async releaseLock(lockName) {
1220
+ const key = this.getPluginKey(null, "locks", lockName);
1221
+ await this.delete(key);
1222
+ }
1223
+ /**
1224
+ * Check if a lock is currently held
1225
+ *
1226
+ * @param {string} lockName - Lock identifier
1227
+ * @returns {Promise<boolean>} True if locked
1228
+ */
1229
+ async isLocked(lockName) {
1230
+ const key = this.getPluginKey(null, "locks", lockName);
1231
+ const lock = await this.get(key);
1232
+ return lock !== null;
1233
+ }
1234
+ /**
1235
+ * Increment a counter value
1236
+ *
1237
+ * @param {string} key - S3 key
1238
+ * @param {number} amount - Amount to increment (default: 1)
1239
+ * @param {Object} options - Options (e.g., ttl)
1240
+ * @returns {Promise<number>} New value
1241
+ */
1242
+ async increment(key, amount = 1, options = {}) {
1243
+ const data = await this.get(key);
1244
+ const value = (data?.value || 0) + amount;
1245
+ await this.set(key, { value }, options);
1246
+ return value;
1247
+ }
1248
+ /**
1249
+ * Decrement a counter value
1250
+ *
1251
+ * @param {string} key - S3 key
1252
+ * @param {number} amount - Amount to decrement (default: 1)
1253
+ * @param {Object} options - Options (e.g., ttl)
1254
+ * @returns {Promise<number>} New value
1255
+ */
1256
+ async decrement(key, amount = 1, options = {}) {
1257
+ return this.increment(key, -amount, options);
1258
+ }
1048
1259
  /**
1049
1260
  * Apply behavior to split data between metadata and body
1050
1261
  *
@@ -1348,12 +1559,18 @@ class AuditPlugin extends Plugin {
1348
1559
  recordId: "string|required",
1349
1560
  userId: "string|optional",
1350
1561
  timestamp: "string|required",
1562
+ createdAt: "string|required",
1563
+ // YYYY-MM-DD for partitioning
1351
1564
  oldData: "string|optional",
1352
1565
  newData: "string|optional",
1353
1566
  partition: "string|optional",
1354
1567
  partitionValues: "string|optional",
1355
1568
  metadata: "string|optional"
1356
1569
  },
1570
+ partitions: {
1571
+ byDate: { fields: { createdAt: "string|maxlength:10" } },
1572
+ byResource: { fields: { resourceName: "string" } }
1573
+ },
1357
1574
  behavior: "body-overflow"
1358
1575
  }));
1359
1576
  this.auditResource = ok ? auditResource : this.database.resources.plg_audits || null;
@@ -1457,10 +1674,13 @@ class AuditPlugin extends Plugin {
1457
1674
  if (!this.auditResource) {
1458
1675
  return;
1459
1676
  }
1677
+ const now = /* @__PURE__ */ new Date();
1460
1678
  const auditRecord = {
1461
1679
  id: `audit-${Date.now()}-${Math.random().toString(36).substring(2, 11)}`,
1462
1680
  userId: this.getCurrentUserId?.() || "system",
1463
- timestamp: (/* @__PURE__ */ new Date()).toISOString(),
1681
+ timestamp: now.toISOString(),
1682
+ createdAt: now.toISOString().slice(0, 10),
1683
+ // YYYY-MM-DD for partitioning
1464
1684
  metadata: JSON.stringify({ source: "audit-plugin", version: "2.0" }),
1465
1685
  resourceName: auditData.resourceName,
1466
1686
  operation: auditData.operation,
@@ -1535,9 +1755,25 @@ class AuditPlugin extends Plugin {
1535
1755
  async getAuditLogs(options = {}) {
1536
1756
  if (!this.auditResource) return [];
1537
1757
  const { resourceName, operation, recordId, partition, startDate, endDate, limit = 100, offset = 0 } = options;
1538
- const hasFilters = resourceName || operation || recordId || partition || startDate || endDate;
1539
1758
  let items = [];
1540
- if (hasFilters) {
1759
+ if (resourceName && !operation && !recordId && !partition && !startDate && !endDate) {
1760
+ const [ok, err, result] = await tryFn(
1761
+ () => this.auditResource.query({ resourceName }, { limit: limit + offset })
1762
+ );
1763
+ items = ok && result ? result : [];
1764
+ return items.slice(offset, offset + limit);
1765
+ } else if (startDate && !resourceName && !operation && !recordId && !partition) {
1766
+ const dates = this._generateDateRange(startDate, endDate);
1767
+ for (const date of dates) {
1768
+ const [ok, err, result] = await tryFn(
1769
+ () => this.auditResource.query({ createdAt: date })
1770
+ );
1771
+ if (ok && result) {
1772
+ items.push(...result);
1773
+ }
1774
+ }
1775
+ return items.slice(offset, offset + limit);
1776
+ } else if (resourceName || operation || recordId || partition || startDate || endDate) {
1541
1777
  const fetchSize = Math.min(1e4, Math.max(1e3, (limit + offset) * 20));
1542
1778
  const result = await this.auditResource.list({ limit: fetchSize });
1543
1779
  items = result || [];
@@ -1567,6 +1803,15 @@ class AuditPlugin extends Plugin {
1567
1803
  return result.items || [];
1568
1804
  }
1569
1805
  }
1806
+ _generateDateRange(startDate, endDate) {
1807
+ const dates = [];
1808
+ const start = new Date(startDate);
1809
+ const end = endDate ? new Date(endDate) : /* @__PURE__ */ new Date();
1810
+ for (let d = new Date(start); d <= end; d.setDate(d.getDate() + 1)) {
1811
+ dates.push(d.toISOString().slice(0, 10));
1812
+ }
1813
+ return dates;
1814
+ }
1570
1815
  async getRecordHistory(resourceName, recordId) {
1571
1816
  return await this.getAuditLogs({ resourceName, recordId });
1572
1817
  }
@@ -1599,6 +1844,37 @@ class AuditPlugin extends Plugin {
1599
1844
  }
1600
1845
  return stats;
1601
1846
  }
1847
+ /**
1848
+ * Clean up audit logs older than retention period
1849
+ * @param {number} retentionDays - Number of days to retain (default: 90)
1850
+ * @returns {Promise<number>} Number of records deleted
1851
+ */
1852
+ async cleanupOldAudits(retentionDays = 90) {
1853
+ if (!this.auditResource) return 0;
1854
+ const cutoffDate = /* @__PURE__ */ new Date();
1855
+ cutoffDate.setDate(cutoffDate.getDate() - retentionDays);
1856
+ const datesToDelete = [];
1857
+ const startDate = new Date(cutoffDate);
1858
+ startDate.setDate(startDate.getDate() - 365);
1859
+ for (let d = new Date(startDate); d < cutoffDate; d.setDate(d.getDate() + 1)) {
1860
+ datesToDelete.push(d.toISOString().slice(0, 10));
1861
+ }
1862
+ let deletedCount = 0;
1863
+ for (const dateStr of datesToDelete) {
1864
+ const [ok, err, oldAudits] = await tryFn(
1865
+ () => this.auditResource.query({ createdAt: dateStr })
1866
+ );
1867
+ if (ok && oldAudits) {
1868
+ for (const audit of oldAudits) {
1869
+ const [delOk] = await tryFn(() => this.auditResource.delete(audit.id));
1870
+ if (delOk) {
1871
+ deletedCount++;
1872
+ }
1873
+ }
1874
+ }
1875
+ }
1876
+ return deletedCount;
1877
+ }
1602
1878
  }
1603
1879
 
1604
1880
  class BaseBackupDriver {
@@ -4834,6 +5110,8 @@ function createConfig(options, detectedTimezone) {
4834
5110
  consolidationWindow: consolidation.window ?? 24,
4835
5111
  autoConsolidate: consolidation.auto !== false,
4836
5112
  mode: consolidation.mode || "async",
5113
+ // ✅ NOVO: Performance tuning - Mark applied concurrency (default 50, antes era 10 hardcoded)
5114
+ markAppliedConcurrency: consolidation.markAppliedConcurrency ?? 50,
4837
5115
  // Late arrivals
4838
5116
  lateArrivalStrategy: lateArrivals.strategy || "warn",
4839
5117
  // Batch transactions
@@ -4937,6 +5215,21 @@ function getTimezoneOffset(timezone, verbose = false) {
4937
5215
  return offsets[timezone] || 0;
4938
5216
  }
4939
5217
  }
5218
+ function getISOWeek(date) {
5219
+ const target = new Date(date.valueOf());
5220
+ const dayNr = (date.getUTCDay() + 6) % 7;
5221
+ target.setUTCDate(target.getUTCDate() - dayNr + 3);
5222
+ const yearStart = new Date(Date.UTC(target.getUTCFullYear(), 0, 1));
5223
+ const firstThursday = new Date(yearStart.valueOf());
5224
+ if (yearStart.getUTCDay() !== 4) {
5225
+ firstThursday.setUTCDate(yearStart.getUTCDate() + (4 - yearStart.getUTCDay() + 7) % 7);
5226
+ }
5227
+ const weekNumber = 1 + Math.round((target - firstThursday) / 6048e5);
5228
+ return {
5229
+ year: target.getUTCFullYear(),
5230
+ week: weekNumber
5231
+ };
5232
+ }
4940
5233
  function getCohortInfo(date, timezone, verbose = false) {
4941
5234
  const offset = getTimezoneOffset(timezone, verbose);
4942
5235
  const localDate = new Date(date.getTime() + offset);
@@ -4944,10 +5237,14 @@ function getCohortInfo(date, timezone, verbose = false) {
4944
5237
  const month = String(localDate.getMonth() + 1).padStart(2, "0");
4945
5238
  const day = String(localDate.getDate()).padStart(2, "0");
4946
5239
  const hour = String(localDate.getHours()).padStart(2, "0");
5240
+ const { year: weekYear, week: weekNumber } = getISOWeek(localDate);
5241
+ const week = `${weekYear}-W${String(weekNumber).padStart(2, "0")}`;
4947
5242
  return {
4948
5243
  date: `${year}-${month}-${day}`,
4949
5244
  hour: `${year}-${month}-${day}T${hour}`,
4950
5245
  // ISO-like format for hour partition
5246
+ week,
5247
+ // ISO 8601 week format (e.g., '2025-W42')
4951
5248
  month: `${year}-${month}`
4952
5249
  };
4953
5250
  }
@@ -5025,6 +5322,11 @@ function createPartitionConfig() {
5025
5322
  cohortDate: "string"
5026
5323
  }
5027
5324
  },
5325
+ byWeek: {
5326
+ fields: {
5327
+ cohortWeek: "string"
5328
+ }
5329
+ },
5028
5330
  byMonth: {
5029
5331
  fields: {
5030
5332
  cohortMonth: "string"
@@ -5064,6 +5366,7 @@ async function createTransaction(handler, data, config) {
5064
5366
  timestamp: now.toISOString(),
5065
5367
  cohortDate: cohortInfo.date,
5066
5368
  cohortHour: cohortInfo.hour,
5369
+ cohortWeek: cohortInfo.week,
5067
5370
  cohortMonth: cohortInfo.month,
5068
5371
  source: data.source || "unknown",
5069
5372
  applied: false
@@ -5104,50 +5407,6 @@ async function flushPendingTransactions(handler) {
5104
5407
  }
5105
5408
  }
5106
5409
 
5107
- async function cleanupStaleLocks(lockResource, config) {
5108
- const now = Date.now();
5109
- const lockTimeoutMs = config.lockTimeout * 1e3;
5110
- const cutoffTime = now - lockTimeoutMs;
5111
- const cleanupLockId = `lock-cleanup-${config.resource}-${config.field}`;
5112
- const [lockAcquired] = await tryFn(
5113
- () => lockResource.insert({
5114
- id: cleanupLockId,
5115
- lockedAt: Date.now(),
5116
- workerId: process.pid ? String(process.pid) : "unknown"
5117
- })
5118
- );
5119
- if (!lockAcquired) {
5120
- if (config.verbose) {
5121
- console.log(`[EventualConsistency] Lock cleanup already running in another container`);
5122
- }
5123
- return;
5124
- }
5125
- try {
5126
- const [ok, err, locks] = await tryFn(() => lockResource.list());
5127
- if (!ok || !locks || locks.length === 0) return;
5128
- const staleLocks = locks.filter(
5129
- (lock) => lock.id !== cleanupLockId && lock.lockedAt < cutoffTime
5130
- );
5131
- if (staleLocks.length === 0) return;
5132
- if (config.verbose) {
5133
- console.log(`[EventualConsistency] Cleaning up ${staleLocks.length} stale locks`);
5134
- }
5135
- const { results, errors } = await PromisePool.for(staleLocks).withConcurrency(5).process(async (lock) => {
5136
- const [deleted] = await tryFn(() => lockResource.delete(lock.id));
5137
- return deleted;
5138
- });
5139
- if (errors && errors.length > 0 && config.verbose) {
5140
- console.warn(`[EventualConsistency] ${errors.length} stale locks failed to delete`);
5141
- }
5142
- } catch (error) {
5143
- if (config.verbose) {
5144
- console.warn(`[EventualConsistency] Error cleaning up stale locks:`, error.message);
5145
- }
5146
- } finally {
5147
- await tryFn(() => lockResource.delete(cleanupLockId));
5148
- }
5149
- }
5150
-
5151
5410
  function startConsolidationTimer(handler, resourceName, fieldName, runConsolidationCallback, config) {
5152
5411
  const intervalMs = config.consolidationInterval * 1e3;
5153
5412
  if (config.verbose) {
@@ -5244,17 +5503,15 @@ async function runConsolidation(transactionResource, consolidateRecordFn, emitFn
5244
5503
  }
5245
5504
  }
5246
5505
  }
5247
- async function consolidateRecord(originalId, transactionResource, targetResource, lockResource, analyticsResource, updateAnalyticsFn, config) {
5248
- await cleanupStaleLocks(lockResource, config);
5249
- const lockId = `lock-${originalId}`;
5250
- const [lockAcquired, lockErr, lock] = await tryFn(
5251
- () => lockResource.insert({
5252
- id: lockId,
5253
- lockedAt: Date.now(),
5254
- workerId: process.pid ? String(process.pid) : "unknown"
5255
- })
5256
- );
5257
- if (!lockAcquired) {
5506
+ async function consolidateRecord(originalId, transactionResource, targetResource, storage, analyticsResource, updateAnalyticsFn, config) {
5507
+ const lockKey = `consolidation-${config.resource}-${config.field}-${originalId}`;
5508
+ const lock = await storage.acquireLock(lockKey, {
5509
+ ttl: config.lockTimeout || 30,
5510
+ timeout: 0,
5511
+ // Don't wait if locked
5512
+ workerId: process.pid ? String(process.pid) : "unknown"
5513
+ });
5514
+ if (!lock) {
5258
5515
  if (config.verbose) {
5259
5516
  console.log(`[EventualConsistency] Lock for ${originalId} already held, skipping`);
5260
5517
  }
@@ -5465,7 +5722,8 @@ async function consolidateRecord(originalId, transactionResource, targetResource
5465
5722
  }
5466
5723
  if (updateOk) {
5467
5724
  const transactionsToUpdate = transactions.filter((txn) => txn.id !== "__synthetic__");
5468
- const { results, errors } = await PromisePool.for(transactionsToUpdate).withConcurrency(10).process(async (txn) => {
5725
+ const markAppliedConcurrency = config.markAppliedConcurrency || 50;
5726
+ const { results, errors } = await PromisePool.for(transactionsToUpdate).withConcurrency(markAppliedConcurrency).process(async (txn) => {
5469
5727
  const [ok2, err2] = await tryFn(
5470
5728
  () => transactionResource.update(txn.id, { applied: true })
5471
5729
  );
@@ -5513,9 +5771,11 @@ async function consolidateRecord(originalId, transactionResource, targetResource
5513
5771
  }
5514
5772
  return consolidatedValue;
5515
5773
  } finally {
5516
- const [lockReleased, lockReleaseErr] = await tryFn(() => lockResource.delete(lockId));
5774
+ const [lockReleased, lockReleaseErr] = await tryFn(
5775
+ () => storage.releaseLock(lockKey)
5776
+ );
5517
5777
  if (!lockReleased && config.verbose) {
5518
- console.warn(`[EventualConsistency] Failed to release lock ${lockId}:`, lockReleaseErr?.message);
5778
+ console.warn(`[EventualConsistency] Failed to release lock ${lockKey}:`, lockReleaseErr?.message);
5519
5779
  }
5520
5780
  }
5521
5781
  }
@@ -5589,17 +5849,15 @@ async function getCohortStats(cohortDate, transactionResource) {
5589
5849
  }
5590
5850
  return stats;
5591
5851
  }
5592
- async function recalculateRecord(originalId, transactionResource, targetResource, lockResource, consolidateRecordFn, config) {
5593
- await cleanupStaleLocks(lockResource, config);
5594
- const lockId = `lock-recalculate-${originalId}`;
5595
- const [lockAcquired, lockErr, lock] = await tryFn(
5596
- () => lockResource.insert({
5597
- id: lockId,
5598
- lockedAt: Date.now(),
5599
- workerId: process.pid ? String(process.pid) : "unknown"
5600
- })
5601
- );
5602
- if (!lockAcquired) {
5852
+ async function recalculateRecord(originalId, transactionResource, targetResource, storage, consolidateRecordFn, config) {
5853
+ const lockKey = `recalculate-${config.resource}-${config.field}-${originalId}`;
5854
+ const lock = await storage.acquireLock(lockKey, {
5855
+ ttl: config.lockTimeout || 30,
5856
+ timeout: 0,
5857
+ // Don't wait if locked
5858
+ workerId: process.pid ? String(process.pid) : "unknown"
5859
+ });
5860
+ if (!lock) {
5603
5861
  if (config.verbose) {
5604
5862
  console.log(`[EventualConsistency] Recalculate lock for ${originalId} already held, skipping`);
5605
5863
  }
@@ -5667,9 +5925,11 @@ async function recalculateRecord(originalId, transactionResource, targetResource
5667
5925
  }
5668
5926
  return consolidatedValue;
5669
5927
  } finally {
5670
- const [lockReleased, lockReleaseErr] = await tryFn(() => lockResource.delete(lockId));
5928
+ const [lockReleased, lockReleaseErr] = await tryFn(
5929
+ () => storage.releaseLock(lockKey)
5930
+ );
5671
5931
  if (!lockReleased && config.verbose) {
5672
- console.warn(`[EventualConsistency] Failed to release recalculate lock ${lockId}:`, lockReleaseErr?.message);
5932
+ console.warn(`[EventualConsistency] Failed to release recalculate lock ${lockKey}:`, lockReleaseErr?.message);
5673
5933
  }
5674
5934
  }
5675
5935
  }
@@ -5681,16 +5941,16 @@ function startGarbageCollectionTimer(handler, resourceName, fieldName, runGCCall
5681
5941
  }, gcIntervalMs);
5682
5942
  return handler.gcTimer;
5683
5943
  }
5684
- async function runGarbageCollection(transactionResource, lockResource, config, emitFn) {
5685
- const gcLockId = `lock-gc-${config.resource}-${config.field}`;
5686
- const [lockAcquired] = await tryFn(
5687
- () => lockResource.insert({
5688
- id: gcLockId,
5689
- lockedAt: Date.now(),
5690
- workerId: process.pid ? String(process.pid) : "unknown"
5691
- })
5692
- );
5693
- if (!lockAcquired) {
5944
+ async function runGarbageCollection(transactionResource, storage, config, emitFn) {
5945
+ const lockKey = `gc-${config.resource}-${config.field}`;
5946
+ const lock = await storage.acquireLock(lockKey, {
5947
+ ttl: 300,
5948
+ // 5 minutes for GC
5949
+ timeout: 0,
5950
+ // Don't wait if locked
5951
+ workerId: process.pid ? String(process.pid) : "unknown"
5952
+ });
5953
+ if (!lock) {
5694
5954
  if (config.verbose) {
5695
5955
  console.log(`[EventualConsistency] GC already running in another container`);
5696
5956
  }
@@ -5748,7 +6008,7 @@ async function runGarbageCollection(transactionResource, lockResource, config, e
5748
6008
  emitFn("eventual-consistency.gc-error", error);
5749
6009
  }
5750
6010
  } finally {
5751
- await tryFn(() => lockResource.delete(gcLockId));
6011
+ await tryFn(() => storage.releaseLock(lockKey));
5752
6012
  }
5753
6013
  }
5754
6014
 
@@ -5773,22 +6033,26 @@ AnalyticsResource: ${analyticsResource?.name || "unknown"}`
5773
6033
  const cohortCount = Object.keys(byHour).length;
5774
6034
  if (config.verbose) {
5775
6035
  console.log(
5776
- `[EventualConsistency] ${config.resource}.${config.field} - Updating ${cohortCount} hourly analytics cohorts...`
6036
+ `[EventualConsistency] ${config.resource}.${config.field} - Updating ${cohortCount} hourly analytics cohorts IN PARALLEL...`
5777
6037
  );
5778
6038
  }
5779
- for (const [cohort, txns] of Object.entries(byHour)) {
5780
- await upsertAnalytics("hour", cohort, txns, analyticsResource, config);
5781
- }
6039
+ await Promise.all(
6040
+ Object.entries(byHour).map(
6041
+ ([cohort, txns]) => upsertAnalytics("hour", cohort, txns, analyticsResource, config)
6042
+ )
6043
+ );
5782
6044
  if (config.analyticsConfig.rollupStrategy === "incremental") {
5783
6045
  const uniqueHours = Object.keys(byHour);
5784
6046
  if (config.verbose) {
5785
6047
  console.log(
5786
- `[EventualConsistency] ${config.resource}.${config.field} - Rolling up ${uniqueHours.length} hours to daily/monthly analytics...`
6048
+ `[EventualConsistency] ${config.resource}.${config.field} - Rolling up ${uniqueHours.length} hours to daily/weekly/monthly analytics IN PARALLEL...`
5787
6049
  );
5788
6050
  }
5789
- for (const cohortHour of uniqueHours) {
5790
- await rollupAnalytics(cohortHour, analyticsResource, config);
5791
- }
6051
+ await Promise.all(
6052
+ uniqueHours.map(
6053
+ (cohortHour) => rollupAnalytics(cohortHour, analyticsResource, config)
6054
+ )
6055
+ );
5792
6056
  }
5793
6057
  if (config.verbose) {
5794
6058
  console.log(
@@ -5891,18 +6155,53 @@ function calculateOperationBreakdown(transactions) {
5891
6155
  async function rollupAnalytics(cohortHour, analyticsResource, config) {
5892
6156
  const cohortDate = cohortHour.substring(0, 10);
5893
6157
  const cohortMonth = cohortHour.substring(0, 7);
6158
+ const date = new Date(cohortDate);
6159
+ const cohortWeek = getCohortWeekFromDate(date);
5894
6160
  await rollupPeriod("day", cohortDate, cohortDate, analyticsResource, config);
6161
+ await rollupPeriod("week", cohortWeek, cohortWeek, analyticsResource, config);
5895
6162
  await rollupPeriod("month", cohortMonth, cohortMonth, analyticsResource, config);
5896
6163
  }
6164
+ function getCohortWeekFromDate(date) {
6165
+ const target = new Date(date.valueOf());
6166
+ const dayNr = (date.getUTCDay() + 6) % 7;
6167
+ target.setUTCDate(target.getUTCDate() - dayNr + 3);
6168
+ const yearStart = new Date(Date.UTC(target.getUTCFullYear(), 0, 1));
6169
+ const firstThursday = new Date(yearStart.valueOf());
6170
+ if (yearStart.getUTCDay() !== 4) {
6171
+ firstThursday.setUTCDate(yearStart.getUTCDate() + (4 - yearStart.getUTCDay() + 7) % 7);
6172
+ }
6173
+ const weekNumber = 1 + Math.round((target - firstThursday) / 6048e5);
6174
+ const weekYear = target.getUTCFullYear();
6175
+ return `${weekYear}-W${String(weekNumber).padStart(2, "0")}`;
6176
+ }
5897
6177
  async function rollupPeriod(period, cohort, sourcePrefix, analyticsResource, config) {
5898
- const sourcePeriod = period === "day" ? "hour" : "day";
6178
+ let sourcePeriod;
6179
+ if (period === "day") {
6180
+ sourcePeriod = "hour";
6181
+ } else if (period === "week") {
6182
+ sourcePeriod = "day";
6183
+ } else if (period === "month") {
6184
+ sourcePeriod = "day";
6185
+ } else {
6186
+ sourcePeriod = "day";
6187
+ }
5899
6188
  const [ok, err, allAnalytics] = await tryFn(
5900
6189
  () => analyticsResource.list()
5901
6190
  );
5902
6191
  if (!ok || !allAnalytics) return;
5903
- const sourceAnalytics = allAnalytics.filter(
5904
- (a) => a.period === sourcePeriod && a.cohort.startsWith(sourcePrefix)
5905
- );
6192
+ let sourceAnalytics;
6193
+ if (period === "week") {
6194
+ sourceAnalytics = allAnalytics.filter((a) => {
6195
+ if (a.period !== sourcePeriod) return false;
6196
+ const dayDate = new Date(a.cohort);
6197
+ const dayWeek = getCohortWeekFromDate(dayDate);
6198
+ return dayWeek === cohort;
6199
+ });
6200
+ } else {
6201
+ sourceAnalytics = allAnalytics.filter(
6202
+ (a) => a.period === sourcePeriod && a.cohort.startsWith(sourcePrefix)
6203
+ );
6204
+ }
5906
6205
  if (sourceAnalytics.length === 0) return;
5907
6206
  const transactionCount = sourceAnalytics.reduce((sum, a) => sum + a.transactionCount, 0);
5908
6207
  const totalValue = sourceAnalytics.reduce((sum, a) => sum + a.totalValue, 0);
@@ -6111,6 +6410,32 @@ async function getYearByMonth(resourceName, field, year, options, fieldHandlers)
6111
6410
  }
6112
6411
  return data;
6113
6412
  }
6413
+ async function getYearByWeek(resourceName, field, year, options, fieldHandlers) {
6414
+ const data = await getAnalytics(resourceName, field, {
6415
+ period: "week",
6416
+ year
6417
+ }, fieldHandlers);
6418
+ if (options.fillGaps) {
6419
+ const startWeek = `${year}-W01`;
6420
+ const endWeek = `${year}-W53`;
6421
+ return fillGaps(data, "week", startWeek, endWeek);
6422
+ }
6423
+ return data;
6424
+ }
6425
+ async function getMonthByWeek(resourceName, field, month, options, fieldHandlers) {
6426
+ const year = parseInt(month.substring(0, 4));
6427
+ const monthNum = parseInt(month.substring(5, 7));
6428
+ const firstDay = new Date(year, monthNum - 1, 1);
6429
+ const lastDay = new Date(year, monthNum, 0);
6430
+ const firstWeek = getCohortWeekFromDate(firstDay);
6431
+ const lastWeek = getCohortWeekFromDate(lastDay);
6432
+ const data = await getAnalytics(resourceName, field, {
6433
+ period: "week",
6434
+ startDate: firstWeek,
6435
+ endDate: lastWeek
6436
+ }, fieldHandlers);
6437
+ return data;
6438
+ }
6114
6439
  async function getMonthByHour(resourceName, field, month, options, fieldHandlers) {
6115
6440
  let year, monthNum;
6116
6441
  if (month === "last") {
@@ -6338,7 +6663,7 @@ async function completeFieldSetup(handler, database, config, plugin) {
6338
6663
  if (!handler.targetResource) return;
6339
6664
  const resourceName = handler.resource;
6340
6665
  const fieldName = handler.field;
6341
- const transactionResourceName = `${resourceName}_transactions_${fieldName}`;
6666
+ const transactionResourceName = `plg_${resourceName}_tx_${fieldName}`;
6342
6667
  const partitionConfig = createPartitionConfig();
6343
6668
  const [ok, err, transactionResource] = await tryFn(
6344
6669
  () => database.createResource({
@@ -6352,6 +6677,7 @@ async function completeFieldSetup(handler, database, config, plugin) {
6352
6677
  timestamp: "string|required",
6353
6678
  cohortDate: "string|required",
6354
6679
  cohortHour: "string|required",
6680
+ cohortWeek: "string|optional",
6355
6681
  cohortMonth: "string|optional",
6356
6682
  source: "string|optional",
6357
6683
  applied: "boolean|optional"
@@ -6367,36 +6693,18 @@ async function completeFieldSetup(handler, database, config, plugin) {
6367
6693
  throw new Error(`Failed to create transaction resource for ${resourceName}.${fieldName}: ${err?.message}`);
6368
6694
  }
6369
6695
  handler.transactionResource = ok ? transactionResource : database.resources[transactionResourceName];
6370
- const lockResourceName = `${resourceName}_consolidation_locks_${fieldName}`;
6371
- const [lockOk, lockErr, lockResource] = await tryFn(
6372
- () => database.createResource({
6373
- name: lockResourceName,
6374
- attributes: {
6375
- id: "string|required",
6376
- lockedAt: "number|required",
6377
- workerId: "string|optional"
6378
- },
6379
- behavior: "body-only",
6380
- timestamps: false,
6381
- createdBy: "EventualConsistencyPlugin"
6382
- })
6383
- );
6384
- if (!lockOk && !database.resources[lockResourceName]) {
6385
- throw new Error(`Failed to create lock resource for ${resourceName}.${fieldName}: ${lockErr?.message}`);
6386
- }
6387
- handler.lockResource = lockOk ? lockResource : database.resources[lockResourceName];
6388
6696
  if (config.enableAnalytics) {
6389
6697
  await createAnalyticsResource(handler, database, resourceName, fieldName);
6390
6698
  }
6391
6699
  addHelperMethodsForHandler(handler, plugin, config);
6392
6700
  if (config.verbose) {
6393
6701
  console.log(
6394
- `[EventualConsistency] ${resourceName}.${fieldName} - Setup complete. Resources: ${transactionResourceName}, ${lockResourceName}${config.enableAnalytics ? `, ${resourceName}_analytics_${fieldName}` : ""}`
6702
+ `[EventualConsistency] ${resourceName}.${fieldName} - Setup complete. Resources: ${transactionResourceName}${config.enableAnalytics ? `, plg_${resourceName}_an_${fieldName}` : ""} (locks via PluginStorage TTL)`
6395
6703
  );
6396
6704
  }
6397
6705
  }
6398
6706
  async function createAnalyticsResource(handler, database, resourceName, fieldName) {
6399
- const analyticsResourceName = `${resourceName}_analytics_${fieldName}`;
6707
+ const analyticsResourceName = `plg_${resourceName}_an_${fieldName}`;
6400
6708
  const [ok, err, analyticsResource] = await tryFn(
6401
6709
  () => database.createResource({
6402
6710
  name: analyticsResourceName,
@@ -6570,7 +6878,7 @@ class EventualConsistencyPlugin extends Plugin {
6570
6878
  originalId,
6571
6879
  this.transactionResource,
6572
6880
  this.targetResource,
6573
- this.lockResource,
6881
+ this.getStorage(),
6574
6882
  this.analyticsResource,
6575
6883
  (transactions) => this.updateAnalytics(transactions),
6576
6884
  this.config
@@ -6606,7 +6914,7 @@ class EventualConsistencyPlugin extends Plugin {
6606
6914
  originalId,
6607
6915
  this.transactionResource,
6608
6916
  this.targetResource,
6609
- this.lockResource,
6917
+ this.getStorage(),
6610
6918
  (id) => this.consolidateRecord(id),
6611
6919
  this.config
6612
6920
  );
@@ -6627,20 +6935,17 @@ class EventualConsistencyPlugin extends Plugin {
6627
6935
  const oldField = this.config.field;
6628
6936
  const oldTransactionResource = this.transactionResource;
6629
6937
  const oldTargetResource = this.targetResource;
6630
- const oldLockResource = this.lockResource;
6631
6938
  const oldAnalyticsResource = this.analyticsResource;
6632
6939
  this.config.resource = handler.resource;
6633
6940
  this.config.field = handler.field;
6634
6941
  this.transactionResource = handler.transactionResource;
6635
6942
  this.targetResource = handler.targetResource;
6636
- this.lockResource = handler.lockResource;
6637
6943
  this.analyticsResource = handler.analyticsResource;
6638
6944
  const result = await this.consolidateRecord(id);
6639
6945
  this.config.resource = oldResource;
6640
6946
  this.config.field = oldField;
6641
6947
  this.transactionResource = oldTransactionResource;
6642
6948
  this.targetResource = oldTargetResource;
6643
- this.lockResource = oldLockResource;
6644
6949
  this.analyticsResource = oldAnalyticsResource;
6645
6950
  return result;
6646
6951
  }
@@ -6653,20 +6958,17 @@ class EventualConsistencyPlugin extends Plugin {
6653
6958
  const oldField = this.config.field;
6654
6959
  const oldTransactionResource = this.transactionResource;
6655
6960
  const oldTargetResource = this.targetResource;
6656
- const oldLockResource = this.lockResource;
6657
6961
  const oldAnalyticsResource = this.analyticsResource;
6658
6962
  this.config.resource = handler.resource;
6659
6963
  this.config.field = handler.field;
6660
6964
  this.transactionResource = handler.transactionResource;
6661
6965
  this.targetResource = handler.targetResource;
6662
- this.lockResource = handler.lockResource;
6663
6966
  this.analyticsResource = handler.analyticsResource;
6664
6967
  const result = await this.consolidateRecord(id);
6665
6968
  this.config.resource = oldResource;
6666
6969
  this.config.field = oldField;
6667
6970
  this.transactionResource = oldTransactionResource;
6668
6971
  this.targetResource = oldTargetResource;
6669
- this.lockResource = oldLockResource;
6670
6972
  this.analyticsResource = oldAnalyticsResource;
6671
6973
  return result;
6672
6974
  }
@@ -6699,20 +7001,17 @@ class EventualConsistencyPlugin extends Plugin {
6699
7001
  const oldField = this.config.field;
6700
7002
  const oldTransactionResource = this.transactionResource;
6701
7003
  const oldTargetResource = this.targetResource;
6702
- const oldLockResource = this.lockResource;
6703
7004
  const oldAnalyticsResource = this.analyticsResource;
6704
7005
  this.config.resource = handler.resource;
6705
7006
  this.config.field = handler.field;
6706
7007
  this.transactionResource = handler.transactionResource;
6707
7008
  this.targetResource = handler.targetResource;
6708
- this.lockResource = handler.lockResource;
6709
7009
  this.analyticsResource = handler.analyticsResource;
6710
7010
  const result = await this.recalculateRecord(id);
6711
7011
  this.config.resource = oldResource;
6712
7012
  this.config.field = oldField;
6713
7013
  this.transactionResource = oldTransactionResource;
6714
7014
  this.targetResource = oldTargetResource;
6715
- this.lockResource = oldLockResource;
6716
7015
  this.analyticsResource = oldAnalyticsResource;
6717
7016
  return result;
6718
7017
  }
@@ -6725,13 +7024,11 @@ class EventualConsistencyPlugin extends Plugin {
6725
7024
  const oldField = this.config.field;
6726
7025
  const oldTransactionResource = this.transactionResource;
6727
7026
  const oldTargetResource = this.targetResource;
6728
- const oldLockResource = this.lockResource;
6729
7027
  const oldAnalyticsResource = this.analyticsResource;
6730
7028
  this.config.resource = resourceName;
6731
7029
  this.config.field = fieldName;
6732
7030
  this.transactionResource = handler.transactionResource;
6733
7031
  this.targetResource = handler.targetResource;
6734
- this.lockResource = handler.lockResource;
6735
7032
  this.analyticsResource = handler.analyticsResource;
6736
7033
  try {
6737
7034
  await runConsolidation(
@@ -6745,7 +7042,6 @@ class EventualConsistencyPlugin extends Plugin {
6745
7042
  this.config.field = oldField;
6746
7043
  this.transactionResource = oldTransactionResource;
6747
7044
  this.targetResource = oldTargetResource;
6748
- this.lockResource = oldLockResource;
6749
7045
  this.analyticsResource = oldAnalyticsResource;
6750
7046
  }
6751
7047
  }
@@ -6758,16 +7054,14 @@ class EventualConsistencyPlugin extends Plugin {
6758
7054
  const oldField = this.config.field;
6759
7055
  const oldTransactionResource = this.transactionResource;
6760
7056
  const oldTargetResource = this.targetResource;
6761
- const oldLockResource = this.lockResource;
6762
7057
  this.config.resource = resourceName;
6763
7058
  this.config.field = fieldName;
6764
7059
  this.transactionResource = handler.transactionResource;
6765
7060
  this.targetResource = handler.targetResource;
6766
- this.lockResource = handler.lockResource;
6767
7061
  try {
6768
7062
  await runGarbageCollection(
6769
7063
  this.transactionResource,
6770
- this.lockResource,
7064
+ this.getStorage(),
6771
7065
  this.config,
6772
7066
  (event, data) => this.emit(event, data)
6773
7067
  );
@@ -6776,7 +7070,6 @@ class EventualConsistencyPlugin extends Plugin {
6776
7070
  this.config.field = oldField;
6777
7071
  this.transactionResource = oldTransactionResource;
6778
7072
  this.targetResource = oldTargetResource;
6779
- this.lockResource = oldLockResource;
6780
7073
  }
6781
7074
  }
6782
7075
  // Public Analytics API
@@ -6845,6 +7138,28 @@ class EventualConsistencyPlugin extends Plugin {
6845
7138
  async getMonthByHour(resourceName, field, month, options = {}) {
6846
7139
  return await getMonthByHour(resourceName, field, month, options, this.fieldHandlers);
6847
7140
  }
7141
+ /**
7142
+ * Get analytics for entire year, broken down by weeks
7143
+ * @param {string} resourceName - Resource name
7144
+ * @param {string} field - Field name
7145
+ * @param {number} year - Year (e.g., 2025)
7146
+ * @param {Object} options - Options
7147
+ * @returns {Promise<Array>} Weekly analytics for the year (up to 53 weeks)
7148
+ */
7149
+ async getYearByWeek(resourceName, field, year, options = {}) {
7150
+ return await getYearByWeek(resourceName, field, year, options, this.fieldHandlers);
7151
+ }
7152
+ /**
7153
+ * Get analytics for entire month, broken down by weeks
7154
+ * @param {string} resourceName - Resource name
7155
+ * @param {string} field - Field name
7156
+ * @param {string} month - Month in YYYY-MM format
7157
+ * @param {Object} options - Options
7158
+ * @returns {Promise<Array>} Weekly analytics for the month
7159
+ */
7160
+ async getMonthByWeek(resourceName, field, month, options = {}) {
7161
+ return await getMonthByWeek(resourceName, field, month, options, this.fieldHandlers);
7162
+ }
6848
7163
  /**
6849
7164
  * Get top records by volume
6850
7165
  * @param {string} resourceName - Resource name
@@ -6867,6 +7182,8 @@ class FullTextPlugin extends Plugin {
6867
7182
  ...options
6868
7183
  };
6869
7184
  this.indexes = /* @__PURE__ */ new Map();
7185
+ this.dirtyIndexes = /* @__PURE__ */ new Set();
7186
+ this.deletedIndexes = /* @__PURE__ */ new Set();
6870
7187
  }
6871
7188
  async onInstall() {
6872
7189
  const [ok, err, indexResource] = await tryFn(() => this.database.createResource({
@@ -6880,7 +7197,11 @@ class FullTextPlugin extends Plugin {
6880
7197
  // Array of record IDs containing this word
6881
7198
  count: "number|required",
6882
7199
  lastUpdated: "string|required"
6883
- }
7200
+ },
7201
+ partitions: {
7202
+ byResource: { fields: { resourceName: "string" } }
7203
+ },
7204
+ behavior: "body-overflow"
6884
7205
  }));
6885
7206
  this.indexResource = ok ? indexResource : this.database.resources.fulltext_indexes;
6886
7207
  await this.loadIndexes();
@@ -6909,22 +7230,53 @@ class FullTextPlugin extends Plugin {
6909
7230
  async saveIndexes() {
6910
7231
  if (!this.indexResource) return;
6911
7232
  const [ok, err] = await tryFn(async () => {
6912
- const existingIndexes = await this.indexResource.getAll();
6913
- for (const index of existingIndexes) {
6914
- await this.indexResource.delete(index.id);
7233
+ for (const key of this.deletedIndexes) {
7234
+ const [resourceName] = key.split(":");
7235
+ const [queryOk, queryErr, results] = await tryFn(
7236
+ () => this.indexResource.query({ resourceName })
7237
+ );
7238
+ if (queryOk && results) {
7239
+ for (const index of results) {
7240
+ const indexKey = `${index.resourceName}:${index.fieldName}:${index.word}`;
7241
+ if (indexKey === key) {
7242
+ await this.indexResource.delete(index.id);
7243
+ }
7244
+ }
7245
+ }
6915
7246
  }
6916
- for (const [key, data] of this.indexes.entries()) {
7247
+ for (const key of this.dirtyIndexes) {
6917
7248
  const [resourceName, fieldName, word] = key.split(":");
6918
- await this.indexResource.insert({
6919
- id: `index-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`,
6920
- resourceName,
6921
- fieldName,
6922
- word,
6923
- recordIds: data.recordIds,
6924
- count: data.count,
6925
- lastUpdated: (/* @__PURE__ */ new Date()).toISOString()
6926
- });
7249
+ const data = this.indexes.get(key);
7250
+ if (!data) continue;
7251
+ const [queryOk, queryErr, results] = await tryFn(
7252
+ () => this.indexResource.query({ resourceName })
7253
+ );
7254
+ let existingRecord = null;
7255
+ if (queryOk && results) {
7256
+ existingRecord = results.find(
7257
+ (index) => index.resourceName === resourceName && index.fieldName === fieldName && index.word === word
7258
+ );
7259
+ }
7260
+ if (existingRecord) {
7261
+ await this.indexResource.update(existingRecord.id, {
7262
+ recordIds: data.recordIds,
7263
+ count: data.count,
7264
+ lastUpdated: (/* @__PURE__ */ new Date()).toISOString()
7265
+ });
7266
+ } else {
7267
+ await this.indexResource.insert({
7268
+ id: `index-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`,
7269
+ resourceName,
7270
+ fieldName,
7271
+ word,
7272
+ recordIds: data.recordIds,
7273
+ count: data.count,
7274
+ lastUpdated: (/* @__PURE__ */ new Date()).toISOString()
7275
+ });
7276
+ }
6927
7277
  }
7278
+ this.dirtyIndexes.clear();
7279
+ this.deletedIndexes.clear();
6928
7280
  });
6929
7281
  }
6930
7282
  installDatabaseHooks() {
@@ -7019,6 +7371,7 @@ class FullTextPlugin extends Plugin {
7019
7371
  existing.count = existing.recordIds.length;
7020
7372
  }
7021
7373
  this.indexes.set(key, existing);
7374
+ this.dirtyIndexes.add(key);
7022
7375
  }
7023
7376
  }
7024
7377
  }
@@ -7031,8 +7384,10 @@ class FullTextPlugin extends Plugin {
7031
7384
  data.count = data.recordIds.length;
7032
7385
  if (data.recordIds.length === 0) {
7033
7386
  this.indexes.delete(key);
7387
+ this.deletedIndexes.add(key);
7034
7388
  } else {
7035
7389
  this.indexes.set(key, data);
7390
+ this.dirtyIndexes.add(key);
7036
7391
  }
7037
7392
  }
7038
7393
  }
@@ -7264,8 +7619,14 @@ class MetricsPlugin extends Plugin {
7264
7619
  errors: "number|required",
7265
7620
  avgTime: "number|required",
7266
7621
  timestamp: "string|required",
7267
- metadata: "json"
7268
- }
7622
+ metadata: "json",
7623
+ createdAt: "string|required"
7624
+ // YYYY-MM-DD for partitioning
7625
+ },
7626
+ partitions: {
7627
+ byDate: { fields: { createdAt: "string|maxlength:10" } }
7628
+ },
7629
+ behavior: "body-overflow"
7269
7630
  }));
7270
7631
  this.metricsResource = ok1 ? metricsResource : this.database.resources.plg_metrics;
7271
7632
  const [ok2, err2, errorsResource] = await tryFn(() => this.database.createResource({
@@ -7276,8 +7637,14 @@ class MetricsPlugin extends Plugin {
7276
7637
  operation: "string|required",
7277
7638
  error: "string|required",
7278
7639
  timestamp: "string|required",
7279
- metadata: "json"
7280
- }
7640
+ metadata: "json",
7641
+ createdAt: "string|required"
7642
+ // YYYY-MM-DD for partitioning
7643
+ },
7644
+ partitions: {
7645
+ byDate: { fields: { createdAt: "string|maxlength:10" } }
7646
+ },
7647
+ behavior: "body-overflow"
7281
7648
  }));
7282
7649
  this.errorsResource = ok2 ? errorsResource : this.database.resources.plg_error_logs;
7283
7650
  const [ok3, err3, performanceResource] = await tryFn(() => this.database.createResource({
@@ -7288,8 +7655,14 @@ class MetricsPlugin extends Plugin {
7288
7655
  operation: "string|required",
7289
7656
  duration: "number|required",
7290
7657
  timestamp: "string|required",
7291
- metadata: "json"
7292
- }
7658
+ metadata: "json",
7659
+ createdAt: "string|required"
7660
+ // YYYY-MM-DD for partitioning
7661
+ },
7662
+ partitions: {
7663
+ byDate: { fields: { createdAt: "string|maxlength:10" } }
7664
+ },
7665
+ behavior: "body-overflow"
7293
7666
  }));
7294
7667
  this.performanceResource = ok3 ? performanceResource : this.database.resources.plg_performance_logs;
7295
7668
  });
@@ -7510,6 +7883,8 @@ class MetricsPlugin extends Plugin {
7510
7883
  errorMetadata = { error: "true" };
7511
7884
  resourceMetadata = { resource: "true" };
7512
7885
  }
7886
+ const now = /* @__PURE__ */ new Date();
7887
+ const createdAt = now.toISOString().slice(0, 10);
7513
7888
  for (const [operation, data] of Object.entries(this.metrics.operations)) {
7514
7889
  if (data.count > 0) {
7515
7890
  await this.metricsResource.insert({
@@ -7521,7 +7896,8 @@ class MetricsPlugin extends Plugin {
7521
7896
  totalTime: data.totalTime,
7522
7897
  errors: data.errors,
7523
7898
  avgTime: data.count > 0 ? data.totalTime / data.count : 0,
7524
- timestamp: (/* @__PURE__ */ new Date()).toISOString(),
7899
+ timestamp: now.toISOString(),
7900
+ createdAt,
7525
7901
  metadata
7526
7902
  });
7527
7903
  }
@@ -7538,7 +7914,8 @@ class MetricsPlugin extends Plugin {
7538
7914
  totalTime: data.totalTime,
7539
7915
  errors: data.errors,
7540
7916
  avgTime: data.count > 0 ? data.totalTime / data.count : 0,
7541
- timestamp: (/* @__PURE__ */ new Date()).toISOString(),
7917
+ timestamp: now.toISOString(),
7918
+ createdAt,
7542
7919
  metadata: resourceMetadata
7543
7920
  });
7544
7921
  }
@@ -7552,6 +7929,8 @@ class MetricsPlugin extends Plugin {
7552
7929
  operation: perf.operation,
7553
7930
  duration: perf.duration,
7554
7931
  timestamp: perf.timestamp,
7932
+ createdAt: perf.timestamp.slice(0, 10),
7933
+ // YYYY-MM-DD from timestamp
7555
7934
  metadata: perfMetadata
7556
7935
  });
7557
7936
  }
@@ -7565,6 +7944,8 @@ class MetricsPlugin extends Plugin {
7565
7944
  error: error.error,
7566
7945
  stack: error.stack,
7567
7946
  timestamp: error.timestamp,
7947
+ createdAt: error.timestamp.slice(0, 10),
7948
+ // YYYY-MM-DD from timestamp
7568
7949
  metadata: errorMetadata
7569
7950
  });
7570
7951
  }
@@ -7696,22 +8077,47 @@ class MetricsPlugin extends Plugin {
7696
8077
  async cleanupOldData() {
7697
8078
  const cutoffDate = /* @__PURE__ */ new Date();
7698
8079
  cutoffDate.setDate(cutoffDate.getDate() - this.config.retentionDays);
8080
+ cutoffDate.toISOString().slice(0, 10);
8081
+ const datesToDelete = [];
8082
+ const startDate = new Date(cutoffDate);
8083
+ startDate.setDate(startDate.getDate() - 365);
8084
+ for (let d = new Date(startDate); d < cutoffDate; d.setDate(d.getDate() + 1)) {
8085
+ datesToDelete.push(d.toISOString().slice(0, 10));
8086
+ }
7699
8087
  if (this.metricsResource) {
7700
- const oldMetrics = await this.getMetrics({ endDate: cutoffDate.toISOString() });
7701
- for (const metric of oldMetrics) {
7702
- await this.metricsResource.delete(metric.id);
8088
+ for (const dateStr of datesToDelete) {
8089
+ const [ok, err, oldMetrics] = await tryFn(
8090
+ () => this.metricsResource.query({ createdAt: dateStr })
8091
+ );
8092
+ if (ok && oldMetrics) {
8093
+ for (const metric of oldMetrics) {
8094
+ await tryFn(() => this.metricsResource.delete(metric.id));
8095
+ }
8096
+ }
7703
8097
  }
7704
8098
  }
7705
8099
  if (this.errorsResource) {
7706
- const oldErrors = await this.getErrorLogs({ endDate: cutoffDate.toISOString() });
7707
- for (const error of oldErrors) {
7708
- await this.errorsResource.delete(error.id);
8100
+ for (const dateStr of datesToDelete) {
8101
+ const [ok, err, oldErrors] = await tryFn(
8102
+ () => this.errorsResource.query({ createdAt: dateStr })
8103
+ );
8104
+ if (ok && oldErrors) {
8105
+ for (const error of oldErrors) {
8106
+ await tryFn(() => this.errorsResource.delete(error.id));
8107
+ }
8108
+ }
7709
8109
  }
7710
8110
  }
7711
8111
  if (this.performanceResource) {
7712
- const oldPerformance = await this.getPerformanceLogs({ endDate: cutoffDate.toISOString() });
7713
- for (const perf of oldPerformance) {
7714
- await this.performanceResource.delete(perf.id);
8112
+ for (const dateStr of datesToDelete) {
8113
+ const [ok, err, oldPerformance] = await tryFn(
8114
+ () => this.performanceResource.query({ createdAt: dateStr })
8115
+ );
8116
+ if (ok && oldPerformance) {
8117
+ for (const perf of oldPerformance) {
8118
+ await tryFn(() => this.performanceResource.delete(perf.id));
8119
+ }
8120
+ }
7715
8121
  }
7716
8122
  }
7717
8123
  }
@@ -12782,7 +13188,7 @@ class Database extends EventEmitter {
12782
13188
  this.id = idGenerator(7);
12783
13189
  this.version = "1";
12784
13190
  this.s3dbVersion = (() => {
12785
- const [ok, err, version] = tryFn(() => true ? "11.0.2" : "latest");
13191
+ const [ok, err, version] = tryFn(() => true ? "11.0.4" : "latest");
12786
13192
  return ok ? version : "latest";
12787
13193
  })();
12788
13194
  this.resources = {};
@@ -15091,28 +15497,6 @@ class S3QueuePlugin extends Plugin {
15091
15497
  throw new Error(`Failed to create queue resource: ${err?.message}`);
15092
15498
  }
15093
15499
  this.queueResource = this.database.resources[queueName];
15094
- const lockName = `${this.config.resource}_locks`;
15095
- const [okLock, errLock] = await tryFn(
15096
- () => this.database.createResource({
15097
- name: lockName,
15098
- attributes: {
15099
- id: "string|required",
15100
- workerId: "string|required",
15101
- timestamp: "number|required",
15102
- ttl: "number|default:5000"
15103
- },
15104
- behavior: "body-overflow",
15105
- timestamps: false
15106
- })
15107
- );
15108
- if (okLock || this.database.resources[lockName]) {
15109
- this.lockResource = this.database.resources[lockName];
15110
- } else {
15111
- this.lockResource = null;
15112
- if (this.config.verbose) {
15113
- console.log(`[S3QueuePlugin] Lock resource creation failed, locking disabled: ${errLock?.message}`);
15114
- }
15115
- }
15116
15500
  this.addHelperMethods();
15117
15501
  if (this.config.deadLetterResource) {
15118
15502
  await this.createDeadLetterResource();
@@ -15183,13 +15567,6 @@ class S3QueuePlugin extends Plugin {
15183
15567
  }
15184
15568
  }
15185
15569
  }, 5e3);
15186
- this.lockCleanupInterval = setInterval(() => {
15187
- this.cleanupStaleLocks().catch((err) => {
15188
- if (this.config.verbose) {
15189
- console.log(`[lockCleanup] Error: ${err.message}`);
15190
- }
15191
- });
15192
- }, 1e4);
15193
15570
  for (let i = 0; i < concurrency; i++) {
15194
15571
  const worker = this.createWorker(messageHandler, i);
15195
15572
  this.workers.push(worker);
@@ -15206,10 +15583,6 @@ class S3QueuePlugin extends Plugin {
15206
15583
  clearInterval(this.cacheCleanupInterval);
15207
15584
  this.cacheCleanupInterval = null;
15208
15585
  }
15209
- if (this.lockCleanupInterval) {
15210
- clearInterval(this.lockCleanupInterval);
15211
- this.lockCleanupInterval = null;
15212
- }
15213
15586
  await Promise.all(this.workers);
15214
15587
  this.workers = [];
15215
15588
  this.processedCache.clear();
@@ -15260,48 +15633,21 @@ class S3QueuePlugin extends Plugin {
15260
15633
  return null;
15261
15634
  }
15262
15635
  /**
15263
- * Acquire a distributed lock using ETag-based conditional updates
15636
+ * Acquire a distributed lock using PluginStorage TTL
15264
15637
  * This ensures only one worker can claim a message at a time
15265
- *
15266
- * Uses a two-step process:
15267
- * 1. Create lock resource (similar to queue resource) if not exists
15268
- * 2. Try to claim lock using ETag-based conditional update
15269
15638
  */
15270
15639
  async acquireLock(messageId) {
15271
- if (!this.lockResource) {
15272
- return true;
15273
- }
15274
- const lockId = `lock-${messageId}`;
15275
- const now = Date.now();
15640
+ const storage = this.getStorage();
15641
+ const lockKey = `msg-${messageId}`;
15276
15642
  try {
15277
- const [okGet, errGet, existingLock] = await tryFn(
15278
- () => this.lockResource.get(lockId)
15279
- );
15280
- if (existingLock) {
15281
- const lockAge = now - existingLock.timestamp;
15282
- if (lockAge < existingLock.ttl) {
15283
- return false;
15284
- }
15285
- const [ok, err, result] = await tryFn(
15286
- () => this.lockResource.updateConditional(lockId, {
15287
- workerId: this.workerId,
15288
- timestamp: now,
15289
- ttl: 5e3
15290
- }, {
15291
- ifMatch: existingLock._etag
15292
- })
15293
- );
15294
- return ok && result.success;
15295
- }
15296
- const [okCreate, errCreate] = await tryFn(
15297
- () => this.lockResource.insert({
15298
- id: lockId,
15299
- workerId: this.workerId,
15300
- timestamp: now,
15301
- ttl: 5e3
15302
- })
15303
- );
15304
- return okCreate;
15643
+ const lock = await storage.acquireLock(lockKey, {
15644
+ ttl: 5,
15645
+ // 5 seconds
15646
+ timeout: 0,
15647
+ // Don't wait if locked
15648
+ workerId: this.workerId
15649
+ });
15650
+ return lock !== null;
15305
15651
  } catch (error) {
15306
15652
  if (this.config.verbose) {
15307
15653
  console.log(`[acquireLock] Error: ${error.message}`);
@@ -15310,15 +15656,13 @@ class S3QueuePlugin extends Plugin {
15310
15656
  }
15311
15657
  }
15312
15658
  /**
15313
- * Release a distributed lock by deleting the lock record
15659
+ * Release a distributed lock via PluginStorage
15314
15660
  */
15315
15661
  async releaseLock(messageId) {
15316
- if (!this.lockResource) {
15317
- return;
15318
- }
15319
- const lockId = `lock-${messageId}`;
15662
+ const storage = this.getStorage();
15663
+ const lockKey = `msg-${messageId}`;
15320
15664
  try {
15321
- await this.lockResource.delete(lockId);
15665
+ await storage.releaseLock(lockKey);
15322
15666
  } catch (error) {
15323
15667
  if (this.config.verbose) {
15324
15668
  console.log(`[releaseLock] Failed to release lock for ${messageId}: ${error.message}`);
@@ -15326,30 +15670,11 @@ class S3QueuePlugin extends Plugin {
15326
15670
  }
15327
15671
  }
15328
15672
  /**
15329
- * Clean up stale locks (older than TTL)
15330
- * This prevents deadlocks if a worker crashes while holding a lock
15673
+ * Clean up stale locks - NO LONGER NEEDED
15674
+ * TTL handles automatic expiration, no manual cleanup required
15331
15675
  */
15332
15676
  async cleanupStaleLocks() {
15333
- if (!this.lockResource) {
15334
- return;
15335
- }
15336
- const now = Date.now();
15337
- try {
15338
- const locks = await this.lockResource.list();
15339
- for (const lock of locks) {
15340
- const lockAge = now - lock.timestamp;
15341
- if (lockAge > lock.ttl) {
15342
- await this.lockResource.delete(lock.id);
15343
- if (this.config.verbose) {
15344
- console.log(`[cleanupStaleLocks] Removed expired lock: ${lock.id}`);
15345
- }
15346
- }
15347
- }
15348
- } catch (error) {
15349
- if (this.config.verbose) {
15350
- console.log(`[cleanupStaleLocks] Error during cleanup: ${error.message}`);
15351
- }
15352
- }
15677
+ return;
15353
15678
  }
15354
15679
  async attemptClaim(msg) {
15355
15680
  const now = Date.now();
@@ -15573,7 +15898,6 @@ class SchedulerPlugin extends Plugin {
15573
15898
  ...options
15574
15899
  };
15575
15900
  this.database = null;
15576
- this.lockResource = null;
15577
15901
  this.jobs = /* @__PURE__ */ new Map();
15578
15902
  this.activeJobs = /* @__PURE__ */ new Map();
15579
15903
  this.timers = /* @__PURE__ */ new Map();
@@ -15612,7 +15936,6 @@ class SchedulerPlugin extends Plugin {
15612
15936
  return true;
15613
15937
  }
15614
15938
  async onInstall() {
15615
- await this._createLockResource();
15616
15939
  if (this.config.persistJobs) {
15617
15940
  await this._createJobHistoryResource();
15618
15941
  }
@@ -15641,25 +15964,6 @@ class SchedulerPlugin extends Plugin {
15641
15964
  await this._startScheduling();
15642
15965
  this.emit("initialized", { jobs: this.jobs.size });
15643
15966
  }
15644
- async _createLockResource() {
15645
- const [ok, err, lockResource] = await tryFn(
15646
- () => this.database.createResource({
15647
- name: "plg_scheduler_job_locks",
15648
- attributes: {
15649
- id: "string|required",
15650
- jobName: "string|required",
15651
- lockedAt: "number|required",
15652
- instanceId: "string|optional"
15653
- },
15654
- behavior: "body-only",
15655
- timestamps: false
15656
- })
15657
- );
15658
- if (!ok && !this.database.resources.plg_scheduler_job_locks) {
15659
- throw new Error(`Failed to create lock resource: ${err?.message}`);
15660
- }
15661
- this.lockResource = ok ? lockResource : this.database.resources.plg_scheduler_job_locks;
15662
- }
15663
15967
  async _createJobHistoryResource() {
15664
15968
  const [ok] = await tryFn(() => this.database.createResource({
15665
15969
  name: this.config.jobHistoryResource,
@@ -15767,16 +16071,16 @@ class SchedulerPlugin extends Plugin {
15767
16071
  return;
15768
16072
  }
15769
16073
  this.activeJobs.set(jobName, "acquiring-lock");
15770
- const lockId = `lock-${jobName}`;
15771
- const [lockAcquired, lockErr] = await tryFn(
15772
- () => this.lockResource.insert({
15773
- id: lockId,
15774
- jobName,
15775
- lockedAt: Date.now(),
15776
- instanceId: process.pid ? String(process.pid) : "unknown"
15777
- })
15778
- );
15779
- if (!lockAcquired) {
16074
+ const storage = this.getStorage();
16075
+ const lockKey = `job-${jobName}`;
16076
+ const lock = await storage.acquireLock(lockKey, {
16077
+ ttl: Math.ceil(job.timeout / 1e3) + 60,
16078
+ // Job timeout + 60 seconds buffer
16079
+ timeout: 0,
16080
+ // Don't wait if locked
16081
+ workerId: process.pid ? String(process.pid) : "unknown"
16082
+ });
16083
+ if (!lock) {
15780
16084
  if (this.config.verbose) {
15781
16085
  console.log(`[SchedulerPlugin] Job '${jobName}' already running on another instance`);
15782
16086
  }
@@ -15880,7 +16184,7 @@ class SchedulerPlugin extends Plugin {
15880
16184
  throw lastError;
15881
16185
  }
15882
16186
  } finally {
15883
- await tryFn(() => this.lockResource.delete(lockId));
16187
+ await tryFn(() => storage.releaseLock(lockKey));
15884
16188
  }
15885
16189
  }
15886
16190
  async _persistJobExecution(jobName, executionId, startTime, endTime, duration, status, result, error, retryCount) {