s3db.js 11.0.2 → 11.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/s3db.cjs.js CHANGED
@@ -837,18 +837,23 @@ class PluginStorage {
837
837
  return `plugin=${this.pluginSlug}/${parts.join("/")}`;
838
838
  }
839
839
  /**
840
- * Save data with metadata encoding and behavior support
840
+ * Save data with metadata encoding, behavior support, and optional TTL
841
841
  *
842
842
  * @param {string} key - S3 key
843
843
  * @param {Object} data - Data to save
844
844
  * @param {Object} options - Options
845
+ * @param {number} options.ttl - Time-to-live in seconds (optional)
845
846
  * @param {string} options.behavior - 'body-overflow' | 'body-only' | 'enforce-limits'
846
847
  * @param {string} options.contentType - Content type (default: application/json)
847
848
  * @returns {Promise<void>}
848
849
  */
849
- async put(key, data, options = {}) {
850
- const { behavior = "body-overflow", contentType = "application/json" } = options;
851
- const { metadata, body } = this._applyBehavior(data, behavior);
850
+ async set(key, data, options = {}) {
851
+ const { ttl, behavior = "body-overflow", contentType = "application/json" } = options;
852
+ const dataToSave = { ...data };
853
+ if (ttl && typeof ttl === "number" && ttl > 0) {
854
+ dataToSave._expiresAt = Date.now() + ttl * 1e3;
855
+ }
856
+ const { metadata, body } = this._applyBehavior(dataToSave, behavior);
852
857
  const putParams = {
853
858
  key,
854
859
  metadata,
@@ -859,14 +864,21 @@ class PluginStorage {
859
864
  }
860
865
  const [ok, err] = await tryFn(() => this.client.putObject(putParams));
861
866
  if (!ok) {
862
- throw new Error(`PluginStorage.put failed for key ${key}: ${err.message}`);
867
+ throw new Error(`PluginStorage.set failed for key ${key}: ${err.message}`);
863
868
  }
864
869
  }
865
870
  /**
866
- * Get data with automatic metadata decoding
871
+ * Alias for set() to maintain backward compatibility
872
+ * @deprecated Use set() instead
873
+ */
874
+ async put(key, data, options = {}) {
875
+ return this.set(key, data, options);
876
+ }
877
+ /**
878
+ * Get data with automatic metadata decoding and TTL check
867
879
  *
868
880
  * @param {string} key - S3 key
869
- * @returns {Promise<Object|null>} Data or null if not found
881
+ * @returns {Promise<Object|null>} Data or null if not found/expired
870
882
  */
871
883
  async get(key) {
872
884
  const [ok, err, response] = await tryFn(() => this.client.getObject(key));
@@ -878,18 +890,28 @@ class PluginStorage {
878
890
  }
879
891
  const metadata = response.Metadata || {};
880
892
  const parsedMetadata = this._parseMetadataValues(metadata);
893
+ let data = parsedMetadata;
881
894
  if (response.Body) {
882
895
  try {
883
896
  const bodyContent = await response.Body.transformToString();
884
897
  if (bodyContent && bodyContent.trim()) {
885
898
  const body = JSON.parse(bodyContent);
886
- return { ...parsedMetadata, ...body };
899
+ data = { ...parsedMetadata, ...body };
887
900
  }
888
901
  } catch (parseErr) {
889
902
  throw new Error(`PluginStorage.get failed to parse body for key ${key}: ${parseErr.message}`);
890
903
  }
891
904
  }
892
- return parsedMetadata;
905
+ const expiresAt = data._expiresat || data._expiresAt;
906
+ if (expiresAt) {
907
+ if (Date.now() > expiresAt) {
908
+ await this.delete(key);
909
+ return null;
910
+ }
911
+ delete data._expiresat;
912
+ delete data._expiresAt;
913
+ }
914
+ return data;
893
915
  }
894
916
  /**
895
917
  * Parse metadata values back to their original types
@@ -972,6 +994,123 @@ class PluginStorage {
972
994
  if (!keyPrefix) return keys;
973
995
  return keys.map((key) => key.replace(keyPrefix, "")).map((key) => key.startsWith("/") ? key.replace("/", "") : key);
974
996
  }
997
+ /**
998
+ * Check if a key exists (not expired)
999
+ *
1000
+ * @param {string} key - S3 key
1001
+ * @returns {Promise<boolean>} True if exists and not expired
1002
+ */
1003
+ async has(key) {
1004
+ const data = await this.get(key);
1005
+ return data !== null;
1006
+ }
1007
+ /**
1008
+ * Check if a key is expired
1009
+ *
1010
+ * @param {string} key - S3 key
1011
+ * @returns {Promise<boolean>} True if expired or not found
1012
+ */
1013
+ async isExpired(key) {
1014
+ const [ok, err, response] = await tryFn(() => this.client.getObject(key));
1015
+ if (!ok) {
1016
+ return true;
1017
+ }
1018
+ const metadata = response.Metadata || {};
1019
+ const parsedMetadata = this._parseMetadataValues(metadata);
1020
+ let data = parsedMetadata;
1021
+ if (response.Body) {
1022
+ try {
1023
+ const bodyContent = await response.Body.transformToString();
1024
+ if (bodyContent && bodyContent.trim()) {
1025
+ const body = JSON.parse(bodyContent);
1026
+ data = { ...parsedMetadata, ...body };
1027
+ }
1028
+ } catch {
1029
+ return true;
1030
+ }
1031
+ }
1032
+ const expiresAt = data._expiresat || data._expiresAt;
1033
+ if (!expiresAt) {
1034
+ return false;
1035
+ }
1036
+ return Date.now() > expiresAt;
1037
+ }
1038
+ /**
1039
+ * Get remaining TTL in seconds
1040
+ *
1041
+ * @param {string} key - S3 key
1042
+ * @returns {Promise<number|null>} Remaining seconds or null if no TTL/not found
1043
+ */
1044
+ async getTTL(key) {
1045
+ const [ok, err, response] = await tryFn(() => this.client.getObject(key));
1046
+ if (!ok) {
1047
+ return null;
1048
+ }
1049
+ const metadata = response.Metadata || {};
1050
+ const parsedMetadata = this._parseMetadataValues(metadata);
1051
+ let data = parsedMetadata;
1052
+ if (response.Body) {
1053
+ try {
1054
+ const bodyContent = await response.Body.transformToString();
1055
+ if (bodyContent && bodyContent.trim()) {
1056
+ const body = JSON.parse(bodyContent);
1057
+ data = { ...parsedMetadata, ...body };
1058
+ }
1059
+ } catch {
1060
+ return null;
1061
+ }
1062
+ }
1063
+ const expiresAt = data._expiresat || data._expiresAt;
1064
+ if (!expiresAt) {
1065
+ return null;
1066
+ }
1067
+ const remaining = Math.max(0, expiresAt - Date.now());
1068
+ return Math.floor(remaining / 1e3);
1069
+ }
1070
+ /**
1071
+ * Extend TTL by adding additional seconds
1072
+ *
1073
+ * @param {string} key - S3 key
1074
+ * @param {number} additionalSeconds - Seconds to add to current TTL
1075
+ * @returns {Promise<boolean>} True if extended, false if not found or no TTL
1076
+ */
1077
+ async touch(key, additionalSeconds) {
1078
+ const [ok, err, response] = await tryFn(() => this.client.getObject(key));
1079
+ if (!ok) {
1080
+ return false;
1081
+ }
1082
+ const metadata = response.Metadata || {};
1083
+ const parsedMetadata = this._parseMetadataValues(metadata);
1084
+ let data = parsedMetadata;
1085
+ if (response.Body) {
1086
+ try {
1087
+ const bodyContent = await response.Body.transformToString();
1088
+ if (bodyContent && bodyContent.trim()) {
1089
+ const body = JSON.parse(bodyContent);
1090
+ data = { ...parsedMetadata, ...body };
1091
+ }
1092
+ } catch {
1093
+ return false;
1094
+ }
1095
+ }
1096
+ const expiresAt = data._expiresat || data._expiresAt;
1097
+ if (!expiresAt) {
1098
+ return false;
1099
+ }
1100
+ data._expiresAt = expiresAt + additionalSeconds * 1e3;
1101
+ delete data._expiresat;
1102
+ const { metadata: newMetadata, body: newBody } = this._applyBehavior(data, "body-overflow");
1103
+ const putParams = {
1104
+ key,
1105
+ metadata: newMetadata,
1106
+ contentType: "application/json"
1107
+ };
1108
+ if (newBody !== null) {
1109
+ putParams.body = JSON.stringify(newBody);
1110
+ }
1111
+ const [putOk] = await tryFn(() => this.client.putObject(putParams));
1112
+ return putOk;
1113
+ }
975
1114
  /**
976
1115
  * Delete a single object
977
1116
  *
@@ -1049,6 +1188,78 @@ class PluginStorage {
1049
1188
  }
1050
1189
  return results;
1051
1190
  }
1191
+ /**
1192
+ * Acquire a distributed lock with TTL and retry logic
1193
+ *
1194
+ * @param {string} lockName - Lock identifier
1195
+ * @param {Object} options - Lock options
1196
+ * @param {number} options.ttl - Lock TTL in seconds (default: 30)
1197
+ * @param {number} options.timeout - Max wait time in ms (default: 0, no wait)
1198
+ * @param {string} options.workerId - Worker identifier (default: 'unknown')
1199
+ * @returns {Promise<Object|null>} Lock object or null if couldn't acquire
1200
+ */
1201
+ async acquireLock(lockName, options = {}) {
1202
+ const { ttl = 30, timeout = 0, workerId = "unknown" } = options;
1203
+ const key = this.getPluginKey(null, "locks", lockName);
1204
+ const startTime = Date.now();
1205
+ while (true) {
1206
+ const existing = await this.get(key);
1207
+ if (!existing) {
1208
+ await this.set(key, { workerId, acquiredAt: Date.now() }, { ttl });
1209
+ return { key, workerId };
1210
+ }
1211
+ if (Date.now() - startTime >= timeout) {
1212
+ return null;
1213
+ }
1214
+ await new Promise((resolve) => setTimeout(resolve, 100));
1215
+ }
1216
+ }
1217
+ /**
1218
+ * Release a distributed lock
1219
+ *
1220
+ * @param {string} lockName - Lock identifier
1221
+ * @returns {Promise<void>}
1222
+ */
1223
+ async releaseLock(lockName) {
1224
+ const key = this.getPluginKey(null, "locks", lockName);
1225
+ await this.delete(key);
1226
+ }
1227
+ /**
1228
+ * Check if a lock is currently held
1229
+ *
1230
+ * @param {string} lockName - Lock identifier
1231
+ * @returns {Promise<boolean>} True if locked
1232
+ */
1233
+ async isLocked(lockName) {
1234
+ const key = this.getPluginKey(null, "locks", lockName);
1235
+ const lock = await this.get(key);
1236
+ return lock !== null;
1237
+ }
1238
+ /**
1239
+ * Increment a counter value
1240
+ *
1241
+ * @param {string} key - S3 key
1242
+ * @param {number} amount - Amount to increment (default: 1)
1243
+ * @param {Object} options - Options (e.g., ttl)
1244
+ * @returns {Promise<number>} New value
1245
+ */
1246
+ async increment(key, amount = 1, options = {}) {
1247
+ const data = await this.get(key);
1248
+ const value = (data?.value || 0) + amount;
1249
+ await this.set(key, { value }, options);
1250
+ return value;
1251
+ }
1252
+ /**
1253
+ * Decrement a counter value
1254
+ *
1255
+ * @param {string} key - S3 key
1256
+ * @param {number} amount - Amount to decrement (default: 1)
1257
+ * @param {Object} options - Options (e.g., ttl)
1258
+ * @returns {Promise<number>} New value
1259
+ */
1260
+ async decrement(key, amount = 1, options = {}) {
1261
+ return this.increment(key, -amount, options);
1262
+ }
1052
1263
  /**
1053
1264
  * Apply behavior to split data between metadata and body
1054
1265
  *
@@ -1352,12 +1563,18 @@ class AuditPlugin extends Plugin {
1352
1563
  recordId: "string|required",
1353
1564
  userId: "string|optional",
1354
1565
  timestamp: "string|required",
1566
+ createdAt: "string|required",
1567
+ // YYYY-MM-DD for partitioning
1355
1568
  oldData: "string|optional",
1356
1569
  newData: "string|optional",
1357
1570
  partition: "string|optional",
1358
1571
  partitionValues: "string|optional",
1359
1572
  metadata: "string|optional"
1360
1573
  },
1574
+ partitions: {
1575
+ byDate: { fields: { createdAt: "string|maxlength:10" } },
1576
+ byResource: { fields: { resourceName: "string" } }
1577
+ },
1361
1578
  behavior: "body-overflow"
1362
1579
  }));
1363
1580
  this.auditResource = ok ? auditResource : this.database.resources.plg_audits || null;
@@ -1461,10 +1678,13 @@ class AuditPlugin extends Plugin {
1461
1678
  if (!this.auditResource) {
1462
1679
  return;
1463
1680
  }
1681
+ const now = /* @__PURE__ */ new Date();
1464
1682
  const auditRecord = {
1465
1683
  id: `audit-${Date.now()}-${Math.random().toString(36).substring(2, 11)}`,
1466
1684
  userId: this.getCurrentUserId?.() || "system",
1467
- timestamp: (/* @__PURE__ */ new Date()).toISOString(),
1685
+ timestamp: now.toISOString(),
1686
+ createdAt: now.toISOString().slice(0, 10),
1687
+ // YYYY-MM-DD for partitioning
1468
1688
  metadata: JSON.stringify({ source: "audit-plugin", version: "2.0" }),
1469
1689
  resourceName: auditData.resourceName,
1470
1690
  operation: auditData.operation,
@@ -1539,9 +1759,25 @@ class AuditPlugin extends Plugin {
1539
1759
  async getAuditLogs(options = {}) {
1540
1760
  if (!this.auditResource) return [];
1541
1761
  const { resourceName, operation, recordId, partition, startDate, endDate, limit = 100, offset = 0 } = options;
1542
- const hasFilters = resourceName || operation || recordId || partition || startDate || endDate;
1543
1762
  let items = [];
1544
- if (hasFilters) {
1763
+ if (resourceName && !operation && !recordId && !partition && !startDate && !endDate) {
1764
+ const [ok, err, result] = await tryFn(
1765
+ () => this.auditResource.query({ resourceName }, { limit: limit + offset })
1766
+ );
1767
+ items = ok && result ? result : [];
1768
+ return items.slice(offset, offset + limit);
1769
+ } else if (startDate && !resourceName && !operation && !recordId && !partition) {
1770
+ const dates = this._generateDateRange(startDate, endDate);
1771
+ for (const date of dates) {
1772
+ const [ok, err, result] = await tryFn(
1773
+ () => this.auditResource.query({ createdAt: date })
1774
+ );
1775
+ if (ok && result) {
1776
+ items.push(...result);
1777
+ }
1778
+ }
1779
+ return items.slice(offset, offset + limit);
1780
+ } else if (resourceName || operation || recordId || partition || startDate || endDate) {
1545
1781
  const fetchSize = Math.min(1e4, Math.max(1e3, (limit + offset) * 20));
1546
1782
  const result = await this.auditResource.list({ limit: fetchSize });
1547
1783
  items = result || [];
@@ -1571,6 +1807,15 @@ class AuditPlugin extends Plugin {
1571
1807
  return result.items || [];
1572
1808
  }
1573
1809
  }
1810
+ _generateDateRange(startDate, endDate) {
1811
+ const dates = [];
1812
+ const start = new Date(startDate);
1813
+ const end = endDate ? new Date(endDate) : /* @__PURE__ */ new Date();
1814
+ for (let d = new Date(start); d <= end; d.setDate(d.getDate() + 1)) {
1815
+ dates.push(d.toISOString().slice(0, 10));
1816
+ }
1817
+ return dates;
1818
+ }
1574
1819
  async getRecordHistory(resourceName, recordId) {
1575
1820
  return await this.getAuditLogs({ resourceName, recordId });
1576
1821
  }
@@ -1603,6 +1848,37 @@ class AuditPlugin extends Plugin {
1603
1848
  }
1604
1849
  return stats;
1605
1850
  }
1851
+ /**
1852
+ * Clean up audit logs older than retention period
1853
+ * @param {number} retentionDays - Number of days to retain (default: 90)
1854
+ * @returns {Promise<number>} Number of records deleted
1855
+ */
1856
+ async cleanupOldAudits(retentionDays = 90) {
1857
+ if (!this.auditResource) return 0;
1858
+ const cutoffDate = /* @__PURE__ */ new Date();
1859
+ cutoffDate.setDate(cutoffDate.getDate() - retentionDays);
1860
+ const datesToDelete = [];
1861
+ const startDate = new Date(cutoffDate);
1862
+ startDate.setDate(startDate.getDate() - 365);
1863
+ for (let d = new Date(startDate); d < cutoffDate; d.setDate(d.getDate() + 1)) {
1864
+ datesToDelete.push(d.toISOString().slice(0, 10));
1865
+ }
1866
+ let deletedCount = 0;
1867
+ for (const dateStr of datesToDelete) {
1868
+ const [ok, err, oldAudits] = await tryFn(
1869
+ () => this.auditResource.query({ createdAt: dateStr })
1870
+ );
1871
+ if (ok && oldAudits) {
1872
+ for (const audit of oldAudits) {
1873
+ const [delOk] = await tryFn(() => this.auditResource.delete(audit.id));
1874
+ if (delOk) {
1875
+ deletedCount++;
1876
+ }
1877
+ }
1878
+ }
1879
+ }
1880
+ return deletedCount;
1881
+ }
1606
1882
  }
1607
1883
 
1608
1884
  class BaseBackupDriver {
@@ -4838,6 +5114,8 @@ function createConfig(options, detectedTimezone) {
4838
5114
  consolidationWindow: consolidation.window ?? 24,
4839
5115
  autoConsolidate: consolidation.auto !== false,
4840
5116
  mode: consolidation.mode || "async",
5117
+ // ✅ NOVO: Performance tuning - Mark applied concurrency (default 50, antes era 10 hardcoded)
5118
+ markAppliedConcurrency: consolidation.markAppliedConcurrency ?? 50,
4841
5119
  // Late arrivals
4842
5120
  lateArrivalStrategy: lateArrivals.strategy || "warn",
4843
5121
  // Batch transactions
@@ -4941,6 +5219,21 @@ function getTimezoneOffset(timezone, verbose = false) {
4941
5219
  return offsets[timezone] || 0;
4942
5220
  }
4943
5221
  }
5222
+ function getISOWeek(date) {
5223
+ const target = new Date(date.valueOf());
5224
+ const dayNr = (date.getUTCDay() + 6) % 7;
5225
+ target.setUTCDate(target.getUTCDate() - dayNr + 3);
5226
+ const yearStart = new Date(Date.UTC(target.getUTCFullYear(), 0, 1));
5227
+ const firstThursday = new Date(yearStart.valueOf());
5228
+ if (yearStart.getUTCDay() !== 4) {
5229
+ firstThursday.setUTCDate(yearStart.getUTCDate() + (4 - yearStart.getUTCDay() + 7) % 7);
5230
+ }
5231
+ const weekNumber = 1 + Math.round((target - firstThursday) / 6048e5);
5232
+ return {
5233
+ year: target.getUTCFullYear(),
5234
+ week: weekNumber
5235
+ };
5236
+ }
4944
5237
  function getCohortInfo(date, timezone, verbose = false) {
4945
5238
  const offset = getTimezoneOffset(timezone, verbose);
4946
5239
  const localDate = new Date(date.getTime() + offset);
@@ -4948,10 +5241,14 @@ function getCohortInfo(date, timezone, verbose = false) {
4948
5241
  const month = String(localDate.getMonth() + 1).padStart(2, "0");
4949
5242
  const day = String(localDate.getDate()).padStart(2, "0");
4950
5243
  const hour = String(localDate.getHours()).padStart(2, "0");
5244
+ const { year: weekYear, week: weekNumber } = getISOWeek(localDate);
5245
+ const week = `${weekYear}-W${String(weekNumber).padStart(2, "0")}`;
4951
5246
  return {
4952
5247
  date: `${year}-${month}-${day}`,
4953
5248
  hour: `${year}-${month}-${day}T${hour}`,
4954
5249
  // ISO-like format for hour partition
5250
+ week,
5251
+ // ISO 8601 week format (e.g., '2025-W42')
4955
5252
  month: `${year}-${month}`
4956
5253
  };
4957
5254
  }
@@ -5029,6 +5326,11 @@ function createPartitionConfig() {
5029
5326
  cohortDate: "string"
5030
5327
  }
5031
5328
  },
5329
+ byWeek: {
5330
+ fields: {
5331
+ cohortWeek: "string"
5332
+ }
5333
+ },
5032
5334
  byMonth: {
5033
5335
  fields: {
5034
5336
  cohortMonth: "string"
@@ -5068,6 +5370,7 @@ async function createTransaction(handler, data, config) {
5068
5370
  timestamp: now.toISOString(),
5069
5371
  cohortDate: cohortInfo.date,
5070
5372
  cohortHour: cohortInfo.hour,
5373
+ cohortWeek: cohortInfo.week,
5071
5374
  cohortMonth: cohortInfo.month,
5072
5375
  source: data.source || "unknown",
5073
5376
  applied: false
@@ -5108,50 +5411,6 @@ async function flushPendingTransactions(handler) {
5108
5411
  }
5109
5412
  }
5110
5413
 
5111
- async function cleanupStaleLocks(lockResource, config) {
5112
- const now = Date.now();
5113
- const lockTimeoutMs = config.lockTimeout * 1e3;
5114
- const cutoffTime = now - lockTimeoutMs;
5115
- const cleanupLockId = `lock-cleanup-${config.resource}-${config.field}`;
5116
- const [lockAcquired] = await tryFn(
5117
- () => lockResource.insert({
5118
- id: cleanupLockId,
5119
- lockedAt: Date.now(),
5120
- workerId: process.pid ? String(process.pid) : "unknown"
5121
- })
5122
- );
5123
- if (!lockAcquired) {
5124
- if (config.verbose) {
5125
- console.log(`[EventualConsistency] Lock cleanup already running in another container`);
5126
- }
5127
- return;
5128
- }
5129
- try {
5130
- const [ok, err, locks] = await tryFn(() => lockResource.list());
5131
- if (!ok || !locks || locks.length === 0) return;
5132
- const staleLocks = locks.filter(
5133
- (lock) => lock.id !== cleanupLockId && lock.lockedAt < cutoffTime
5134
- );
5135
- if (staleLocks.length === 0) return;
5136
- if (config.verbose) {
5137
- console.log(`[EventualConsistency] Cleaning up ${staleLocks.length} stale locks`);
5138
- }
5139
- const { results, errors } = await promisePool.PromisePool.for(staleLocks).withConcurrency(5).process(async (lock) => {
5140
- const [deleted] = await tryFn(() => lockResource.delete(lock.id));
5141
- return deleted;
5142
- });
5143
- if (errors && errors.length > 0 && config.verbose) {
5144
- console.warn(`[EventualConsistency] ${errors.length} stale locks failed to delete`);
5145
- }
5146
- } catch (error) {
5147
- if (config.verbose) {
5148
- console.warn(`[EventualConsistency] Error cleaning up stale locks:`, error.message);
5149
- }
5150
- } finally {
5151
- await tryFn(() => lockResource.delete(cleanupLockId));
5152
- }
5153
- }
5154
-
5155
5414
  function startConsolidationTimer(handler, resourceName, fieldName, runConsolidationCallback, config) {
5156
5415
  const intervalMs = config.consolidationInterval * 1e3;
5157
5416
  if (config.verbose) {
@@ -5248,17 +5507,15 @@ async function runConsolidation(transactionResource, consolidateRecordFn, emitFn
5248
5507
  }
5249
5508
  }
5250
5509
  }
5251
- async function consolidateRecord(originalId, transactionResource, targetResource, lockResource, analyticsResource, updateAnalyticsFn, config) {
5252
- await cleanupStaleLocks(lockResource, config);
5253
- const lockId = `lock-${originalId}`;
5254
- const [lockAcquired, lockErr, lock] = await tryFn(
5255
- () => lockResource.insert({
5256
- id: lockId,
5257
- lockedAt: Date.now(),
5258
- workerId: process.pid ? String(process.pid) : "unknown"
5259
- })
5260
- );
5261
- if (!lockAcquired) {
5510
+ async function consolidateRecord(originalId, transactionResource, targetResource, storage, analyticsResource, updateAnalyticsFn, config) {
5511
+ const lockKey = `consolidation-${config.resource}-${config.field}-${originalId}`;
5512
+ const lock = await storage.acquireLock(lockKey, {
5513
+ ttl: config.lockTimeout || 30,
5514
+ timeout: 0,
5515
+ // Don't wait if locked
5516
+ workerId: process.pid ? String(process.pid) : "unknown"
5517
+ });
5518
+ if (!lock) {
5262
5519
  if (config.verbose) {
5263
5520
  console.log(`[EventualConsistency] Lock for ${originalId} already held, skipping`);
5264
5521
  }
@@ -5469,7 +5726,8 @@ async function consolidateRecord(originalId, transactionResource, targetResource
5469
5726
  }
5470
5727
  if (updateOk) {
5471
5728
  const transactionsToUpdate = transactions.filter((txn) => txn.id !== "__synthetic__");
5472
- const { results, errors } = await promisePool.PromisePool.for(transactionsToUpdate).withConcurrency(10).process(async (txn) => {
5729
+ const markAppliedConcurrency = config.markAppliedConcurrency || 50;
5730
+ const { results, errors } = await promisePool.PromisePool.for(transactionsToUpdate).withConcurrency(markAppliedConcurrency).process(async (txn) => {
5473
5731
  const [ok2, err2] = await tryFn(
5474
5732
  () => transactionResource.update(txn.id, { applied: true })
5475
5733
  );
@@ -5517,9 +5775,11 @@ async function consolidateRecord(originalId, transactionResource, targetResource
5517
5775
  }
5518
5776
  return consolidatedValue;
5519
5777
  } finally {
5520
- const [lockReleased, lockReleaseErr] = await tryFn(() => lockResource.delete(lockId));
5778
+ const [lockReleased, lockReleaseErr] = await tryFn(
5779
+ () => storage.releaseLock(lockKey)
5780
+ );
5521
5781
  if (!lockReleased && config.verbose) {
5522
- console.warn(`[EventualConsistency] Failed to release lock ${lockId}:`, lockReleaseErr?.message);
5782
+ console.warn(`[EventualConsistency] Failed to release lock ${lockKey}:`, lockReleaseErr?.message);
5523
5783
  }
5524
5784
  }
5525
5785
  }
@@ -5593,17 +5853,15 @@ async function getCohortStats(cohortDate, transactionResource) {
5593
5853
  }
5594
5854
  return stats;
5595
5855
  }
5596
- async function recalculateRecord(originalId, transactionResource, targetResource, lockResource, consolidateRecordFn, config) {
5597
- await cleanupStaleLocks(lockResource, config);
5598
- const lockId = `lock-recalculate-${originalId}`;
5599
- const [lockAcquired, lockErr, lock] = await tryFn(
5600
- () => lockResource.insert({
5601
- id: lockId,
5602
- lockedAt: Date.now(),
5603
- workerId: process.pid ? String(process.pid) : "unknown"
5604
- })
5605
- );
5606
- if (!lockAcquired) {
5856
+ async function recalculateRecord(originalId, transactionResource, targetResource, storage, consolidateRecordFn, config) {
5857
+ const lockKey = `recalculate-${config.resource}-${config.field}-${originalId}`;
5858
+ const lock = await storage.acquireLock(lockKey, {
5859
+ ttl: config.lockTimeout || 30,
5860
+ timeout: 0,
5861
+ // Don't wait if locked
5862
+ workerId: process.pid ? String(process.pid) : "unknown"
5863
+ });
5864
+ if (!lock) {
5607
5865
  if (config.verbose) {
5608
5866
  console.log(`[EventualConsistency] Recalculate lock for ${originalId} already held, skipping`);
5609
5867
  }
@@ -5671,9 +5929,11 @@ async function recalculateRecord(originalId, transactionResource, targetResource
5671
5929
  }
5672
5930
  return consolidatedValue;
5673
5931
  } finally {
5674
- const [lockReleased, lockReleaseErr] = await tryFn(() => lockResource.delete(lockId));
5932
+ const [lockReleased, lockReleaseErr] = await tryFn(
5933
+ () => storage.releaseLock(lockKey)
5934
+ );
5675
5935
  if (!lockReleased && config.verbose) {
5676
- console.warn(`[EventualConsistency] Failed to release recalculate lock ${lockId}:`, lockReleaseErr?.message);
5936
+ console.warn(`[EventualConsistency] Failed to release recalculate lock ${lockKey}:`, lockReleaseErr?.message);
5677
5937
  }
5678
5938
  }
5679
5939
  }
@@ -5685,16 +5945,16 @@ function startGarbageCollectionTimer(handler, resourceName, fieldName, runGCCall
5685
5945
  }, gcIntervalMs);
5686
5946
  return handler.gcTimer;
5687
5947
  }
5688
- async function runGarbageCollection(transactionResource, lockResource, config, emitFn) {
5689
- const gcLockId = `lock-gc-${config.resource}-${config.field}`;
5690
- const [lockAcquired] = await tryFn(
5691
- () => lockResource.insert({
5692
- id: gcLockId,
5693
- lockedAt: Date.now(),
5694
- workerId: process.pid ? String(process.pid) : "unknown"
5695
- })
5696
- );
5697
- if (!lockAcquired) {
5948
+ async function runGarbageCollection(transactionResource, storage, config, emitFn) {
5949
+ const lockKey = `gc-${config.resource}-${config.field}`;
5950
+ const lock = await storage.acquireLock(lockKey, {
5951
+ ttl: 300,
5952
+ // 5 minutes for GC
5953
+ timeout: 0,
5954
+ // Don't wait if locked
5955
+ workerId: process.pid ? String(process.pid) : "unknown"
5956
+ });
5957
+ if (!lock) {
5698
5958
  if (config.verbose) {
5699
5959
  console.log(`[EventualConsistency] GC already running in another container`);
5700
5960
  }
@@ -5752,7 +6012,7 @@ async function runGarbageCollection(transactionResource, lockResource, config, e
5752
6012
  emitFn("eventual-consistency.gc-error", error);
5753
6013
  }
5754
6014
  } finally {
5755
- await tryFn(() => lockResource.delete(gcLockId));
6015
+ await tryFn(() => storage.releaseLock(lockKey));
5756
6016
  }
5757
6017
  }
5758
6018
 
@@ -5777,22 +6037,26 @@ AnalyticsResource: ${analyticsResource?.name || "unknown"}`
5777
6037
  const cohortCount = Object.keys(byHour).length;
5778
6038
  if (config.verbose) {
5779
6039
  console.log(
5780
- `[EventualConsistency] ${config.resource}.${config.field} - Updating ${cohortCount} hourly analytics cohorts...`
6040
+ `[EventualConsistency] ${config.resource}.${config.field} - Updating ${cohortCount} hourly analytics cohorts IN PARALLEL...`
5781
6041
  );
5782
6042
  }
5783
- for (const [cohort, txns] of Object.entries(byHour)) {
5784
- await upsertAnalytics("hour", cohort, txns, analyticsResource, config);
5785
- }
6043
+ await Promise.all(
6044
+ Object.entries(byHour).map(
6045
+ ([cohort, txns]) => upsertAnalytics("hour", cohort, txns, analyticsResource, config)
6046
+ )
6047
+ );
5786
6048
  if (config.analyticsConfig.rollupStrategy === "incremental") {
5787
6049
  const uniqueHours = Object.keys(byHour);
5788
6050
  if (config.verbose) {
5789
6051
  console.log(
5790
- `[EventualConsistency] ${config.resource}.${config.field} - Rolling up ${uniqueHours.length} hours to daily/monthly analytics...`
6052
+ `[EventualConsistency] ${config.resource}.${config.field} - Rolling up ${uniqueHours.length} hours to daily/weekly/monthly analytics IN PARALLEL...`
5791
6053
  );
5792
6054
  }
5793
- for (const cohortHour of uniqueHours) {
5794
- await rollupAnalytics(cohortHour, analyticsResource, config);
5795
- }
6055
+ await Promise.all(
6056
+ uniqueHours.map(
6057
+ (cohortHour) => rollupAnalytics(cohortHour, analyticsResource, config)
6058
+ )
6059
+ );
5796
6060
  }
5797
6061
  if (config.verbose) {
5798
6062
  console.log(
@@ -5895,18 +6159,53 @@ function calculateOperationBreakdown(transactions) {
5895
6159
  async function rollupAnalytics(cohortHour, analyticsResource, config) {
5896
6160
  const cohortDate = cohortHour.substring(0, 10);
5897
6161
  const cohortMonth = cohortHour.substring(0, 7);
6162
+ const date = new Date(cohortDate);
6163
+ const cohortWeek = getCohortWeekFromDate(date);
5898
6164
  await rollupPeriod("day", cohortDate, cohortDate, analyticsResource, config);
6165
+ await rollupPeriod("week", cohortWeek, cohortWeek, analyticsResource, config);
5899
6166
  await rollupPeriod("month", cohortMonth, cohortMonth, analyticsResource, config);
5900
6167
  }
6168
+ function getCohortWeekFromDate(date) {
6169
+ const target = new Date(date.valueOf());
6170
+ const dayNr = (date.getUTCDay() + 6) % 7;
6171
+ target.setUTCDate(target.getUTCDate() - dayNr + 3);
6172
+ const yearStart = new Date(Date.UTC(target.getUTCFullYear(), 0, 1));
6173
+ const firstThursday = new Date(yearStart.valueOf());
6174
+ if (yearStart.getUTCDay() !== 4) {
6175
+ firstThursday.setUTCDate(yearStart.getUTCDate() + (4 - yearStart.getUTCDay() + 7) % 7);
6176
+ }
6177
+ const weekNumber = 1 + Math.round((target - firstThursday) / 6048e5);
6178
+ const weekYear = target.getUTCFullYear();
6179
+ return `${weekYear}-W${String(weekNumber).padStart(2, "0")}`;
6180
+ }
5901
6181
  async function rollupPeriod(period, cohort, sourcePrefix, analyticsResource, config) {
5902
- const sourcePeriod = period === "day" ? "hour" : "day";
6182
+ let sourcePeriod;
6183
+ if (period === "day") {
6184
+ sourcePeriod = "hour";
6185
+ } else if (period === "week") {
6186
+ sourcePeriod = "day";
6187
+ } else if (period === "month") {
6188
+ sourcePeriod = "day";
6189
+ } else {
6190
+ sourcePeriod = "day";
6191
+ }
5903
6192
  const [ok, err, allAnalytics] = await tryFn(
5904
6193
  () => analyticsResource.list()
5905
6194
  );
5906
6195
  if (!ok || !allAnalytics) return;
5907
- const sourceAnalytics = allAnalytics.filter(
5908
- (a) => a.period === sourcePeriod && a.cohort.startsWith(sourcePrefix)
5909
- );
6196
+ let sourceAnalytics;
6197
+ if (period === "week") {
6198
+ sourceAnalytics = allAnalytics.filter((a) => {
6199
+ if (a.period !== sourcePeriod) return false;
6200
+ const dayDate = new Date(a.cohort);
6201
+ const dayWeek = getCohortWeekFromDate(dayDate);
6202
+ return dayWeek === cohort;
6203
+ });
6204
+ } else {
6205
+ sourceAnalytics = allAnalytics.filter(
6206
+ (a) => a.period === sourcePeriod && a.cohort.startsWith(sourcePrefix)
6207
+ );
6208
+ }
5910
6209
  if (sourceAnalytics.length === 0) return;
5911
6210
  const transactionCount = sourceAnalytics.reduce((sum, a) => sum + a.transactionCount, 0);
5912
6211
  const totalValue = sourceAnalytics.reduce((sum, a) => sum + a.totalValue, 0);
@@ -6115,6 +6414,32 @@ async function getYearByMonth(resourceName, field, year, options, fieldHandlers)
6115
6414
  }
6116
6415
  return data;
6117
6416
  }
6417
+ async function getYearByWeek(resourceName, field, year, options, fieldHandlers) {
6418
+ const data = await getAnalytics(resourceName, field, {
6419
+ period: "week",
6420
+ year
6421
+ }, fieldHandlers);
6422
+ if (options.fillGaps) {
6423
+ const startWeek = `${year}-W01`;
6424
+ const endWeek = `${year}-W53`;
6425
+ return fillGaps(data, "week", startWeek, endWeek);
6426
+ }
6427
+ return data;
6428
+ }
6429
+ async function getMonthByWeek(resourceName, field, month, options, fieldHandlers) {
6430
+ const year = parseInt(month.substring(0, 4));
6431
+ const monthNum = parseInt(month.substring(5, 7));
6432
+ const firstDay = new Date(year, monthNum - 1, 1);
6433
+ const lastDay = new Date(year, monthNum, 0);
6434
+ const firstWeek = getCohortWeekFromDate(firstDay);
6435
+ const lastWeek = getCohortWeekFromDate(lastDay);
6436
+ const data = await getAnalytics(resourceName, field, {
6437
+ period: "week",
6438
+ startDate: firstWeek,
6439
+ endDate: lastWeek
6440
+ }, fieldHandlers);
6441
+ return data;
6442
+ }
6118
6443
  async function getMonthByHour(resourceName, field, month, options, fieldHandlers) {
6119
6444
  let year, monthNum;
6120
6445
  if (month === "last") {
@@ -6342,7 +6667,7 @@ async function completeFieldSetup(handler, database, config, plugin) {
6342
6667
  if (!handler.targetResource) return;
6343
6668
  const resourceName = handler.resource;
6344
6669
  const fieldName = handler.field;
6345
- const transactionResourceName = `${resourceName}_transactions_${fieldName}`;
6670
+ const transactionResourceName = `plg_${resourceName}_tx_${fieldName}`;
6346
6671
  const partitionConfig = createPartitionConfig();
6347
6672
  const [ok, err, transactionResource] = await tryFn(
6348
6673
  () => database.createResource({
@@ -6356,6 +6681,7 @@ async function completeFieldSetup(handler, database, config, plugin) {
6356
6681
  timestamp: "string|required",
6357
6682
  cohortDate: "string|required",
6358
6683
  cohortHour: "string|required",
6684
+ cohortWeek: "string|optional",
6359
6685
  cohortMonth: "string|optional",
6360
6686
  source: "string|optional",
6361
6687
  applied: "boolean|optional"
@@ -6371,36 +6697,18 @@ async function completeFieldSetup(handler, database, config, plugin) {
6371
6697
  throw new Error(`Failed to create transaction resource for ${resourceName}.${fieldName}: ${err?.message}`);
6372
6698
  }
6373
6699
  handler.transactionResource = ok ? transactionResource : database.resources[transactionResourceName];
6374
- const lockResourceName = `${resourceName}_consolidation_locks_${fieldName}`;
6375
- const [lockOk, lockErr, lockResource] = await tryFn(
6376
- () => database.createResource({
6377
- name: lockResourceName,
6378
- attributes: {
6379
- id: "string|required",
6380
- lockedAt: "number|required",
6381
- workerId: "string|optional"
6382
- },
6383
- behavior: "body-only",
6384
- timestamps: false,
6385
- createdBy: "EventualConsistencyPlugin"
6386
- })
6387
- );
6388
- if (!lockOk && !database.resources[lockResourceName]) {
6389
- throw new Error(`Failed to create lock resource for ${resourceName}.${fieldName}: ${lockErr?.message}`);
6390
- }
6391
- handler.lockResource = lockOk ? lockResource : database.resources[lockResourceName];
6392
6700
  if (config.enableAnalytics) {
6393
6701
  await createAnalyticsResource(handler, database, resourceName, fieldName);
6394
6702
  }
6395
6703
  addHelperMethodsForHandler(handler, plugin, config);
6396
6704
  if (config.verbose) {
6397
6705
  console.log(
6398
- `[EventualConsistency] ${resourceName}.${fieldName} - Setup complete. Resources: ${transactionResourceName}, ${lockResourceName}${config.enableAnalytics ? `, ${resourceName}_analytics_${fieldName}` : ""}`
6706
+ `[EventualConsistency] ${resourceName}.${fieldName} - Setup complete. Resources: ${transactionResourceName}${config.enableAnalytics ? `, plg_${resourceName}_an_${fieldName}` : ""} (locks via PluginStorage TTL)`
6399
6707
  );
6400
6708
  }
6401
6709
  }
6402
6710
  async function createAnalyticsResource(handler, database, resourceName, fieldName) {
6403
- const analyticsResourceName = `${resourceName}_analytics_${fieldName}`;
6711
+ const analyticsResourceName = `plg_${resourceName}_an_${fieldName}`;
6404
6712
  const [ok, err, analyticsResource] = await tryFn(
6405
6713
  () => database.createResource({
6406
6714
  name: analyticsResourceName,
@@ -6574,7 +6882,7 @@ class EventualConsistencyPlugin extends Plugin {
6574
6882
  originalId,
6575
6883
  this.transactionResource,
6576
6884
  this.targetResource,
6577
- this.lockResource,
6885
+ this.getStorage(),
6578
6886
  this.analyticsResource,
6579
6887
  (transactions) => this.updateAnalytics(transactions),
6580
6888
  this.config
@@ -6610,7 +6918,7 @@ class EventualConsistencyPlugin extends Plugin {
6610
6918
  originalId,
6611
6919
  this.transactionResource,
6612
6920
  this.targetResource,
6613
- this.lockResource,
6921
+ this.getStorage(),
6614
6922
  (id) => this.consolidateRecord(id),
6615
6923
  this.config
6616
6924
  );
@@ -6631,20 +6939,17 @@ class EventualConsistencyPlugin extends Plugin {
6631
6939
  const oldField = this.config.field;
6632
6940
  const oldTransactionResource = this.transactionResource;
6633
6941
  const oldTargetResource = this.targetResource;
6634
- const oldLockResource = this.lockResource;
6635
6942
  const oldAnalyticsResource = this.analyticsResource;
6636
6943
  this.config.resource = handler.resource;
6637
6944
  this.config.field = handler.field;
6638
6945
  this.transactionResource = handler.transactionResource;
6639
6946
  this.targetResource = handler.targetResource;
6640
- this.lockResource = handler.lockResource;
6641
6947
  this.analyticsResource = handler.analyticsResource;
6642
6948
  const result = await this.consolidateRecord(id);
6643
6949
  this.config.resource = oldResource;
6644
6950
  this.config.field = oldField;
6645
6951
  this.transactionResource = oldTransactionResource;
6646
6952
  this.targetResource = oldTargetResource;
6647
- this.lockResource = oldLockResource;
6648
6953
  this.analyticsResource = oldAnalyticsResource;
6649
6954
  return result;
6650
6955
  }
@@ -6657,20 +6962,17 @@ class EventualConsistencyPlugin extends Plugin {
6657
6962
  const oldField = this.config.field;
6658
6963
  const oldTransactionResource = this.transactionResource;
6659
6964
  const oldTargetResource = this.targetResource;
6660
- const oldLockResource = this.lockResource;
6661
6965
  const oldAnalyticsResource = this.analyticsResource;
6662
6966
  this.config.resource = handler.resource;
6663
6967
  this.config.field = handler.field;
6664
6968
  this.transactionResource = handler.transactionResource;
6665
6969
  this.targetResource = handler.targetResource;
6666
- this.lockResource = handler.lockResource;
6667
6970
  this.analyticsResource = handler.analyticsResource;
6668
6971
  const result = await this.consolidateRecord(id);
6669
6972
  this.config.resource = oldResource;
6670
6973
  this.config.field = oldField;
6671
6974
  this.transactionResource = oldTransactionResource;
6672
6975
  this.targetResource = oldTargetResource;
6673
- this.lockResource = oldLockResource;
6674
6976
  this.analyticsResource = oldAnalyticsResource;
6675
6977
  return result;
6676
6978
  }
@@ -6703,20 +7005,17 @@ class EventualConsistencyPlugin extends Plugin {
6703
7005
  const oldField = this.config.field;
6704
7006
  const oldTransactionResource = this.transactionResource;
6705
7007
  const oldTargetResource = this.targetResource;
6706
- const oldLockResource = this.lockResource;
6707
7008
  const oldAnalyticsResource = this.analyticsResource;
6708
7009
  this.config.resource = handler.resource;
6709
7010
  this.config.field = handler.field;
6710
7011
  this.transactionResource = handler.transactionResource;
6711
7012
  this.targetResource = handler.targetResource;
6712
- this.lockResource = handler.lockResource;
6713
7013
  this.analyticsResource = handler.analyticsResource;
6714
7014
  const result = await this.recalculateRecord(id);
6715
7015
  this.config.resource = oldResource;
6716
7016
  this.config.field = oldField;
6717
7017
  this.transactionResource = oldTransactionResource;
6718
7018
  this.targetResource = oldTargetResource;
6719
- this.lockResource = oldLockResource;
6720
7019
  this.analyticsResource = oldAnalyticsResource;
6721
7020
  return result;
6722
7021
  }
@@ -6729,13 +7028,11 @@ class EventualConsistencyPlugin extends Plugin {
6729
7028
  const oldField = this.config.field;
6730
7029
  const oldTransactionResource = this.transactionResource;
6731
7030
  const oldTargetResource = this.targetResource;
6732
- const oldLockResource = this.lockResource;
6733
7031
  const oldAnalyticsResource = this.analyticsResource;
6734
7032
  this.config.resource = resourceName;
6735
7033
  this.config.field = fieldName;
6736
7034
  this.transactionResource = handler.transactionResource;
6737
7035
  this.targetResource = handler.targetResource;
6738
- this.lockResource = handler.lockResource;
6739
7036
  this.analyticsResource = handler.analyticsResource;
6740
7037
  try {
6741
7038
  await runConsolidation(
@@ -6749,7 +7046,6 @@ class EventualConsistencyPlugin extends Plugin {
6749
7046
  this.config.field = oldField;
6750
7047
  this.transactionResource = oldTransactionResource;
6751
7048
  this.targetResource = oldTargetResource;
6752
- this.lockResource = oldLockResource;
6753
7049
  this.analyticsResource = oldAnalyticsResource;
6754
7050
  }
6755
7051
  }
@@ -6762,16 +7058,14 @@ class EventualConsistencyPlugin extends Plugin {
6762
7058
  const oldField = this.config.field;
6763
7059
  const oldTransactionResource = this.transactionResource;
6764
7060
  const oldTargetResource = this.targetResource;
6765
- const oldLockResource = this.lockResource;
6766
7061
  this.config.resource = resourceName;
6767
7062
  this.config.field = fieldName;
6768
7063
  this.transactionResource = handler.transactionResource;
6769
7064
  this.targetResource = handler.targetResource;
6770
- this.lockResource = handler.lockResource;
6771
7065
  try {
6772
7066
  await runGarbageCollection(
6773
7067
  this.transactionResource,
6774
- this.lockResource,
7068
+ this.getStorage(),
6775
7069
  this.config,
6776
7070
  (event, data) => this.emit(event, data)
6777
7071
  );
@@ -6780,7 +7074,6 @@ class EventualConsistencyPlugin extends Plugin {
6780
7074
  this.config.field = oldField;
6781
7075
  this.transactionResource = oldTransactionResource;
6782
7076
  this.targetResource = oldTargetResource;
6783
- this.lockResource = oldLockResource;
6784
7077
  }
6785
7078
  }
6786
7079
  // Public Analytics API
@@ -6849,6 +7142,28 @@ class EventualConsistencyPlugin extends Plugin {
6849
7142
  async getMonthByHour(resourceName, field, month, options = {}) {
6850
7143
  return await getMonthByHour(resourceName, field, month, options, this.fieldHandlers);
6851
7144
  }
7145
+ /**
7146
+ * Get analytics for entire year, broken down by weeks
7147
+ * @param {string} resourceName - Resource name
7148
+ * @param {string} field - Field name
7149
+ * @param {number} year - Year (e.g., 2025)
7150
+ * @param {Object} options - Options
7151
+ * @returns {Promise<Array>} Weekly analytics for the year (up to 53 weeks)
7152
+ */
7153
+ async getYearByWeek(resourceName, field, year, options = {}) {
7154
+ return await getYearByWeek(resourceName, field, year, options, this.fieldHandlers);
7155
+ }
7156
+ /**
7157
+ * Get analytics for entire month, broken down by weeks
7158
+ * @param {string} resourceName - Resource name
7159
+ * @param {string} field - Field name
7160
+ * @param {string} month - Month in YYYY-MM format
7161
+ * @param {Object} options - Options
7162
+ * @returns {Promise<Array>} Weekly analytics for the month
7163
+ */
7164
+ async getMonthByWeek(resourceName, field, month, options = {}) {
7165
+ return await getMonthByWeek(resourceName, field, month, options, this.fieldHandlers);
7166
+ }
6852
7167
  /**
6853
7168
  * Get top records by volume
6854
7169
  * @param {string} resourceName - Resource name
@@ -6871,6 +7186,8 @@ class FullTextPlugin extends Plugin {
6871
7186
  ...options
6872
7187
  };
6873
7188
  this.indexes = /* @__PURE__ */ new Map();
7189
+ this.dirtyIndexes = /* @__PURE__ */ new Set();
7190
+ this.deletedIndexes = /* @__PURE__ */ new Set();
6874
7191
  }
6875
7192
  async onInstall() {
6876
7193
  const [ok, err, indexResource] = await tryFn(() => this.database.createResource({
@@ -6884,7 +7201,11 @@ class FullTextPlugin extends Plugin {
6884
7201
  // Array of record IDs containing this word
6885
7202
  count: "number|required",
6886
7203
  lastUpdated: "string|required"
6887
- }
7204
+ },
7205
+ partitions: {
7206
+ byResource: { fields: { resourceName: "string" } }
7207
+ },
7208
+ behavior: "body-overflow"
6888
7209
  }));
6889
7210
  this.indexResource = ok ? indexResource : this.database.resources.fulltext_indexes;
6890
7211
  await this.loadIndexes();
@@ -6913,22 +7234,53 @@ class FullTextPlugin extends Plugin {
6913
7234
  async saveIndexes() {
6914
7235
  if (!this.indexResource) return;
6915
7236
  const [ok, err] = await tryFn(async () => {
6916
- const existingIndexes = await this.indexResource.getAll();
6917
- for (const index of existingIndexes) {
6918
- await this.indexResource.delete(index.id);
7237
+ for (const key of this.deletedIndexes) {
7238
+ const [resourceName] = key.split(":");
7239
+ const [queryOk, queryErr, results] = await tryFn(
7240
+ () => this.indexResource.query({ resourceName })
7241
+ );
7242
+ if (queryOk && results) {
7243
+ for (const index of results) {
7244
+ const indexKey = `${index.resourceName}:${index.fieldName}:${index.word}`;
7245
+ if (indexKey === key) {
7246
+ await this.indexResource.delete(index.id);
7247
+ }
7248
+ }
7249
+ }
6919
7250
  }
6920
- for (const [key, data] of this.indexes.entries()) {
7251
+ for (const key of this.dirtyIndexes) {
6921
7252
  const [resourceName, fieldName, word] = key.split(":");
6922
- await this.indexResource.insert({
6923
- id: `index-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`,
6924
- resourceName,
6925
- fieldName,
6926
- word,
6927
- recordIds: data.recordIds,
6928
- count: data.count,
6929
- lastUpdated: (/* @__PURE__ */ new Date()).toISOString()
6930
- });
7253
+ const data = this.indexes.get(key);
7254
+ if (!data) continue;
7255
+ const [queryOk, queryErr, results] = await tryFn(
7256
+ () => this.indexResource.query({ resourceName })
7257
+ );
7258
+ let existingRecord = null;
7259
+ if (queryOk && results) {
7260
+ existingRecord = results.find(
7261
+ (index) => index.resourceName === resourceName && index.fieldName === fieldName && index.word === word
7262
+ );
7263
+ }
7264
+ if (existingRecord) {
7265
+ await this.indexResource.update(existingRecord.id, {
7266
+ recordIds: data.recordIds,
7267
+ count: data.count,
7268
+ lastUpdated: (/* @__PURE__ */ new Date()).toISOString()
7269
+ });
7270
+ } else {
7271
+ await this.indexResource.insert({
7272
+ id: `index-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`,
7273
+ resourceName,
7274
+ fieldName,
7275
+ word,
7276
+ recordIds: data.recordIds,
7277
+ count: data.count,
7278
+ lastUpdated: (/* @__PURE__ */ new Date()).toISOString()
7279
+ });
7280
+ }
6931
7281
  }
7282
+ this.dirtyIndexes.clear();
7283
+ this.deletedIndexes.clear();
6932
7284
  });
6933
7285
  }
6934
7286
  installDatabaseHooks() {
@@ -7023,6 +7375,7 @@ class FullTextPlugin extends Plugin {
7023
7375
  existing.count = existing.recordIds.length;
7024
7376
  }
7025
7377
  this.indexes.set(key, existing);
7378
+ this.dirtyIndexes.add(key);
7026
7379
  }
7027
7380
  }
7028
7381
  }
@@ -7035,8 +7388,10 @@ class FullTextPlugin extends Plugin {
7035
7388
  data.count = data.recordIds.length;
7036
7389
  if (data.recordIds.length === 0) {
7037
7390
  this.indexes.delete(key);
7391
+ this.deletedIndexes.add(key);
7038
7392
  } else {
7039
7393
  this.indexes.set(key, data);
7394
+ this.dirtyIndexes.add(key);
7040
7395
  }
7041
7396
  }
7042
7397
  }
@@ -7268,8 +7623,14 @@ class MetricsPlugin extends Plugin {
7268
7623
  errors: "number|required",
7269
7624
  avgTime: "number|required",
7270
7625
  timestamp: "string|required",
7271
- metadata: "json"
7272
- }
7626
+ metadata: "json",
7627
+ createdAt: "string|required"
7628
+ // YYYY-MM-DD for partitioning
7629
+ },
7630
+ partitions: {
7631
+ byDate: { fields: { createdAt: "string|maxlength:10" } }
7632
+ },
7633
+ behavior: "body-overflow"
7273
7634
  }));
7274
7635
  this.metricsResource = ok1 ? metricsResource : this.database.resources.plg_metrics;
7275
7636
  const [ok2, err2, errorsResource] = await tryFn(() => this.database.createResource({
@@ -7280,8 +7641,14 @@ class MetricsPlugin extends Plugin {
7280
7641
  operation: "string|required",
7281
7642
  error: "string|required",
7282
7643
  timestamp: "string|required",
7283
- metadata: "json"
7284
- }
7644
+ metadata: "json",
7645
+ createdAt: "string|required"
7646
+ // YYYY-MM-DD for partitioning
7647
+ },
7648
+ partitions: {
7649
+ byDate: { fields: { createdAt: "string|maxlength:10" } }
7650
+ },
7651
+ behavior: "body-overflow"
7285
7652
  }));
7286
7653
  this.errorsResource = ok2 ? errorsResource : this.database.resources.plg_error_logs;
7287
7654
  const [ok3, err3, performanceResource] = await tryFn(() => this.database.createResource({
@@ -7292,8 +7659,14 @@ class MetricsPlugin extends Plugin {
7292
7659
  operation: "string|required",
7293
7660
  duration: "number|required",
7294
7661
  timestamp: "string|required",
7295
- metadata: "json"
7296
- }
7662
+ metadata: "json",
7663
+ createdAt: "string|required"
7664
+ // YYYY-MM-DD for partitioning
7665
+ },
7666
+ partitions: {
7667
+ byDate: { fields: { createdAt: "string|maxlength:10" } }
7668
+ },
7669
+ behavior: "body-overflow"
7297
7670
  }));
7298
7671
  this.performanceResource = ok3 ? performanceResource : this.database.resources.plg_performance_logs;
7299
7672
  });
@@ -7514,6 +7887,8 @@ class MetricsPlugin extends Plugin {
7514
7887
  errorMetadata = { error: "true" };
7515
7888
  resourceMetadata = { resource: "true" };
7516
7889
  }
7890
+ const now = /* @__PURE__ */ new Date();
7891
+ const createdAt = now.toISOString().slice(0, 10);
7517
7892
  for (const [operation, data] of Object.entries(this.metrics.operations)) {
7518
7893
  if (data.count > 0) {
7519
7894
  await this.metricsResource.insert({
@@ -7525,7 +7900,8 @@ class MetricsPlugin extends Plugin {
7525
7900
  totalTime: data.totalTime,
7526
7901
  errors: data.errors,
7527
7902
  avgTime: data.count > 0 ? data.totalTime / data.count : 0,
7528
- timestamp: (/* @__PURE__ */ new Date()).toISOString(),
7903
+ timestamp: now.toISOString(),
7904
+ createdAt,
7529
7905
  metadata
7530
7906
  });
7531
7907
  }
@@ -7542,7 +7918,8 @@ class MetricsPlugin extends Plugin {
7542
7918
  totalTime: data.totalTime,
7543
7919
  errors: data.errors,
7544
7920
  avgTime: data.count > 0 ? data.totalTime / data.count : 0,
7545
- timestamp: (/* @__PURE__ */ new Date()).toISOString(),
7921
+ timestamp: now.toISOString(),
7922
+ createdAt,
7546
7923
  metadata: resourceMetadata
7547
7924
  });
7548
7925
  }
@@ -7556,6 +7933,8 @@ class MetricsPlugin extends Plugin {
7556
7933
  operation: perf.operation,
7557
7934
  duration: perf.duration,
7558
7935
  timestamp: perf.timestamp,
7936
+ createdAt: perf.timestamp.slice(0, 10),
7937
+ // YYYY-MM-DD from timestamp
7559
7938
  metadata: perfMetadata
7560
7939
  });
7561
7940
  }
@@ -7569,6 +7948,8 @@ class MetricsPlugin extends Plugin {
7569
7948
  error: error.error,
7570
7949
  stack: error.stack,
7571
7950
  timestamp: error.timestamp,
7951
+ createdAt: error.timestamp.slice(0, 10),
7952
+ // YYYY-MM-DD from timestamp
7572
7953
  metadata: errorMetadata
7573
7954
  });
7574
7955
  }
@@ -7700,22 +8081,47 @@ class MetricsPlugin extends Plugin {
7700
8081
  async cleanupOldData() {
7701
8082
  const cutoffDate = /* @__PURE__ */ new Date();
7702
8083
  cutoffDate.setDate(cutoffDate.getDate() - this.config.retentionDays);
8084
+ cutoffDate.toISOString().slice(0, 10);
8085
+ const datesToDelete = [];
8086
+ const startDate = new Date(cutoffDate);
8087
+ startDate.setDate(startDate.getDate() - 365);
8088
+ for (let d = new Date(startDate); d < cutoffDate; d.setDate(d.getDate() + 1)) {
8089
+ datesToDelete.push(d.toISOString().slice(0, 10));
8090
+ }
7703
8091
  if (this.metricsResource) {
7704
- const oldMetrics = await this.getMetrics({ endDate: cutoffDate.toISOString() });
7705
- for (const metric of oldMetrics) {
7706
- await this.metricsResource.delete(metric.id);
8092
+ for (const dateStr of datesToDelete) {
8093
+ const [ok, err, oldMetrics] = await tryFn(
8094
+ () => this.metricsResource.query({ createdAt: dateStr })
8095
+ );
8096
+ if (ok && oldMetrics) {
8097
+ for (const metric of oldMetrics) {
8098
+ await tryFn(() => this.metricsResource.delete(metric.id));
8099
+ }
8100
+ }
7707
8101
  }
7708
8102
  }
7709
8103
  if (this.errorsResource) {
7710
- const oldErrors = await this.getErrorLogs({ endDate: cutoffDate.toISOString() });
7711
- for (const error of oldErrors) {
7712
- await this.errorsResource.delete(error.id);
8104
+ for (const dateStr of datesToDelete) {
8105
+ const [ok, err, oldErrors] = await tryFn(
8106
+ () => this.errorsResource.query({ createdAt: dateStr })
8107
+ );
8108
+ if (ok && oldErrors) {
8109
+ for (const error of oldErrors) {
8110
+ await tryFn(() => this.errorsResource.delete(error.id));
8111
+ }
8112
+ }
7713
8113
  }
7714
8114
  }
7715
8115
  if (this.performanceResource) {
7716
- const oldPerformance = await this.getPerformanceLogs({ endDate: cutoffDate.toISOString() });
7717
- for (const perf of oldPerformance) {
7718
- await this.performanceResource.delete(perf.id);
8116
+ for (const dateStr of datesToDelete) {
8117
+ const [ok, err, oldPerformance] = await tryFn(
8118
+ () => this.performanceResource.query({ createdAt: dateStr })
8119
+ );
8120
+ if (ok && oldPerformance) {
8121
+ for (const perf of oldPerformance) {
8122
+ await tryFn(() => this.performanceResource.delete(perf.id));
8123
+ }
8124
+ }
7719
8125
  }
7720
8126
  }
7721
8127
  }
@@ -12786,7 +13192,7 @@ class Database extends EventEmitter {
12786
13192
  this.id = idGenerator(7);
12787
13193
  this.version = "1";
12788
13194
  this.s3dbVersion = (() => {
12789
- const [ok, err, version] = tryFn(() => true ? "11.0.2" : "latest");
13195
+ const [ok, err, version] = tryFn(() => true ? "11.0.4" : "latest");
12790
13196
  return ok ? version : "latest";
12791
13197
  })();
12792
13198
  this.resources = {};
@@ -15095,28 +15501,6 @@ class S3QueuePlugin extends Plugin {
15095
15501
  throw new Error(`Failed to create queue resource: ${err?.message}`);
15096
15502
  }
15097
15503
  this.queueResource = this.database.resources[queueName];
15098
- const lockName = `${this.config.resource}_locks`;
15099
- const [okLock, errLock] = await tryFn(
15100
- () => this.database.createResource({
15101
- name: lockName,
15102
- attributes: {
15103
- id: "string|required",
15104
- workerId: "string|required",
15105
- timestamp: "number|required",
15106
- ttl: "number|default:5000"
15107
- },
15108
- behavior: "body-overflow",
15109
- timestamps: false
15110
- })
15111
- );
15112
- if (okLock || this.database.resources[lockName]) {
15113
- this.lockResource = this.database.resources[lockName];
15114
- } else {
15115
- this.lockResource = null;
15116
- if (this.config.verbose) {
15117
- console.log(`[S3QueuePlugin] Lock resource creation failed, locking disabled: ${errLock?.message}`);
15118
- }
15119
- }
15120
15504
  this.addHelperMethods();
15121
15505
  if (this.config.deadLetterResource) {
15122
15506
  await this.createDeadLetterResource();
@@ -15187,13 +15571,6 @@ class S3QueuePlugin extends Plugin {
15187
15571
  }
15188
15572
  }
15189
15573
  }, 5e3);
15190
- this.lockCleanupInterval = setInterval(() => {
15191
- this.cleanupStaleLocks().catch((err) => {
15192
- if (this.config.verbose) {
15193
- console.log(`[lockCleanup] Error: ${err.message}`);
15194
- }
15195
- });
15196
- }, 1e4);
15197
15574
  for (let i = 0; i < concurrency; i++) {
15198
15575
  const worker = this.createWorker(messageHandler, i);
15199
15576
  this.workers.push(worker);
@@ -15210,10 +15587,6 @@ class S3QueuePlugin extends Plugin {
15210
15587
  clearInterval(this.cacheCleanupInterval);
15211
15588
  this.cacheCleanupInterval = null;
15212
15589
  }
15213
- if (this.lockCleanupInterval) {
15214
- clearInterval(this.lockCleanupInterval);
15215
- this.lockCleanupInterval = null;
15216
- }
15217
15590
  await Promise.all(this.workers);
15218
15591
  this.workers = [];
15219
15592
  this.processedCache.clear();
@@ -15264,48 +15637,21 @@ class S3QueuePlugin extends Plugin {
15264
15637
  return null;
15265
15638
  }
15266
15639
  /**
15267
- * Acquire a distributed lock using ETag-based conditional updates
15640
+ * Acquire a distributed lock using PluginStorage TTL
15268
15641
  * This ensures only one worker can claim a message at a time
15269
- *
15270
- * Uses a two-step process:
15271
- * 1. Create lock resource (similar to queue resource) if not exists
15272
- * 2. Try to claim lock using ETag-based conditional update
15273
15642
  */
15274
15643
  async acquireLock(messageId) {
15275
- if (!this.lockResource) {
15276
- return true;
15277
- }
15278
- const lockId = `lock-${messageId}`;
15279
- const now = Date.now();
15644
+ const storage = this.getStorage();
15645
+ const lockKey = `msg-${messageId}`;
15280
15646
  try {
15281
- const [okGet, errGet, existingLock] = await tryFn(
15282
- () => this.lockResource.get(lockId)
15283
- );
15284
- if (existingLock) {
15285
- const lockAge = now - existingLock.timestamp;
15286
- if (lockAge < existingLock.ttl) {
15287
- return false;
15288
- }
15289
- const [ok, err, result] = await tryFn(
15290
- () => this.lockResource.updateConditional(lockId, {
15291
- workerId: this.workerId,
15292
- timestamp: now,
15293
- ttl: 5e3
15294
- }, {
15295
- ifMatch: existingLock._etag
15296
- })
15297
- );
15298
- return ok && result.success;
15299
- }
15300
- const [okCreate, errCreate] = await tryFn(
15301
- () => this.lockResource.insert({
15302
- id: lockId,
15303
- workerId: this.workerId,
15304
- timestamp: now,
15305
- ttl: 5e3
15306
- })
15307
- );
15308
- return okCreate;
15647
+ const lock = await storage.acquireLock(lockKey, {
15648
+ ttl: 5,
15649
+ // 5 seconds
15650
+ timeout: 0,
15651
+ // Don't wait if locked
15652
+ workerId: this.workerId
15653
+ });
15654
+ return lock !== null;
15309
15655
  } catch (error) {
15310
15656
  if (this.config.verbose) {
15311
15657
  console.log(`[acquireLock] Error: ${error.message}`);
@@ -15314,15 +15660,13 @@ class S3QueuePlugin extends Plugin {
15314
15660
  }
15315
15661
  }
15316
15662
  /**
15317
- * Release a distributed lock by deleting the lock record
15663
+ * Release a distributed lock via PluginStorage
15318
15664
  */
15319
15665
  async releaseLock(messageId) {
15320
- if (!this.lockResource) {
15321
- return;
15322
- }
15323
- const lockId = `lock-${messageId}`;
15666
+ const storage = this.getStorage();
15667
+ const lockKey = `msg-${messageId}`;
15324
15668
  try {
15325
- await this.lockResource.delete(lockId);
15669
+ await storage.releaseLock(lockKey);
15326
15670
  } catch (error) {
15327
15671
  if (this.config.verbose) {
15328
15672
  console.log(`[releaseLock] Failed to release lock for ${messageId}: ${error.message}`);
@@ -15330,30 +15674,11 @@ class S3QueuePlugin extends Plugin {
15330
15674
  }
15331
15675
  }
15332
15676
  /**
15333
- * Clean up stale locks (older than TTL)
15334
- * This prevents deadlocks if a worker crashes while holding a lock
15677
+ * Clean up stale locks - NO LONGER NEEDED
15678
+ * TTL handles automatic expiration, no manual cleanup required
15335
15679
  */
15336
15680
  async cleanupStaleLocks() {
15337
- if (!this.lockResource) {
15338
- return;
15339
- }
15340
- const now = Date.now();
15341
- try {
15342
- const locks = await this.lockResource.list();
15343
- for (const lock of locks) {
15344
- const lockAge = now - lock.timestamp;
15345
- if (lockAge > lock.ttl) {
15346
- await this.lockResource.delete(lock.id);
15347
- if (this.config.verbose) {
15348
- console.log(`[cleanupStaleLocks] Removed expired lock: ${lock.id}`);
15349
- }
15350
- }
15351
- }
15352
- } catch (error) {
15353
- if (this.config.verbose) {
15354
- console.log(`[cleanupStaleLocks] Error during cleanup: ${error.message}`);
15355
- }
15356
- }
15681
+ return;
15357
15682
  }
15358
15683
  async attemptClaim(msg) {
15359
15684
  const now = Date.now();
@@ -15577,7 +15902,6 @@ class SchedulerPlugin extends Plugin {
15577
15902
  ...options
15578
15903
  };
15579
15904
  this.database = null;
15580
- this.lockResource = null;
15581
15905
  this.jobs = /* @__PURE__ */ new Map();
15582
15906
  this.activeJobs = /* @__PURE__ */ new Map();
15583
15907
  this.timers = /* @__PURE__ */ new Map();
@@ -15616,7 +15940,6 @@ class SchedulerPlugin extends Plugin {
15616
15940
  return true;
15617
15941
  }
15618
15942
  async onInstall() {
15619
- await this._createLockResource();
15620
15943
  if (this.config.persistJobs) {
15621
15944
  await this._createJobHistoryResource();
15622
15945
  }
@@ -15645,25 +15968,6 @@ class SchedulerPlugin extends Plugin {
15645
15968
  await this._startScheduling();
15646
15969
  this.emit("initialized", { jobs: this.jobs.size });
15647
15970
  }
15648
- async _createLockResource() {
15649
- const [ok, err, lockResource] = await tryFn(
15650
- () => this.database.createResource({
15651
- name: "plg_scheduler_job_locks",
15652
- attributes: {
15653
- id: "string|required",
15654
- jobName: "string|required",
15655
- lockedAt: "number|required",
15656
- instanceId: "string|optional"
15657
- },
15658
- behavior: "body-only",
15659
- timestamps: false
15660
- })
15661
- );
15662
- if (!ok && !this.database.resources.plg_scheduler_job_locks) {
15663
- throw new Error(`Failed to create lock resource: ${err?.message}`);
15664
- }
15665
- this.lockResource = ok ? lockResource : this.database.resources.plg_scheduler_job_locks;
15666
- }
15667
15971
  async _createJobHistoryResource() {
15668
15972
  const [ok] = await tryFn(() => this.database.createResource({
15669
15973
  name: this.config.jobHistoryResource,
@@ -15771,16 +16075,16 @@ class SchedulerPlugin extends Plugin {
15771
16075
  return;
15772
16076
  }
15773
16077
  this.activeJobs.set(jobName, "acquiring-lock");
15774
- const lockId = `lock-${jobName}`;
15775
- const [lockAcquired, lockErr] = await tryFn(
15776
- () => this.lockResource.insert({
15777
- id: lockId,
15778
- jobName,
15779
- lockedAt: Date.now(),
15780
- instanceId: process.pid ? String(process.pid) : "unknown"
15781
- })
15782
- );
15783
- if (!lockAcquired) {
16078
+ const storage = this.getStorage();
16079
+ const lockKey = `job-${jobName}`;
16080
+ const lock = await storage.acquireLock(lockKey, {
16081
+ ttl: Math.ceil(job.timeout / 1e3) + 60,
16082
+ // Job timeout + 60 seconds buffer
16083
+ timeout: 0,
16084
+ // Don't wait if locked
16085
+ workerId: process.pid ? String(process.pid) : "unknown"
16086
+ });
16087
+ if (!lock) {
15784
16088
  if (this.config.verbose) {
15785
16089
  console.log(`[SchedulerPlugin] Job '${jobName}' already running on another instance`);
15786
16090
  }
@@ -15884,7 +16188,7 @@ class SchedulerPlugin extends Plugin {
15884
16188
  throw lastError;
15885
16189
  }
15886
16190
  } finally {
15887
- await tryFn(() => this.lockResource.delete(lockId));
16191
+ await tryFn(() => storage.releaseLock(lockKey));
15888
16192
  }
15889
16193
  }
15890
16194
  async _persistJobExecution(jobName, executionId, startTime, endTime, duration, status, result, error, retryCount) {