s3db.js 11.0.2 → 11.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/s3db.cjs.js +612 -308
- package/dist/s3db.cjs.js.map +1 -1
- package/dist/s3db.es.js +612 -308
- package/dist/s3db.es.js.map +1 -1
- package/package.json +1 -1
- package/src/concerns/plugin-storage.js +274 -9
- package/src/plugins/audit.plugin.js +94 -18
- package/src/plugins/eventual-consistency/analytics.js +350 -15
- package/src/plugins/eventual-consistency/config.js +3 -0
- package/src/plugins/eventual-consistency/consolidation.js +32 -36
- package/src/plugins/eventual-consistency/garbage-collection.js +11 -13
- package/src/plugins/eventual-consistency/index.js +28 -19
- package/src/plugins/eventual-consistency/install.js +9 -26
- package/src/plugins/eventual-consistency/partitions.js +5 -0
- package/src/plugins/eventual-consistency/transactions.js +1 -0
- package/src/plugins/eventual-consistency/utils.js +36 -1
- package/src/plugins/fulltext.plugin.js +76 -22
- package/src/plugins/metrics.plugin.js +70 -20
- package/src/plugins/s3-queue.plugin.js +21 -120
- package/src/plugins/scheduler.plugin.js +11 -37
package/dist/s3db.cjs.js
CHANGED
|
@@ -837,18 +837,23 @@ class PluginStorage {
|
|
|
837
837
|
return `plugin=${this.pluginSlug}/${parts.join("/")}`;
|
|
838
838
|
}
|
|
839
839
|
/**
|
|
840
|
-
* Save data with metadata encoding and
|
|
840
|
+
* Save data with metadata encoding, behavior support, and optional TTL
|
|
841
841
|
*
|
|
842
842
|
* @param {string} key - S3 key
|
|
843
843
|
* @param {Object} data - Data to save
|
|
844
844
|
* @param {Object} options - Options
|
|
845
|
+
* @param {number} options.ttl - Time-to-live in seconds (optional)
|
|
845
846
|
* @param {string} options.behavior - 'body-overflow' | 'body-only' | 'enforce-limits'
|
|
846
847
|
* @param {string} options.contentType - Content type (default: application/json)
|
|
847
848
|
* @returns {Promise<void>}
|
|
848
849
|
*/
|
|
849
|
-
async
|
|
850
|
-
const { behavior = "body-overflow", contentType = "application/json" } = options;
|
|
851
|
-
const
|
|
850
|
+
async set(key, data, options = {}) {
|
|
851
|
+
const { ttl, behavior = "body-overflow", contentType = "application/json" } = options;
|
|
852
|
+
const dataToSave = { ...data };
|
|
853
|
+
if (ttl && typeof ttl === "number" && ttl > 0) {
|
|
854
|
+
dataToSave._expiresAt = Date.now() + ttl * 1e3;
|
|
855
|
+
}
|
|
856
|
+
const { metadata, body } = this._applyBehavior(dataToSave, behavior);
|
|
852
857
|
const putParams = {
|
|
853
858
|
key,
|
|
854
859
|
metadata,
|
|
@@ -859,14 +864,21 @@ class PluginStorage {
|
|
|
859
864
|
}
|
|
860
865
|
const [ok, err] = await tryFn(() => this.client.putObject(putParams));
|
|
861
866
|
if (!ok) {
|
|
862
|
-
throw new Error(`PluginStorage.
|
|
867
|
+
throw new Error(`PluginStorage.set failed for key ${key}: ${err.message}`);
|
|
863
868
|
}
|
|
864
869
|
}
|
|
865
870
|
/**
|
|
866
|
-
*
|
|
871
|
+
* Alias for set() to maintain backward compatibility
|
|
872
|
+
* @deprecated Use set() instead
|
|
873
|
+
*/
|
|
874
|
+
async put(key, data, options = {}) {
|
|
875
|
+
return this.set(key, data, options);
|
|
876
|
+
}
|
|
877
|
+
/**
|
|
878
|
+
* Get data with automatic metadata decoding and TTL check
|
|
867
879
|
*
|
|
868
880
|
* @param {string} key - S3 key
|
|
869
|
-
* @returns {Promise<Object|null>} Data or null if not found
|
|
881
|
+
* @returns {Promise<Object|null>} Data or null if not found/expired
|
|
870
882
|
*/
|
|
871
883
|
async get(key) {
|
|
872
884
|
const [ok, err, response] = await tryFn(() => this.client.getObject(key));
|
|
@@ -878,18 +890,28 @@ class PluginStorage {
|
|
|
878
890
|
}
|
|
879
891
|
const metadata = response.Metadata || {};
|
|
880
892
|
const parsedMetadata = this._parseMetadataValues(metadata);
|
|
893
|
+
let data = parsedMetadata;
|
|
881
894
|
if (response.Body) {
|
|
882
895
|
try {
|
|
883
896
|
const bodyContent = await response.Body.transformToString();
|
|
884
897
|
if (bodyContent && bodyContent.trim()) {
|
|
885
898
|
const body = JSON.parse(bodyContent);
|
|
886
|
-
|
|
899
|
+
data = { ...parsedMetadata, ...body };
|
|
887
900
|
}
|
|
888
901
|
} catch (parseErr) {
|
|
889
902
|
throw new Error(`PluginStorage.get failed to parse body for key ${key}: ${parseErr.message}`);
|
|
890
903
|
}
|
|
891
904
|
}
|
|
892
|
-
|
|
905
|
+
const expiresAt = data._expiresat || data._expiresAt;
|
|
906
|
+
if (expiresAt) {
|
|
907
|
+
if (Date.now() > expiresAt) {
|
|
908
|
+
await this.delete(key);
|
|
909
|
+
return null;
|
|
910
|
+
}
|
|
911
|
+
delete data._expiresat;
|
|
912
|
+
delete data._expiresAt;
|
|
913
|
+
}
|
|
914
|
+
return data;
|
|
893
915
|
}
|
|
894
916
|
/**
|
|
895
917
|
* Parse metadata values back to their original types
|
|
@@ -972,6 +994,123 @@ class PluginStorage {
|
|
|
972
994
|
if (!keyPrefix) return keys;
|
|
973
995
|
return keys.map((key) => key.replace(keyPrefix, "")).map((key) => key.startsWith("/") ? key.replace("/", "") : key);
|
|
974
996
|
}
|
|
997
|
+
/**
|
|
998
|
+
* Check if a key exists (not expired)
|
|
999
|
+
*
|
|
1000
|
+
* @param {string} key - S3 key
|
|
1001
|
+
* @returns {Promise<boolean>} True if exists and not expired
|
|
1002
|
+
*/
|
|
1003
|
+
async has(key) {
|
|
1004
|
+
const data = await this.get(key);
|
|
1005
|
+
return data !== null;
|
|
1006
|
+
}
|
|
1007
|
+
/**
|
|
1008
|
+
* Check if a key is expired
|
|
1009
|
+
*
|
|
1010
|
+
* @param {string} key - S3 key
|
|
1011
|
+
* @returns {Promise<boolean>} True if expired or not found
|
|
1012
|
+
*/
|
|
1013
|
+
async isExpired(key) {
|
|
1014
|
+
const [ok, err, response] = await tryFn(() => this.client.getObject(key));
|
|
1015
|
+
if (!ok) {
|
|
1016
|
+
return true;
|
|
1017
|
+
}
|
|
1018
|
+
const metadata = response.Metadata || {};
|
|
1019
|
+
const parsedMetadata = this._parseMetadataValues(metadata);
|
|
1020
|
+
let data = parsedMetadata;
|
|
1021
|
+
if (response.Body) {
|
|
1022
|
+
try {
|
|
1023
|
+
const bodyContent = await response.Body.transformToString();
|
|
1024
|
+
if (bodyContent && bodyContent.trim()) {
|
|
1025
|
+
const body = JSON.parse(bodyContent);
|
|
1026
|
+
data = { ...parsedMetadata, ...body };
|
|
1027
|
+
}
|
|
1028
|
+
} catch {
|
|
1029
|
+
return true;
|
|
1030
|
+
}
|
|
1031
|
+
}
|
|
1032
|
+
const expiresAt = data._expiresat || data._expiresAt;
|
|
1033
|
+
if (!expiresAt) {
|
|
1034
|
+
return false;
|
|
1035
|
+
}
|
|
1036
|
+
return Date.now() > expiresAt;
|
|
1037
|
+
}
|
|
1038
|
+
/**
|
|
1039
|
+
* Get remaining TTL in seconds
|
|
1040
|
+
*
|
|
1041
|
+
* @param {string} key - S3 key
|
|
1042
|
+
* @returns {Promise<number|null>} Remaining seconds or null if no TTL/not found
|
|
1043
|
+
*/
|
|
1044
|
+
async getTTL(key) {
|
|
1045
|
+
const [ok, err, response] = await tryFn(() => this.client.getObject(key));
|
|
1046
|
+
if (!ok) {
|
|
1047
|
+
return null;
|
|
1048
|
+
}
|
|
1049
|
+
const metadata = response.Metadata || {};
|
|
1050
|
+
const parsedMetadata = this._parseMetadataValues(metadata);
|
|
1051
|
+
let data = parsedMetadata;
|
|
1052
|
+
if (response.Body) {
|
|
1053
|
+
try {
|
|
1054
|
+
const bodyContent = await response.Body.transformToString();
|
|
1055
|
+
if (bodyContent && bodyContent.trim()) {
|
|
1056
|
+
const body = JSON.parse(bodyContent);
|
|
1057
|
+
data = { ...parsedMetadata, ...body };
|
|
1058
|
+
}
|
|
1059
|
+
} catch {
|
|
1060
|
+
return null;
|
|
1061
|
+
}
|
|
1062
|
+
}
|
|
1063
|
+
const expiresAt = data._expiresat || data._expiresAt;
|
|
1064
|
+
if (!expiresAt) {
|
|
1065
|
+
return null;
|
|
1066
|
+
}
|
|
1067
|
+
const remaining = Math.max(0, expiresAt - Date.now());
|
|
1068
|
+
return Math.floor(remaining / 1e3);
|
|
1069
|
+
}
|
|
1070
|
+
/**
|
|
1071
|
+
* Extend TTL by adding additional seconds
|
|
1072
|
+
*
|
|
1073
|
+
* @param {string} key - S3 key
|
|
1074
|
+
* @param {number} additionalSeconds - Seconds to add to current TTL
|
|
1075
|
+
* @returns {Promise<boolean>} True if extended, false if not found or no TTL
|
|
1076
|
+
*/
|
|
1077
|
+
async touch(key, additionalSeconds) {
|
|
1078
|
+
const [ok, err, response] = await tryFn(() => this.client.getObject(key));
|
|
1079
|
+
if (!ok) {
|
|
1080
|
+
return false;
|
|
1081
|
+
}
|
|
1082
|
+
const metadata = response.Metadata || {};
|
|
1083
|
+
const parsedMetadata = this._parseMetadataValues(metadata);
|
|
1084
|
+
let data = parsedMetadata;
|
|
1085
|
+
if (response.Body) {
|
|
1086
|
+
try {
|
|
1087
|
+
const bodyContent = await response.Body.transformToString();
|
|
1088
|
+
if (bodyContent && bodyContent.trim()) {
|
|
1089
|
+
const body = JSON.parse(bodyContent);
|
|
1090
|
+
data = { ...parsedMetadata, ...body };
|
|
1091
|
+
}
|
|
1092
|
+
} catch {
|
|
1093
|
+
return false;
|
|
1094
|
+
}
|
|
1095
|
+
}
|
|
1096
|
+
const expiresAt = data._expiresat || data._expiresAt;
|
|
1097
|
+
if (!expiresAt) {
|
|
1098
|
+
return false;
|
|
1099
|
+
}
|
|
1100
|
+
data._expiresAt = expiresAt + additionalSeconds * 1e3;
|
|
1101
|
+
delete data._expiresat;
|
|
1102
|
+
const { metadata: newMetadata, body: newBody } = this._applyBehavior(data, "body-overflow");
|
|
1103
|
+
const putParams = {
|
|
1104
|
+
key,
|
|
1105
|
+
metadata: newMetadata,
|
|
1106
|
+
contentType: "application/json"
|
|
1107
|
+
};
|
|
1108
|
+
if (newBody !== null) {
|
|
1109
|
+
putParams.body = JSON.stringify(newBody);
|
|
1110
|
+
}
|
|
1111
|
+
const [putOk] = await tryFn(() => this.client.putObject(putParams));
|
|
1112
|
+
return putOk;
|
|
1113
|
+
}
|
|
975
1114
|
/**
|
|
976
1115
|
* Delete a single object
|
|
977
1116
|
*
|
|
@@ -1049,6 +1188,78 @@ class PluginStorage {
|
|
|
1049
1188
|
}
|
|
1050
1189
|
return results;
|
|
1051
1190
|
}
|
|
1191
|
+
/**
|
|
1192
|
+
* Acquire a distributed lock with TTL and retry logic
|
|
1193
|
+
*
|
|
1194
|
+
* @param {string} lockName - Lock identifier
|
|
1195
|
+
* @param {Object} options - Lock options
|
|
1196
|
+
* @param {number} options.ttl - Lock TTL in seconds (default: 30)
|
|
1197
|
+
* @param {number} options.timeout - Max wait time in ms (default: 0, no wait)
|
|
1198
|
+
* @param {string} options.workerId - Worker identifier (default: 'unknown')
|
|
1199
|
+
* @returns {Promise<Object|null>} Lock object or null if couldn't acquire
|
|
1200
|
+
*/
|
|
1201
|
+
async acquireLock(lockName, options = {}) {
|
|
1202
|
+
const { ttl = 30, timeout = 0, workerId = "unknown" } = options;
|
|
1203
|
+
const key = this.getPluginKey(null, "locks", lockName);
|
|
1204
|
+
const startTime = Date.now();
|
|
1205
|
+
while (true) {
|
|
1206
|
+
const existing = await this.get(key);
|
|
1207
|
+
if (!existing) {
|
|
1208
|
+
await this.set(key, { workerId, acquiredAt: Date.now() }, { ttl });
|
|
1209
|
+
return { key, workerId };
|
|
1210
|
+
}
|
|
1211
|
+
if (Date.now() - startTime >= timeout) {
|
|
1212
|
+
return null;
|
|
1213
|
+
}
|
|
1214
|
+
await new Promise((resolve) => setTimeout(resolve, 100));
|
|
1215
|
+
}
|
|
1216
|
+
}
|
|
1217
|
+
/**
|
|
1218
|
+
* Release a distributed lock
|
|
1219
|
+
*
|
|
1220
|
+
* @param {string} lockName - Lock identifier
|
|
1221
|
+
* @returns {Promise<void>}
|
|
1222
|
+
*/
|
|
1223
|
+
async releaseLock(lockName) {
|
|
1224
|
+
const key = this.getPluginKey(null, "locks", lockName);
|
|
1225
|
+
await this.delete(key);
|
|
1226
|
+
}
|
|
1227
|
+
/**
|
|
1228
|
+
* Check if a lock is currently held
|
|
1229
|
+
*
|
|
1230
|
+
* @param {string} lockName - Lock identifier
|
|
1231
|
+
* @returns {Promise<boolean>} True if locked
|
|
1232
|
+
*/
|
|
1233
|
+
async isLocked(lockName) {
|
|
1234
|
+
const key = this.getPluginKey(null, "locks", lockName);
|
|
1235
|
+
const lock = await this.get(key);
|
|
1236
|
+
return lock !== null;
|
|
1237
|
+
}
|
|
1238
|
+
/**
|
|
1239
|
+
* Increment a counter value
|
|
1240
|
+
*
|
|
1241
|
+
* @param {string} key - S3 key
|
|
1242
|
+
* @param {number} amount - Amount to increment (default: 1)
|
|
1243
|
+
* @param {Object} options - Options (e.g., ttl)
|
|
1244
|
+
* @returns {Promise<number>} New value
|
|
1245
|
+
*/
|
|
1246
|
+
async increment(key, amount = 1, options = {}) {
|
|
1247
|
+
const data = await this.get(key);
|
|
1248
|
+
const value = (data?.value || 0) + amount;
|
|
1249
|
+
await this.set(key, { value }, options);
|
|
1250
|
+
return value;
|
|
1251
|
+
}
|
|
1252
|
+
/**
|
|
1253
|
+
* Decrement a counter value
|
|
1254
|
+
*
|
|
1255
|
+
* @param {string} key - S3 key
|
|
1256
|
+
* @param {number} amount - Amount to decrement (default: 1)
|
|
1257
|
+
* @param {Object} options - Options (e.g., ttl)
|
|
1258
|
+
* @returns {Promise<number>} New value
|
|
1259
|
+
*/
|
|
1260
|
+
async decrement(key, amount = 1, options = {}) {
|
|
1261
|
+
return this.increment(key, -amount, options);
|
|
1262
|
+
}
|
|
1052
1263
|
/**
|
|
1053
1264
|
* Apply behavior to split data between metadata and body
|
|
1054
1265
|
*
|
|
@@ -1352,12 +1563,18 @@ class AuditPlugin extends Plugin {
|
|
|
1352
1563
|
recordId: "string|required",
|
|
1353
1564
|
userId: "string|optional",
|
|
1354
1565
|
timestamp: "string|required",
|
|
1566
|
+
createdAt: "string|required",
|
|
1567
|
+
// YYYY-MM-DD for partitioning
|
|
1355
1568
|
oldData: "string|optional",
|
|
1356
1569
|
newData: "string|optional",
|
|
1357
1570
|
partition: "string|optional",
|
|
1358
1571
|
partitionValues: "string|optional",
|
|
1359
1572
|
metadata: "string|optional"
|
|
1360
1573
|
},
|
|
1574
|
+
partitions: {
|
|
1575
|
+
byDate: { fields: { createdAt: "string|maxlength:10" } },
|
|
1576
|
+
byResource: { fields: { resourceName: "string" } }
|
|
1577
|
+
},
|
|
1361
1578
|
behavior: "body-overflow"
|
|
1362
1579
|
}));
|
|
1363
1580
|
this.auditResource = ok ? auditResource : this.database.resources.plg_audits || null;
|
|
@@ -1461,10 +1678,13 @@ class AuditPlugin extends Plugin {
|
|
|
1461
1678
|
if (!this.auditResource) {
|
|
1462
1679
|
return;
|
|
1463
1680
|
}
|
|
1681
|
+
const now = /* @__PURE__ */ new Date();
|
|
1464
1682
|
const auditRecord = {
|
|
1465
1683
|
id: `audit-${Date.now()}-${Math.random().toString(36).substring(2, 11)}`,
|
|
1466
1684
|
userId: this.getCurrentUserId?.() || "system",
|
|
1467
|
-
timestamp:
|
|
1685
|
+
timestamp: now.toISOString(),
|
|
1686
|
+
createdAt: now.toISOString().slice(0, 10),
|
|
1687
|
+
// YYYY-MM-DD for partitioning
|
|
1468
1688
|
metadata: JSON.stringify({ source: "audit-plugin", version: "2.0" }),
|
|
1469
1689
|
resourceName: auditData.resourceName,
|
|
1470
1690
|
operation: auditData.operation,
|
|
@@ -1539,9 +1759,25 @@ class AuditPlugin extends Plugin {
|
|
|
1539
1759
|
async getAuditLogs(options = {}) {
|
|
1540
1760
|
if (!this.auditResource) return [];
|
|
1541
1761
|
const { resourceName, operation, recordId, partition, startDate, endDate, limit = 100, offset = 0 } = options;
|
|
1542
|
-
const hasFilters = resourceName || operation || recordId || partition || startDate || endDate;
|
|
1543
1762
|
let items = [];
|
|
1544
|
-
if (
|
|
1763
|
+
if (resourceName && !operation && !recordId && !partition && !startDate && !endDate) {
|
|
1764
|
+
const [ok, err, result] = await tryFn(
|
|
1765
|
+
() => this.auditResource.query({ resourceName }, { limit: limit + offset })
|
|
1766
|
+
);
|
|
1767
|
+
items = ok && result ? result : [];
|
|
1768
|
+
return items.slice(offset, offset + limit);
|
|
1769
|
+
} else if (startDate && !resourceName && !operation && !recordId && !partition) {
|
|
1770
|
+
const dates = this._generateDateRange(startDate, endDate);
|
|
1771
|
+
for (const date of dates) {
|
|
1772
|
+
const [ok, err, result] = await tryFn(
|
|
1773
|
+
() => this.auditResource.query({ createdAt: date })
|
|
1774
|
+
);
|
|
1775
|
+
if (ok && result) {
|
|
1776
|
+
items.push(...result);
|
|
1777
|
+
}
|
|
1778
|
+
}
|
|
1779
|
+
return items.slice(offset, offset + limit);
|
|
1780
|
+
} else if (resourceName || operation || recordId || partition || startDate || endDate) {
|
|
1545
1781
|
const fetchSize = Math.min(1e4, Math.max(1e3, (limit + offset) * 20));
|
|
1546
1782
|
const result = await this.auditResource.list({ limit: fetchSize });
|
|
1547
1783
|
items = result || [];
|
|
@@ -1571,6 +1807,15 @@ class AuditPlugin extends Plugin {
|
|
|
1571
1807
|
return result.items || [];
|
|
1572
1808
|
}
|
|
1573
1809
|
}
|
|
1810
|
+
_generateDateRange(startDate, endDate) {
|
|
1811
|
+
const dates = [];
|
|
1812
|
+
const start = new Date(startDate);
|
|
1813
|
+
const end = endDate ? new Date(endDate) : /* @__PURE__ */ new Date();
|
|
1814
|
+
for (let d = new Date(start); d <= end; d.setDate(d.getDate() + 1)) {
|
|
1815
|
+
dates.push(d.toISOString().slice(0, 10));
|
|
1816
|
+
}
|
|
1817
|
+
return dates;
|
|
1818
|
+
}
|
|
1574
1819
|
async getRecordHistory(resourceName, recordId) {
|
|
1575
1820
|
return await this.getAuditLogs({ resourceName, recordId });
|
|
1576
1821
|
}
|
|
@@ -1603,6 +1848,37 @@ class AuditPlugin extends Plugin {
|
|
|
1603
1848
|
}
|
|
1604
1849
|
return stats;
|
|
1605
1850
|
}
|
|
1851
|
+
/**
|
|
1852
|
+
* Clean up audit logs older than retention period
|
|
1853
|
+
* @param {number} retentionDays - Number of days to retain (default: 90)
|
|
1854
|
+
* @returns {Promise<number>} Number of records deleted
|
|
1855
|
+
*/
|
|
1856
|
+
async cleanupOldAudits(retentionDays = 90) {
|
|
1857
|
+
if (!this.auditResource) return 0;
|
|
1858
|
+
const cutoffDate = /* @__PURE__ */ new Date();
|
|
1859
|
+
cutoffDate.setDate(cutoffDate.getDate() - retentionDays);
|
|
1860
|
+
const datesToDelete = [];
|
|
1861
|
+
const startDate = new Date(cutoffDate);
|
|
1862
|
+
startDate.setDate(startDate.getDate() - 365);
|
|
1863
|
+
for (let d = new Date(startDate); d < cutoffDate; d.setDate(d.getDate() + 1)) {
|
|
1864
|
+
datesToDelete.push(d.toISOString().slice(0, 10));
|
|
1865
|
+
}
|
|
1866
|
+
let deletedCount = 0;
|
|
1867
|
+
for (const dateStr of datesToDelete) {
|
|
1868
|
+
const [ok, err, oldAudits] = await tryFn(
|
|
1869
|
+
() => this.auditResource.query({ createdAt: dateStr })
|
|
1870
|
+
);
|
|
1871
|
+
if (ok && oldAudits) {
|
|
1872
|
+
for (const audit of oldAudits) {
|
|
1873
|
+
const [delOk] = await tryFn(() => this.auditResource.delete(audit.id));
|
|
1874
|
+
if (delOk) {
|
|
1875
|
+
deletedCount++;
|
|
1876
|
+
}
|
|
1877
|
+
}
|
|
1878
|
+
}
|
|
1879
|
+
}
|
|
1880
|
+
return deletedCount;
|
|
1881
|
+
}
|
|
1606
1882
|
}
|
|
1607
1883
|
|
|
1608
1884
|
class BaseBackupDriver {
|
|
@@ -4838,6 +5114,8 @@ function createConfig(options, detectedTimezone) {
|
|
|
4838
5114
|
consolidationWindow: consolidation.window ?? 24,
|
|
4839
5115
|
autoConsolidate: consolidation.auto !== false,
|
|
4840
5116
|
mode: consolidation.mode || "async",
|
|
5117
|
+
// ✅ NOVO: Performance tuning - Mark applied concurrency (default 50, antes era 10 hardcoded)
|
|
5118
|
+
markAppliedConcurrency: consolidation.markAppliedConcurrency ?? 50,
|
|
4841
5119
|
// Late arrivals
|
|
4842
5120
|
lateArrivalStrategy: lateArrivals.strategy || "warn",
|
|
4843
5121
|
// Batch transactions
|
|
@@ -4941,6 +5219,21 @@ function getTimezoneOffset(timezone, verbose = false) {
|
|
|
4941
5219
|
return offsets[timezone] || 0;
|
|
4942
5220
|
}
|
|
4943
5221
|
}
|
|
5222
|
+
function getISOWeek(date) {
|
|
5223
|
+
const target = new Date(date.valueOf());
|
|
5224
|
+
const dayNr = (date.getUTCDay() + 6) % 7;
|
|
5225
|
+
target.setUTCDate(target.getUTCDate() - dayNr + 3);
|
|
5226
|
+
const yearStart = new Date(Date.UTC(target.getUTCFullYear(), 0, 1));
|
|
5227
|
+
const firstThursday = new Date(yearStart.valueOf());
|
|
5228
|
+
if (yearStart.getUTCDay() !== 4) {
|
|
5229
|
+
firstThursday.setUTCDate(yearStart.getUTCDate() + (4 - yearStart.getUTCDay() + 7) % 7);
|
|
5230
|
+
}
|
|
5231
|
+
const weekNumber = 1 + Math.round((target - firstThursday) / 6048e5);
|
|
5232
|
+
return {
|
|
5233
|
+
year: target.getUTCFullYear(),
|
|
5234
|
+
week: weekNumber
|
|
5235
|
+
};
|
|
5236
|
+
}
|
|
4944
5237
|
function getCohortInfo(date, timezone, verbose = false) {
|
|
4945
5238
|
const offset = getTimezoneOffset(timezone, verbose);
|
|
4946
5239
|
const localDate = new Date(date.getTime() + offset);
|
|
@@ -4948,10 +5241,14 @@ function getCohortInfo(date, timezone, verbose = false) {
|
|
|
4948
5241
|
const month = String(localDate.getMonth() + 1).padStart(2, "0");
|
|
4949
5242
|
const day = String(localDate.getDate()).padStart(2, "0");
|
|
4950
5243
|
const hour = String(localDate.getHours()).padStart(2, "0");
|
|
5244
|
+
const { year: weekYear, week: weekNumber } = getISOWeek(localDate);
|
|
5245
|
+
const week = `${weekYear}-W${String(weekNumber).padStart(2, "0")}`;
|
|
4951
5246
|
return {
|
|
4952
5247
|
date: `${year}-${month}-${day}`,
|
|
4953
5248
|
hour: `${year}-${month}-${day}T${hour}`,
|
|
4954
5249
|
// ISO-like format for hour partition
|
|
5250
|
+
week,
|
|
5251
|
+
// ISO 8601 week format (e.g., '2025-W42')
|
|
4955
5252
|
month: `${year}-${month}`
|
|
4956
5253
|
};
|
|
4957
5254
|
}
|
|
@@ -5029,6 +5326,11 @@ function createPartitionConfig() {
|
|
|
5029
5326
|
cohortDate: "string"
|
|
5030
5327
|
}
|
|
5031
5328
|
},
|
|
5329
|
+
byWeek: {
|
|
5330
|
+
fields: {
|
|
5331
|
+
cohortWeek: "string"
|
|
5332
|
+
}
|
|
5333
|
+
},
|
|
5032
5334
|
byMonth: {
|
|
5033
5335
|
fields: {
|
|
5034
5336
|
cohortMonth: "string"
|
|
@@ -5068,6 +5370,7 @@ async function createTransaction(handler, data, config) {
|
|
|
5068
5370
|
timestamp: now.toISOString(),
|
|
5069
5371
|
cohortDate: cohortInfo.date,
|
|
5070
5372
|
cohortHour: cohortInfo.hour,
|
|
5373
|
+
cohortWeek: cohortInfo.week,
|
|
5071
5374
|
cohortMonth: cohortInfo.month,
|
|
5072
5375
|
source: data.source || "unknown",
|
|
5073
5376
|
applied: false
|
|
@@ -5108,50 +5411,6 @@ async function flushPendingTransactions(handler) {
|
|
|
5108
5411
|
}
|
|
5109
5412
|
}
|
|
5110
5413
|
|
|
5111
|
-
async function cleanupStaleLocks(lockResource, config) {
|
|
5112
|
-
const now = Date.now();
|
|
5113
|
-
const lockTimeoutMs = config.lockTimeout * 1e3;
|
|
5114
|
-
const cutoffTime = now - lockTimeoutMs;
|
|
5115
|
-
const cleanupLockId = `lock-cleanup-${config.resource}-${config.field}`;
|
|
5116
|
-
const [lockAcquired] = await tryFn(
|
|
5117
|
-
() => lockResource.insert({
|
|
5118
|
-
id: cleanupLockId,
|
|
5119
|
-
lockedAt: Date.now(),
|
|
5120
|
-
workerId: process.pid ? String(process.pid) : "unknown"
|
|
5121
|
-
})
|
|
5122
|
-
);
|
|
5123
|
-
if (!lockAcquired) {
|
|
5124
|
-
if (config.verbose) {
|
|
5125
|
-
console.log(`[EventualConsistency] Lock cleanup already running in another container`);
|
|
5126
|
-
}
|
|
5127
|
-
return;
|
|
5128
|
-
}
|
|
5129
|
-
try {
|
|
5130
|
-
const [ok, err, locks] = await tryFn(() => lockResource.list());
|
|
5131
|
-
if (!ok || !locks || locks.length === 0) return;
|
|
5132
|
-
const staleLocks = locks.filter(
|
|
5133
|
-
(lock) => lock.id !== cleanupLockId && lock.lockedAt < cutoffTime
|
|
5134
|
-
);
|
|
5135
|
-
if (staleLocks.length === 0) return;
|
|
5136
|
-
if (config.verbose) {
|
|
5137
|
-
console.log(`[EventualConsistency] Cleaning up ${staleLocks.length} stale locks`);
|
|
5138
|
-
}
|
|
5139
|
-
const { results, errors } = await promisePool.PromisePool.for(staleLocks).withConcurrency(5).process(async (lock) => {
|
|
5140
|
-
const [deleted] = await tryFn(() => lockResource.delete(lock.id));
|
|
5141
|
-
return deleted;
|
|
5142
|
-
});
|
|
5143
|
-
if (errors && errors.length > 0 && config.verbose) {
|
|
5144
|
-
console.warn(`[EventualConsistency] ${errors.length} stale locks failed to delete`);
|
|
5145
|
-
}
|
|
5146
|
-
} catch (error) {
|
|
5147
|
-
if (config.verbose) {
|
|
5148
|
-
console.warn(`[EventualConsistency] Error cleaning up stale locks:`, error.message);
|
|
5149
|
-
}
|
|
5150
|
-
} finally {
|
|
5151
|
-
await tryFn(() => lockResource.delete(cleanupLockId));
|
|
5152
|
-
}
|
|
5153
|
-
}
|
|
5154
|
-
|
|
5155
5414
|
function startConsolidationTimer(handler, resourceName, fieldName, runConsolidationCallback, config) {
|
|
5156
5415
|
const intervalMs = config.consolidationInterval * 1e3;
|
|
5157
5416
|
if (config.verbose) {
|
|
@@ -5248,17 +5507,15 @@ async function runConsolidation(transactionResource, consolidateRecordFn, emitFn
|
|
|
5248
5507
|
}
|
|
5249
5508
|
}
|
|
5250
5509
|
}
|
|
5251
|
-
async function consolidateRecord(originalId, transactionResource, targetResource,
|
|
5252
|
-
|
|
5253
|
-
const
|
|
5254
|
-
|
|
5255
|
-
|
|
5256
|
-
|
|
5257
|
-
|
|
5258
|
-
|
|
5259
|
-
|
|
5260
|
-
);
|
|
5261
|
-
if (!lockAcquired) {
|
|
5510
|
+
async function consolidateRecord(originalId, transactionResource, targetResource, storage, analyticsResource, updateAnalyticsFn, config) {
|
|
5511
|
+
const lockKey = `consolidation-${config.resource}-${config.field}-${originalId}`;
|
|
5512
|
+
const lock = await storage.acquireLock(lockKey, {
|
|
5513
|
+
ttl: config.lockTimeout || 30,
|
|
5514
|
+
timeout: 0,
|
|
5515
|
+
// Don't wait if locked
|
|
5516
|
+
workerId: process.pid ? String(process.pid) : "unknown"
|
|
5517
|
+
});
|
|
5518
|
+
if (!lock) {
|
|
5262
5519
|
if (config.verbose) {
|
|
5263
5520
|
console.log(`[EventualConsistency] Lock for ${originalId} already held, skipping`);
|
|
5264
5521
|
}
|
|
@@ -5469,7 +5726,8 @@ async function consolidateRecord(originalId, transactionResource, targetResource
|
|
|
5469
5726
|
}
|
|
5470
5727
|
if (updateOk) {
|
|
5471
5728
|
const transactionsToUpdate = transactions.filter((txn) => txn.id !== "__synthetic__");
|
|
5472
|
-
const
|
|
5729
|
+
const markAppliedConcurrency = config.markAppliedConcurrency || 50;
|
|
5730
|
+
const { results, errors } = await promisePool.PromisePool.for(transactionsToUpdate).withConcurrency(markAppliedConcurrency).process(async (txn) => {
|
|
5473
5731
|
const [ok2, err2] = await tryFn(
|
|
5474
5732
|
() => transactionResource.update(txn.id, { applied: true })
|
|
5475
5733
|
);
|
|
@@ -5517,9 +5775,11 @@ async function consolidateRecord(originalId, transactionResource, targetResource
|
|
|
5517
5775
|
}
|
|
5518
5776
|
return consolidatedValue;
|
|
5519
5777
|
} finally {
|
|
5520
|
-
const [lockReleased, lockReleaseErr] = await tryFn(
|
|
5778
|
+
const [lockReleased, lockReleaseErr] = await tryFn(
|
|
5779
|
+
() => storage.releaseLock(lockKey)
|
|
5780
|
+
);
|
|
5521
5781
|
if (!lockReleased && config.verbose) {
|
|
5522
|
-
console.warn(`[EventualConsistency] Failed to release lock ${
|
|
5782
|
+
console.warn(`[EventualConsistency] Failed to release lock ${lockKey}:`, lockReleaseErr?.message);
|
|
5523
5783
|
}
|
|
5524
5784
|
}
|
|
5525
5785
|
}
|
|
@@ -5593,17 +5853,15 @@ async function getCohortStats(cohortDate, transactionResource) {
|
|
|
5593
5853
|
}
|
|
5594
5854
|
return stats;
|
|
5595
5855
|
}
|
|
5596
|
-
async function recalculateRecord(originalId, transactionResource, targetResource,
|
|
5597
|
-
|
|
5598
|
-
const
|
|
5599
|
-
|
|
5600
|
-
|
|
5601
|
-
|
|
5602
|
-
|
|
5603
|
-
|
|
5604
|
-
|
|
5605
|
-
);
|
|
5606
|
-
if (!lockAcquired) {
|
|
5856
|
+
async function recalculateRecord(originalId, transactionResource, targetResource, storage, consolidateRecordFn, config) {
|
|
5857
|
+
const lockKey = `recalculate-${config.resource}-${config.field}-${originalId}`;
|
|
5858
|
+
const lock = await storage.acquireLock(lockKey, {
|
|
5859
|
+
ttl: config.lockTimeout || 30,
|
|
5860
|
+
timeout: 0,
|
|
5861
|
+
// Don't wait if locked
|
|
5862
|
+
workerId: process.pid ? String(process.pid) : "unknown"
|
|
5863
|
+
});
|
|
5864
|
+
if (!lock) {
|
|
5607
5865
|
if (config.verbose) {
|
|
5608
5866
|
console.log(`[EventualConsistency] Recalculate lock for ${originalId} already held, skipping`);
|
|
5609
5867
|
}
|
|
@@ -5671,9 +5929,11 @@ async function recalculateRecord(originalId, transactionResource, targetResource
|
|
|
5671
5929
|
}
|
|
5672
5930
|
return consolidatedValue;
|
|
5673
5931
|
} finally {
|
|
5674
|
-
const [lockReleased, lockReleaseErr] = await tryFn(
|
|
5932
|
+
const [lockReleased, lockReleaseErr] = await tryFn(
|
|
5933
|
+
() => storage.releaseLock(lockKey)
|
|
5934
|
+
);
|
|
5675
5935
|
if (!lockReleased && config.verbose) {
|
|
5676
|
-
console.warn(`[EventualConsistency] Failed to release recalculate lock ${
|
|
5936
|
+
console.warn(`[EventualConsistency] Failed to release recalculate lock ${lockKey}:`, lockReleaseErr?.message);
|
|
5677
5937
|
}
|
|
5678
5938
|
}
|
|
5679
5939
|
}
|
|
@@ -5685,16 +5945,16 @@ function startGarbageCollectionTimer(handler, resourceName, fieldName, runGCCall
|
|
|
5685
5945
|
}, gcIntervalMs);
|
|
5686
5946
|
return handler.gcTimer;
|
|
5687
5947
|
}
|
|
5688
|
-
async function runGarbageCollection(transactionResource,
|
|
5689
|
-
const
|
|
5690
|
-
const
|
|
5691
|
-
|
|
5692
|
-
|
|
5693
|
-
|
|
5694
|
-
|
|
5695
|
-
|
|
5696
|
-
);
|
|
5697
|
-
if (!
|
|
5948
|
+
async function runGarbageCollection(transactionResource, storage, config, emitFn) {
|
|
5949
|
+
const lockKey = `gc-${config.resource}-${config.field}`;
|
|
5950
|
+
const lock = await storage.acquireLock(lockKey, {
|
|
5951
|
+
ttl: 300,
|
|
5952
|
+
// 5 minutes for GC
|
|
5953
|
+
timeout: 0,
|
|
5954
|
+
// Don't wait if locked
|
|
5955
|
+
workerId: process.pid ? String(process.pid) : "unknown"
|
|
5956
|
+
});
|
|
5957
|
+
if (!lock) {
|
|
5698
5958
|
if (config.verbose) {
|
|
5699
5959
|
console.log(`[EventualConsistency] GC already running in another container`);
|
|
5700
5960
|
}
|
|
@@ -5752,7 +6012,7 @@ async function runGarbageCollection(transactionResource, lockResource, config, e
|
|
|
5752
6012
|
emitFn("eventual-consistency.gc-error", error);
|
|
5753
6013
|
}
|
|
5754
6014
|
} finally {
|
|
5755
|
-
await tryFn(() =>
|
|
6015
|
+
await tryFn(() => storage.releaseLock(lockKey));
|
|
5756
6016
|
}
|
|
5757
6017
|
}
|
|
5758
6018
|
|
|
@@ -5777,22 +6037,26 @@ AnalyticsResource: ${analyticsResource?.name || "unknown"}`
|
|
|
5777
6037
|
const cohortCount = Object.keys(byHour).length;
|
|
5778
6038
|
if (config.verbose) {
|
|
5779
6039
|
console.log(
|
|
5780
|
-
`[EventualConsistency] ${config.resource}.${config.field} - Updating ${cohortCount} hourly analytics cohorts...`
|
|
6040
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Updating ${cohortCount} hourly analytics cohorts IN PARALLEL...`
|
|
5781
6041
|
);
|
|
5782
6042
|
}
|
|
5783
|
-
|
|
5784
|
-
|
|
5785
|
-
|
|
6043
|
+
await Promise.all(
|
|
6044
|
+
Object.entries(byHour).map(
|
|
6045
|
+
([cohort, txns]) => upsertAnalytics("hour", cohort, txns, analyticsResource, config)
|
|
6046
|
+
)
|
|
6047
|
+
);
|
|
5786
6048
|
if (config.analyticsConfig.rollupStrategy === "incremental") {
|
|
5787
6049
|
const uniqueHours = Object.keys(byHour);
|
|
5788
6050
|
if (config.verbose) {
|
|
5789
6051
|
console.log(
|
|
5790
|
-
`[EventualConsistency] ${config.resource}.${config.field} - Rolling up ${uniqueHours.length} hours to daily/monthly analytics...`
|
|
6052
|
+
`[EventualConsistency] ${config.resource}.${config.field} - Rolling up ${uniqueHours.length} hours to daily/weekly/monthly analytics IN PARALLEL...`
|
|
5791
6053
|
);
|
|
5792
6054
|
}
|
|
5793
|
-
|
|
5794
|
-
|
|
5795
|
-
|
|
6055
|
+
await Promise.all(
|
|
6056
|
+
uniqueHours.map(
|
|
6057
|
+
(cohortHour) => rollupAnalytics(cohortHour, analyticsResource, config)
|
|
6058
|
+
)
|
|
6059
|
+
);
|
|
5796
6060
|
}
|
|
5797
6061
|
if (config.verbose) {
|
|
5798
6062
|
console.log(
|
|
@@ -5895,18 +6159,53 @@ function calculateOperationBreakdown(transactions) {
|
|
|
5895
6159
|
async function rollupAnalytics(cohortHour, analyticsResource, config) {
|
|
5896
6160
|
const cohortDate = cohortHour.substring(0, 10);
|
|
5897
6161
|
const cohortMonth = cohortHour.substring(0, 7);
|
|
6162
|
+
const date = new Date(cohortDate);
|
|
6163
|
+
const cohortWeek = getCohortWeekFromDate(date);
|
|
5898
6164
|
await rollupPeriod("day", cohortDate, cohortDate, analyticsResource, config);
|
|
6165
|
+
await rollupPeriod("week", cohortWeek, cohortWeek, analyticsResource, config);
|
|
5899
6166
|
await rollupPeriod("month", cohortMonth, cohortMonth, analyticsResource, config);
|
|
5900
6167
|
}
|
|
6168
|
+
function getCohortWeekFromDate(date) {
|
|
6169
|
+
const target = new Date(date.valueOf());
|
|
6170
|
+
const dayNr = (date.getUTCDay() + 6) % 7;
|
|
6171
|
+
target.setUTCDate(target.getUTCDate() - dayNr + 3);
|
|
6172
|
+
const yearStart = new Date(Date.UTC(target.getUTCFullYear(), 0, 1));
|
|
6173
|
+
const firstThursday = new Date(yearStart.valueOf());
|
|
6174
|
+
if (yearStart.getUTCDay() !== 4) {
|
|
6175
|
+
firstThursday.setUTCDate(yearStart.getUTCDate() + (4 - yearStart.getUTCDay() + 7) % 7);
|
|
6176
|
+
}
|
|
6177
|
+
const weekNumber = 1 + Math.round((target - firstThursday) / 6048e5);
|
|
6178
|
+
const weekYear = target.getUTCFullYear();
|
|
6179
|
+
return `${weekYear}-W${String(weekNumber).padStart(2, "0")}`;
|
|
6180
|
+
}
|
|
5901
6181
|
async function rollupPeriod(period, cohort, sourcePrefix, analyticsResource, config) {
|
|
5902
|
-
|
|
6182
|
+
let sourcePeriod;
|
|
6183
|
+
if (period === "day") {
|
|
6184
|
+
sourcePeriod = "hour";
|
|
6185
|
+
} else if (period === "week") {
|
|
6186
|
+
sourcePeriod = "day";
|
|
6187
|
+
} else if (period === "month") {
|
|
6188
|
+
sourcePeriod = "day";
|
|
6189
|
+
} else {
|
|
6190
|
+
sourcePeriod = "day";
|
|
6191
|
+
}
|
|
5903
6192
|
const [ok, err, allAnalytics] = await tryFn(
|
|
5904
6193
|
() => analyticsResource.list()
|
|
5905
6194
|
);
|
|
5906
6195
|
if (!ok || !allAnalytics) return;
|
|
5907
|
-
|
|
5908
|
-
|
|
5909
|
-
|
|
6196
|
+
let sourceAnalytics;
|
|
6197
|
+
if (period === "week") {
|
|
6198
|
+
sourceAnalytics = allAnalytics.filter((a) => {
|
|
6199
|
+
if (a.period !== sourcePeriod) return false;
|
|
6200
|
+
const dayDate = new Date(a.cohort);
|
|
6201
|
+
const dayWeek = getCohortWeekFromDate(dayDate);
|
|
6202
|
+
return dayWeek === cohort;
|
|
6203
|
+
});
|
|
6204
|
+
} else {
|
|
6205
|
+
sourceAnalytics = allAnalytics.filter(
|
|
6206
|
+
(a) => a.period === sourcePeriod && a.cohort.startsWith(sourcePrefix)
|
|
6207
|
+
);
|
|
6208
|
+
}
|
|
5910
6209
|
if (sourceAnalytics.length === 0) return;
|
|
5911
6210
|
const transactionCount = sourceAnalytics.reduce((sum, a) => sum + a.transactionCount, 0);
|
|
5912
6211
|
const totalValue = sourceAnalytics.reduce((sum, a) => sum + a.totalValue, 0);
|
|
@@ -6115,6 +6414,32 @@ async function getYearByMonth(resourceName, field, year, options, fieldHandlers)
|
|
|
6115
6414
|
}
|
|
6116
6415
|
return data;
|
|
6117
6416
|
}
|
|
6417
|
+
async function getYearByWeek(resourceName, field, year, options, fieldHandlers) {
|
|
6418
|
+
const data = await getAnalytics(resourceName, field, {
|
|
6419
|
+
period: "week",
|
|
6420
|
+
year
|
|
6421
|
+
}, fieldHandlers);
|
|
6422
|
+
if (options.fillGaps) {
|
|
6423
|
+
const startWeek = `${year}-W01`;
|
|
6424
|
+
const endWeek = `${year}-W53`;
|
|
6425
|
+
return fillGaps(data, "week", startWeek, endWeek);
|
|
6426
|
+
}
|
|
6427
|
+
return data;
|
|
6428
|
+
}
|
|
6429
|
+
async function getMonthByWeek(resourceName, field, month, options, fieldHandlers) {
|
|
6430
|
+
const year = parseInt(month.substring(0, 4));
|
|
6431
|
+
const monthNum = parseInt(month.substring(5, 7));
|
|
6432
|
+
const firstDay = new Date(year, monthNum - 1, 1);
|
|
6433
|
+
const lastDay = new Date(year, monthNum, 0);
|
|
6434
|
+
const firstWeek = getCohortWeekFromDate(firstDay);
|
|
6435
|
+
const lastWeek = getCohortWeekFromDate(lastDay);
|
|
6436
|
+
const data = await getAnalytics(resourceName, field, {
|
|
6437
|
+
period: "week",
|
|
6438
|
+
startDate: firstWeek,
|
|
6439
|
+
endDate: lastWeek
|
|
6440
|
+
}, fieldHandlers);
|
|
6441
|
+
return data;
|
|
6442
|
+
}
|
|
6118
6443
|
async function getMonthByHour(resourceName, field, month, options, fieldHandlers) {
|
|
6119
6444
|
let year, monthNum;
|
|
6120
6445
|
if (month === "last") {
|
|
@@ -6342,7 +6667,7 @@ async function completeFieldSetup(handler, database, config, plugin) {
|
|
|
6342
6667
|
if (!handler.targetResource) return;
|
|
6343
6668
|
const resourceName = handler.resource;
|
|
6344
6669
|
const fieldName = handler.field;
|
|
6345
|
-
const transactionResourceName =
|
|
6670
|
+
const transactionResourceName = `plg_${resourceName}_tx_${fieldName}`;
|
|
6346
6671
|
const partitionConfig = createPartitionConfig();
|
|
6347
6672
|
const [ok, err, transactionResource] = await tryFn(
|
|
6348
6673
|
() => database.createResource({
|
|
@@ -6356,6 +6681,7 @@ async function completeFieldSetup(handler, database, config, plugin) {
|
|
|
6356
6681
|
timestamp: "string|required",
|
|
6357
6682
|
cohortDate: "string|required",
|
|
6358
6683
|
cohortHour: "string|required",
|
|
6684
|
+
cohortWeek: "string|optional",
|
|
6359
6685
|
cohortMonth: "string|optional",
|
|
6360
6686
|
source: "string|optional",
|
|
6361
6687
|
applied: "boolean|optional"
|
|
@@ -6371,36 +6697,18 @@ async function completeFieldSetup(handler, database, config, plugin) {
|
|
|
6371
6697
|
throw new Error(`Failed to create transaction resource for ${resourceName}.${fieldName}: ${err?.message}`);
|
|
6372
6698
|
}
|
|
6373
6699
|
handler.transactionResource = ok ? transactionResource : database.resources[transactionResourceName];
|
|
6374
|
-
const lockResourceName = `${resourceName}_consolidation_locks_${fieldName}`;
|
|
6375
|
-
const [lockOk, lockErr, lockResource] = await tryFn(
|
|
6376
|
-
() => database.createResource({
|
|
6377
|
-
name: lockResourceName,
|
|
6378
|
-
attributes: {
|
|
6379
|
-
id: "string|required",
|
|
6380
|
-
lockedAt: "number|required",
|
|
6381
|
-
workerId: "string|optional"
|
|
6382
|
-
},
|
|
6383
|
-
behavior: "body-only",
|
|
6384
|
-
timestamps: false,
|
|
6385
|
-
createdBy: "EventualConsistencyPlugin"
|
|
6386
|
-
})
|
|
6387
|
-
);
|
|
6388
|
-
if (!lockOk && !database.resources[lockResourceName]) {
|
|
6389
|
-
throw new Error(`Failed to create lock resource for ${resourceName}.${fieldName}: ${lockErr?.message}`);
|
|
6390
|
-
}
|
|
6391
|
-
handler.lockResource = lockOk ? lockResource : database.resources[lockResourceName];
|
|
6392
6700
|
if (config.enableAnalytics) {
|
|
6393
6701
|
await createAnalyticsResource(handler, database, resourceName, fieldName);
|
|
6394
6702
|
}
|
|
6395
6703
|
addHelperMethodsForHandler(handler, plugin, config);
|
|
6396
6704
|
if (config.verbose) {
|
|
6397
6705
|
console.log(
|
|
6398
|
-
`[EventualConsistency] ${resourceName}.${fieldName} - Setup complete. Resources: ${transactionResourceName}
|
|
6706
|
+
`[EventualConsistency] ${resourceName}.${fieldName} - Setup complete. Resources: ${transactionResourceName}${config.enableAnalytics ? `, plg_${resourceName}_an_${fieldName}` : ""} (locks via PluginStorage TTL)`
|
|
6399
6707
|
);
|
|
6400
6708
|
}
|
|
6401
6709
|
}
|
|
6402
6710
|
async function createAnalyticsResource(handler, database, resourceName, fieldName) {
|
|
6403
|
-
const analyticsResourceName =
|
|
6711
|
+
const analyticsResourceName = `plg_${resourceName}_an_${fieldName}`;
|
|
6404
6712
|
const [ok, err, analyticsResource] = await tryFn(
|
|
6405
6713
|
() => database.createResource({
|
|
6406
6714
|
name: analyticsResourceName,
|
|
@@ -6574,7 +6882,7 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
6574
6882
|
originalId,
|
|
6575
6883
|
this.transactionResource,
|
|
6576
6884
|
this.targetResource,
|
|
6577
|
-
this.
|
|
6885
|
+
this.getStorage(),
|
|
6578
6886
|
this.analyticsResource,
|
|
6579
6887
|
(transactions) => this.updateAnalytics(transactions),
|
|
6580
6888
|
this.config
|
|
@@ -6610,7 +6918,7 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
6610
6918
|
originalId,
|
|
6611
6919
|
this.transactionResource,
|
|
6612
6920
|
this.targetResource,
|
|
6613
|
-
this.
|
|
6921
|
+
this.getStorage(),
|
|
6614
6922
|
(id) => this.consolidateRecord(id),
|
|
6615
6923
|
this.config
|
|
6616
6924
|
);
|
|
@@ -6631,20 +6939,17 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
6631
6939
|
const oldField = this.config.field;
|
|
6632
6940
|
const oldTransactionResource = this.transactionResource;
|
|
6633
6941
|
const oldTargetResource = this.targetResource;
|
|
6634
|
-
const oldLockResource = this.lockResource;
|
|
6635
6942
|
const oldAnalyticsResource = this.analyticsResource;
|
|
6636
6943
|
this.config.resource = handler.resource;
|
|
6637
6944
|
this.config.field = handler.field;
|
|
6638
6945
|
this.transactionResource = handler.transactionResource;
|
|
6639
6946
|
this.targetResource = handler.targetResource;
|
|
6640
|
-
this.lockResource = handler.lockResource;
|
|
6641
6947
|
this.analyticsResource = handler.analyticsResource;
|
|
6642
6948
|
const result = await this.consolidateRecord(id);
|
|
6643
6949
|
this.config.resource = oldResource;
|
|
6644
6950
|
this.config.field = oldField;
|
|
6645
6951
|
this.transactionResource = oldTransactionResource;
|
|
6646
6952
|
this.targetResource = oldTargetResource;
|
|
6647
|
-
this.lockResource = oldLockResource;
|
|
6648
6953
|
this.analyticsResource = oldAnalyticsResource;
|
|
6649
6954
|
return result;
|
|
6650
6955
|
}
|
|
@@ -6657,20 +6962,17 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
6657
6962
|
const oldField = this.config.field;
|
|
6658
6963
|
const oldTransactionResource = this.transactionResource;
|
|
6659
6964
|
const oldTargetResource = this.targetResource;
|
|
6660
|
-
const oldLockResource = this.lockResource;
|
|
6661
6965
|
const oldAnalyticsResource = this.analyticsResource;
|
|
6662
6966
|
this.config.resource = handler.resource;
|
|
6663
6967
|
this.config.field = handler.field;
|
|
6664
6968
|
this.transactionResource = handler.transactionResource;
|
|
6665
6969
|
this.targetResource = handler.targetResource;
|
|
6666
|
-
this.lockResource = handler.lockResource;
|
|
6667
6970
|
this.analyticsResource = handler.analyticsResource;
|
|
6668
6971
|
const result = await this.consolidateRecord(id);
|
|
6669
6972
|
this.config.resource = oldResource;
|
|
6670
6973
|
this.config.field = oldField;
|
|
6671
6974
|
this.transactionResource = oldTransactionResource;
|
|
6672
6975
|
this.targetResource = oldTargetResource;
|
|
6673
|
-
this.lockResource = oldLockResource;
|
|
6674
6976
|
this.analyticsResource = oldAnalyticsResource;
|
|
6675
6977
|
return result;
|
|
6676
6978
|
}
|
|
@@ -6703,20 +7005,17 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
6703
7005
|
const oldField = this.config.field;
|
|
6704
7006
|
const oldTransactionResource = this.transactionResource;
|
|
6705
7007
|
const oldTargetResource = this.targetResource;
|
|
6706
|
-
const oldLockResource = this.lockResource;
|
|
6707
7008
|
const oldAnalyticsResource = this.analyticsResource;
|
|
6708
7009
|
this.config.resource = handler.resource;
|
|
6709
7010
|
this.config.field = handler.field;
|
|
6710
7011
|
this.transactionResource = handler.transactionResource;
|
|
6711
7012
|
this.targetResource = handler.targetResource;
|
|
6712
|
-
this.lockResource = handler.lockResource;
|
|
6713
7013
|
this.analyticsResource = handler.analyticsResource;
|
|
6714
7014
|
const result = await this.recalculateRecord(id);
|
|
6715
7015
|
this.config.resource = oldResource;
|
|
6716
7016
|
this.config.field = oldField;
|
|
6717
7017
|
this.transactionResource = oldTransactionResource;
|
|
6718
7018
|
this.targetResource = oldTargetResource;
|
|
6719
|
-
this.lockResource = oldLockResource;
|
|
6720
7019
|
this.analyticsResource = oldAnalyticsResource;
|
|
6721
7020
|
return result;
|
|
6722
7021
|
}
|
|
@@ -6729,13 +7028,11 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
6729
7028
|
const oldField = this.config.field;
|
|
6730
7029
|
const oldTransactionResource = this.transactionResource;
|
|
6731
7030
|
const oldTargetResource = this.targetResource;
|
|
6732
|
-
const oldLockResource = this.lockResource;
|
|
6733
7031
|
const oldAnalyticsResource = this.analyticsResource;
|
|
6734
7032
|
this.config.resource = resourceName;
|
|
6735
7033
|
this.config.field = fieldName;
|
|
6736
7034
|
this.transactionResource = handler.transactionResource;
|
|
6737
7035
|
this.targetResource = handler.targetResource;
|
|
6738
|
-
this.lockResource = handler.lockResource;
|
|
6739
7036
|
this.analyticsResource = handler.analyticsResource;
|
|
6740
7037
|
try {
|
|
6741
7038
|
await runConsolidation(
|
|
@@ -6749,7 +7046,6 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
6749
7046
|
this.config.field = oldField;
|
|
6750
7047
|
this.transactionResource = oldTransactionResource;
|
|
6751
7048
|
this.targetResource = oldTargetResource;
|
|
6752
|
-
this.lockResource = oldLockResource;
|
|
6753
7049
|
this.analyticsResource = oldAnalyticsResource;
|
|
6754
7050
|
}
|
|
6755
7051
|
}
|
|
@@ -6762,16 +7058,14 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
6762
7058
|
const oldField = this.config.field;
|
|
6763
7059
|
const oldTransactionResource = this.transactionResource;
|
|
6764
7060
|
const oldTargetResource = this.targetResource;
|
|
6765
|
-
const oldLockResource = this.lockResource;
|
|
6766
7061
|
this.config.resource = resourceName;
|
|
6767
7062
|
this.config.field = fieldName;
|
|
6768
7063
|
this.transactionResource = handler.transactionResource;
|
|
6769
7064
|
this.targetResource = handler.targetResource;
|
|
6770
|
-
this.lockResource = handler.lockResource;
|
|
6771
7065
|
try {
|
|
6772
7066
|
await runGarbageCollection(
|
|
6773
7067
|
this.transactionResource,
|
|
6774
|
-
this.
|
|
7068
|
+
this.getStorage(),
|
|
6775
7069
|
this.config,
|
|
6776
7070
|
(event, data) => this.emit(event, data)
|
|
6777
7071
|
);
|
|
@@ -6780,7 +7074,6 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
6780
7074
|
this.config.field = oldField;
|
|
6781
7075
|
this.transactionResource = oldTransactionResource;
|
|
6782
7076
|
this.targetResource = oldTargetResource;
|
|
6783
|
-
this.lockResource = oldLockResource;
|
|
6784
7077
|
}
|
|
6785
7078
|
}
|
|
6786
7079
|
// Public Analytics API
|
|
@@ -6849,6 +7142,28 @@ class EventualConsistencyPlugin extends Plugin {
|
|
|
6849
7142
|
async getMonthByHour(resourceName, field, month, options = {}) {
|
|
6850
7143
|
return await getMonthByHour(resourceName, field, month, options, this.fieldHandlers);
|
|
6851
7144
|
}
|
|
7145
|
+
/**
|
|
7146
|
+
* Get analytics for entire year, broken down by weeks
|
|
7147
|
+
* @param {string} resourceName - Resource name
|
|
7148
|
+
* @param {string} field - Field name
|
|
7149
|
+
* @param {number} year - Year (e.g., 2025)
|
|
7150
|
+
* @param {Object} options - Options
|
|
7151
|
+
* @returns {Promise<Array>} Weekly analytics for the year (up to 53 weeks)
|
|
7152
|
+
*/
|
|
7153
|
+
async getYearByWeek(resourceName, field, year, options = {}) {
|
|
7154
|
+
return await getYearByWeek(resourceName, field, year, options, this.fieldHandlers);
|
|
7155
|
+
}
|
|
7156
|
+
/**
|
|
7157
|
+
* Get analytics for entire month, broken down by weeks
|
|
7158
|
+
* @param {string} resourceName - Resource name
|
|
7159
|
+
* @param {string} field - Field name
|
|
7160
|
+
* @param {string} month - Month in YYYY-MM format
|
|
7161
|
+
* @param {Object} options - Options
|
|
7162
|
+
* @returns {Promise<Array>} Weekly analytics for the month
|
|
7163
|
+
*/
|
|
7164
|
+
async getMonthByWeek(resourceName, field, month, options = {}) {
|
|
7165
|
+
return await getMonthByWeek(resourceName, field, month, options, this.fieldHandlers);
|
|
7166
|
+
}
|
|
6852
7167
|
/**
|
|
6853
7168
|
* Get top records by volume
|
|
6854
7169
|
* @param {string} resourceName - Resource name
|
|
@@ -6871,6 +7186,8 @@ class FullTextPlugin extends Plugin {
|
|
|
6871
7186
|
...options
|
|
6872
7187
|
};
|
|
6873
7188
|
this.indexes = /* @__PURE__ */ new Map();
|
|
7189
|
+
this.dirtyIndexes = /* @__PURE__ */ new Set();
|
|
7190
|
+
this.deletedIndexes = /* @__PURE__ */ new Set();
|
|
6874
7191
|
}
|
|
6875
7192
|
async onInstall() {
|
|
6876
7193
|
const [ok, err, indexResource] = await tryFn(() => this.database.createResource({
|
|
@@ -6884,7 +7201,11 @@ class FullTextPlugin extends Plugin {
|
|
|
6884
7201
|
// Array of record IDs containing this word
|
|
6885
7202
|
count: "number|required",
|
|
6886
7203
|
lastUpdated: "string|required"
|
|
6887
|
-
}
|
|
7204
|
+
},
|
|
7205
|
+
partitions: {
|
|
7206
|
+
byResource: { fields: { resourceName: "string" } }
|
|
7207
|
+
},
|
|
7208
|
+
behavior: "body-overflow"
|
|
6888
7209
|
}));
|
|
6889
7210
|
this.indexResource = ok ? indexResource : this.database.resources.fulltext_indexes;
|
|
6890
7211
|
await this.loadIndexes();
|
|
@@ -6913,22 +7234,53 @@ class FullTextPlugin extends Plugin {
|
|
|
6913
7234
|
async saveIndexes() {
|
|
6914
7235
|
if (!this.indexResource) return;
|
|
6915
7236
|
const [ok, err] = await tryFn(async () => {
|
|
6916
|
-
const
|
|
6917
|
-
|
|
6918
|
-
await
|
|
7237
|
+
for (const key of this.deletedIndexes) {
|
|
7238
|
+
const [resourceName] = key.split(":");
|
|
7239
|
+
const [queryOk, queryErr, results] = await tryFn(
|
|
7240
|
+
() => this.indexResource.query({ resourceName })
|
|
7241
|
+
);
|
|
7242
|
+
if (queryOk && results) {
|
|
7243
|
+
for (const index of results) {
|
|
7244
|
+
const indexKey = `${index.resourceName}:${index.fieldName}:${index.word}`;
|
|
7245
|
+
if (indexKey === key) {
|
|
7246
|
+
await this.indexResource.delete(index.id);
|
|
7247
|
+
}
|
|
7248
|
+
}
|
|
7249
|
+
}
|
|
6919
7250
|
}
|
|
6920
|
-
for (const
|
|
7251
|
+
for (const key of this.dirtyIndexes) {
|
|
6921
7252
|
const [resourceName, fieldName, word] = key.split(":");
|
|
6922
|
-
|
|
6923
|
-
|
|
6924
|
-
|
|
6925
|
-
|
|
6926
|
-
|
|
6927
|
-
|
|
6928
|
-
|
|
6929
|
-
|
|
6930
|
-
|
|
7253
|
+
const data = this.indexes.get(key);
|
|
7254
|
+
if (!data) continue;
|
|
7255
|
+
const [queryOk, queryErr, results] = await tryFn(
|
|
7256
|
+
() => this.indexResource.query({ resourceName })
|
|
7257
|
+
);
|
|
7258
|
+
let existingRecord = null;
|
|
7259
|
+
if (queryOk && results) {
|
|
7260
|
+
existingRecord = results.find(
|
|
7261
|
+
(index) => index.resourceName === resourceName && index.fieldName === fieldName && index.word === word
|
|
7262
|
+
);
|
|
7263
|
+
}
|
|
7264
|
+
if (existingRecord) {
|
|
7265
|
+
await this.indexResource.update(existingRecord.id, {
|
|
7266
|
+
recordIds: data.recordIds,
|
|
7267
|
+
count: data.count,
|
|
7268
|
+
lastUpdated: (/* @__PURE__ */ new Date()).toISOString()
|
|
7269
|
+
});
|
|
7270
|
+
} else {
|
|
7271
|
+
await this.indexResource.insert({
|
|
7272
|
+
id: `index-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`,
|
|
7273
|
+
resourceName,
|
|
7274
|
+
fieldName,
|
|
7275
|
+
word,
|
|
7276
|
+
recordIds: data.recordIds,
|
|
7277
|
+
count: data.count,
|
|
7278
|
+
lastUpdated: (/* @__PURE__ */ new Date()).toISOString()
|
|
7279
|
+
});
|
|
7280
|
+
}
|
|
6931
7281
|
}
|
|
7282
|
+
this.dirtyIndexes.clear();
|
|
7283
|
+
this.deletedIndexes.clear();
|
|
6932
7284
|
});
|
|
6933
7285
|
}
|
|
6934
7286
|
installDatabaseHooks() {
|
|
@@ -7023,6 +7375,7 @@ class FullTextPlugin extends Plugin {
|
|
|
7023
7375
|
existing.count = existing.recordIds.length;
|
|
7024
7376
|
}
|
|
7025
7377
|
this.indexes.set(key, existing);
|
|
7378
|
+
this.dirtyIndexes.add(key);
|
|
7026
7379
|
}
|
|
7027
7380
|
}
|
|
7028
7381
|
}
|
|
@@ -7035,8 +7388,10 @@ class FullTextPlugin extends Plugin {
|
|
|
7035
7388
|
data.count = data.recordIds.length;
|
|
7036
7389
|
if (data.recordIds.length === 0) {
|
|
7037
7390
|
this.indexes.delete(key);
|
|
7391
|
+
this.deletedIndexes.add(key);
|
|
7038
7392
|
} else {
|
|
7039
7393
|
this.indexes.set(key, data);
|
|
7394
|
+
this.dirtyIndexes.add(key);
|
|
7040
7395
|
}
|
|
7041
7396
|
}
|
|
7042
7397
|
}
|
|
@@ -7268,8 +7623,14 @@ class MetricsPlugin extends Plugin {
|
|
|
7268
7623
|
errors: "number|required",
|
|
7269
7624
|
avgTime: "number|required",
|
|
7270
7625
|
timestamp: "string|required",
|
|
7271
|
-
metadata: "json"
|
|
7272
|
-
|
|
7626
|
+
metadata: "json",
|
|
7627
|
+
createdAt: "string|required"
|
|
7628
|
+
// YYYY-MM-DD for partitioning
|
|
7629
|
+
},
|
|
7630
|
+
partitions: {
|
|
7631
|
+
byDate: { fields: { createdAt: "string|maxlength:10" } }
|
|
7632
|
+
},
|
|
7633
|
+
behavior: "body-overflow"
|
|
7273
7634
|
}));
|
|
7274
7635
|
this.metricsResource = ok1 ? metricsResource : this.database.resources.plg_metrics;
|
|
7275
7636
|
const [ok2, err2, errorsResource] = await tryFn(() => this.database.createResource({
|
|
@@ -7280,8 +7641,14 @@ class MetricsPlugin extends Plugin {
|
|
|
7280
7641
|
operation: "string|required",
|
|
7281
7642
|
error: "string|required",
|
|
7282
7643
|
timestamp: "string|required",
|
|
7283
|
-
metadata: "json"
|
|
7284
|
-
|
|
7644
|
+
metadata: "json",
|
|
7645
|
+
createdAt: "string|required"
|
|
7646
|
+
// YYYY-MM-DD for partitioning
|
|
7647
|
+
},
|
|
7648
|
+
partitions: {
|
|
7649
|
+
byDate: { fields: { createdAt: "string|maxlength:10" } }
|
|
7650
|
+
},
|
|
7651
|
+
behavior: "body-overflow"
|
|
7285
7652
|
}));
|
|
7286
7653
|
this.errorsResource = ok2 ? errorsResource : this.database.resources.plg_error_logs;
|
|
7287
7654
|
const [ok3, err3, performanceResource] = await tryFn(() => this.database.createResource({
|
|
@@ -7292,8 +7659,14 @@ class MetricsPlugin extends Plugin {
|
|
|
7292
7659
|
operation: "string|required",
|
|
7293
7660
|
duration: "number|required",
|
|
7294
7661
|
timestamp: "string|required",
|
|
7295
|
-
metadata: "json"
|
|
7296
|
-
|
|
7662
|
+
metadata: "json",
|
|
7663
|
+
createdAt: "string|required"
|
|
7664
|
+
// YYYY-MM-DD for partitioning
|
|
7665
|
+
},
|
|
7666
|
+
partitions: {
|
|
7667
|
+
byDate: { fields: { createdAt: "string|maxlength:10" } }
|
|
7668
|
+
},
|
|
7669
|
+
behavior: "body-overflow"
|
|
7297
7670
|
}));
|
|
7298
7671
|
this.performanceResource = ok3 ? performanceResource : this.database.resources.plg_performance_logs;
|
|
7299
7672
|
});
|
|
@@ -7514,6 +7887,8 @@ class MetricsPlugin extends Plugin {
|
|
|
7514
7887
|
errorMetadata = { error: "true" };
|
|
7515
7888
|
resourceMetadata = { resource: "true" };
|
|
7516
7889
|
}
|
|
7890
|
+
const now = /* @__PURE__ */ new Date();
|
|
7891
|
+
const createdAt = now.toISOString().slice(0, 10);
|
|
7517
7892
|
for (const [operation, data] of Object.entries(this.metrics.operations)) {
|
|
7518
7893
|
if (data.count > 0) {
|
|
7519
7894
|
await this.metricsResource.insert({
|
|
@@ -7525,7 +7900,8 @@ class MetricsPlugin extends Plugin {
|
|
|
7525
7900
|
totalTime: data.totalTime,
|
|
7526
7901
|
errors: data.errors,
|
|
7527
7902
|
avgTime: data.count > 0 ? data.totalTime / data.count : 0,
|
|
7528
|
-
timestamp:
|
|
7903
|
+
timestamp: now.toISOString(),
|
|
7904
|
+
createdAt,
|
|
7529
7905
|
metadata
|
|
7530
7906
|
});
|
|
7531
7907
|
}
|
|
@@ -7542,7 +7918,8 @@ class MetricsPlugin extends Plugin {
|
|
|
7542
7918
|
totalTime: data.totalTime,
|
|
7543
7919
|
errors: data.errors,
|
|
7544
7920
|
avgTime: data.count > 0 ? data.totalTime / data.count : 0,
|
|
7545
|
-
timestamp:
|
|
7921
|
+
timestamp: now.toISOString(),
|
|
7922
|
+
createdAt,
|
|
7546
7923
|
metadata: resourceMetadata
|
|
7547
7924
|
});
|
|
7548
7925
|
}
|
|
@@ -7556,6 +7933,8 @@ class MetricsPlugin extends Plugin {
|
|
|
7556
7933
|
operation: perf.operation,
|
|
7557
7934
|
duration: perf.duration,
|
|
7558
7935
|
timestamp: perf.timestamp,
|
|
7936
|
+
createdAt: perf.timestamp.slice(0, 10),
|
|
7937
|
+
// YYYY-MM-DD from timestamp
|
|
7559
7938
|
metadata: perfMetadata
|
|
7560
7939
|
});
|
|
7561
7940
|
}
|
|
@@ -7569,6 +7948,8 @@ class MetricsPlugin extends Plugin {
|
|
|
7569
7948
|
error: error.error,
|
|
7570
7949
|
stack: error.stack,
|
|
7571
7950
|
timestamp: error.timestamp,
|
|
7951
|
+
createdAt: error.timestamp.slice(0, 10),
|
|
7952
|
+
// YYYY-MM-DD from timestamp
|
|
7572
7953
|
metadata: errorMetadata
|
|
7573
7954
|
});
|
|
7574
7955
|
}
|
|
@@ -7700,22 +8081,47 @@ class MetricsPlugin extends Plugin {
|
|
|
7700
8081
|
async cleanupOldData() {
|
|
7701
8082
|
const cutoffDate = /* @__PURE__ */ new Date();
|
|
7702
8083
|
cutoffDate.setDate(cutoffDate.getDate() - this.config.retentionDays);
|
|
8084
|
+
cutoffDate.toISOString().slice(0, 10);
|
|
8085
|
+
const datesToDelete = [];
|
|
8086
|
+
const startDate = new Date(cutoffDate);
|
|
8087
|
+
startDate.setDate(startDate.getDate() - 365);
|
|
8088
|
+
for (let d = new Date(startDate); d < cutoffDate; d.setDate(d.getDate() + 1)) {
|
|
8089
|
+
datesToDelete.push(d.toISOString().slice(0, 10));
|
|
8090
|
+
}
|
|
7703
8091
|
if (this.metricsResource) {
|
|
7704
|
-
const
|
|
7705
|
-
|
|
7706
|
-
|
|
8092
|
+
for (const dateStr of datesToDelete) {
|
|
8093
|
+
const [ok, err, oldMetrics] = await tryFn(
|
|
8094
|
+
() => this.metricsResource.query({ createdAt: dateStr })
|
|
8095
|
+
);
|
|
8096
|
+
if (ok && oldMetrics) {
|
|
8097
|
+
for (const metric of oldMetrics) {
|
|
8098
|
+
await tryFn(() => this.metricsResource.delete(metric.id));
|
|
8099
|
+
}
|
|
8100
|
+
}
|
|
7707
8101
|
}
|
|
7708
8102
|
}
|
|
7709
8103
|
if (this.errorsResource) {
|
|
7710
|
-
const
|
|
7711
|
-
|
|
7712
|
-
|
|
8104
|
+
for (const dateStr of datesToDelete) {
|
|
8105
|
+
const [ok, err, oldErrors] = await tryFn(
|
|
8106
|
+
() => this.errorsResource.query({ createdAt: dateStr })
|
|
8107
|
+
);
|
|
8108
|
+
if (ok && oldErrors) {
|
|
8109
|
+
for (const error of oldErrors) {
|
|
8110
|
+
await tryFn(() => this.errorsResource.delete(error.id));
|
|
8111
|
+
}
|
|
8112
|
+
}
|
|
7713
8113
|
}
|
|
7714
8114
|
}
|
|
7715
8115
|
if (this.performanceResource) {
|
|
7716
|
-
const
|
|
7717
|
-
|
|
7718
|
-
|
|
8116
|
+
for (const dateStr of datesToDelete) {
|
|
8117
|
+
const [ok, err, oldPerformance] = await tryFn(
|
|
8118
|
+
() => this.performanceResource.query({ createdAt: dateStr })
|
|
8119
|
+
);
|
|
8120
|
+
if (ok && oldPerformance) {
|
|
8121
|
+
for (const perf of oldPerformance) {
|
|
8122
|
+
await tryFn(() => this.performanceResource.delete(perf.id));
|
|
8123
|
+
}
|
|
8124
|
+
}
|
|
7719
8125
|
}
|
|
7720
8126
|
}
|
|
7721
8127
|
}
|
|
@@ -12786,7 +13192,7 @@ class Database extends EventEmitter {
|
|
|
12786
13192
|
this.id = idGenerator(7);
|
|
12787
13193
|
this.version = "1";
|
|
12788
13194
|
this.s3dbVersion = (() => {
|
|
12789
|
-
const [ok, err, version] = tryFn(() => true ? "11.0.
|
|
13195
|
+
const [ok, err, version] = tryFn(() => true ? "11.0.4" : "latest");
|
|
12790
13196
|
return ok ? version : "latest";
|
|
12791
13197
|
})();
|
|
12792
13198
|
this.resources = {};
|
|
@@ -15095,28 +15501,6 @@ class S3QueuePlugin extends Plugin {
|
|
|
15095
15501
|
throw new Error(`Failed to create queue resource: ${err?.message}`);
|
|
15096
15502
|
}
|
|
15097
15503
|
this.queueResource = this.database.resources[queueName];
|
|
15098
|
-
const lockName = `${this.config.resource}_locks`;
|
|
15099
|
-
const [okLock, errLock] = await tryFn(
|
|
15100
|
-
() => this.database.createResource({
|
|
15101
|
-
name: lockName,
|
|
15102
|
-
attributes: {
|
|
15103
|
-
id: "string|required",
|
|
15104
|
-
workerId: "string|required",
|
|
15105
|
-
timestamp: "number|required",
|
|
15106
|
-
ttl: "number|default:5000"
|
|
15107
|
-
},
|
|
15108
|
-
behavior: "body-overflow",
|
|
15109
|
-
timestamps: false
|
|
15110
|
-
})
|
|
15111
|
-
);
|
|
15112
|
-
if (okLock || this.database.resources[lockName]) {
|
|
15113
|
-
this.lockResource = this.database.resources[lockName];
|
|
15114
|
-
} else {
|
|
15115
|
-
this.lockResource = null;
|
|
15116
|
-
if (this.config.verbose) {
|
|
15117
|
-
console.log(`[S3QueuePlugin] Lock resource creation failed, locking disabled: ${errLock?.message}`);
|
|
15118
|
-
}
|
|
15119
|
-
}
|
|
15120
15504
|
this.addHelperMethods();
|
|
15121
15505
|
if (this.config.deadLetterResource) {
|
|
15122
15506
|
await this.createDeadLetterResource();
|
|
@@ -15187,13 +15571,6 @@ class S3QueuePlugin extends Plugin {
|
|
|
15187
15571
|
}
|
|
15188
15572
|
}
|
|
15189
15573
|
}, 5e3);
|
|
15190
|
-
this.lockCleanupInterval = setInterval(() => {
|
|
15191
|
-
this.cleanupStaleLocks().catch((err) => {
|
|
15192
|
-
if (this.config.verbose) {
|
|
15193
|
-
console.log(`[lockCleanup] Error: ${err.message}`);
|
|
15194
|
-
}
|
|
15195
|
-
});
|
|
15196
|
-
}, 1e4);
|
|
15197
15574
|
for (let i = 0; i < concurrency; i++) {
|
|
15198
15575
|
const worker = this.createWorker(messageHandler, i);
|
|
15199
15576
|
this.workers.push(worker);
|
|
@@ -15210,10 +15587,6 @@ class S3QueuePlugin extends Plugin {
|
|
|
15210
15587
|
clearInterval(this.cacheCleanupInterval);
|
|
15211
15588
|
this.cacheCleanupInterval = null;
|
|
15212
15589
|
}
|
|
15213
|
-
if (this.lockCleanupInterval) {
|
|
15214
|
-
clearInterval(this.lockCleanupInterval);
|
|
15215
|
-
this.lockCleanupInterval = null;
|
|
15216
|
-
}
|
|
15217
15590
|
await Promise.all(this.workers);
|
|
15218
15591
|
this.workers = [];
|
|
15219
15592
|
this.processedCache.clear();
|
|
@@ -15264,48 +15637,21 @@ class S3QueuePlugin extends Plugin {
|
|
|
15264
15637
|
return null;
|
|
15265
15638
|
}
|
|
15266
15639
|
/**
|
|
15267
|
-
* Acquire a distributed lock using
|
|
15640
|
+
* Acquire a distributed lock using PluginStorage TTL
|
|
15268
15641
|
* This ensures only one worker can claim a message at a time
|
|
15269
|
-
*
|
|
15270
|
-
* Uses a two-step process:
|
|
15271
|
-
* 1. Create lock resource (similar to queue resource) if not exists
|
|
15272
|
-
* 2. Try to claim lock using ETag-based conditional update
|
|
15273
15642
|
*/
|
|
15274
15643
|
async acquireLock(messageId) {
|
|
15275
|
-
|
|
15276
|
-
|
|
15277
|
-
}
|
|
15278
|
-
const lockId = `lock-${messageId}`;
|
|
15279
|
-
const now = Date.now();
|
|
15644
|
+
const storage = this.getStorage();
|
|
15645
|
+
const lockKey = `msg-${messageId}`;
|
|
15280
15646
|
try {
|
|
15281
|
-
const
|
|
15282
|
-
|
|
15283
|
-
|
|
15284
|
-
|
|
15285
|
-
|
|
15286
|
-
|
|
15287
|
-
|
|
15288
|
-
|
|
15289
|
-
const [ok, err, result] = await tryFn(
|
|
15290
|
-
() => this.lockResource.updateConditional(lockId, {
|
|
15291
|
-
workerId: this.workerId,
|
|
15292
|
-
timestamp: now,
|
|
15293
|
-
ttl: 5e3
|
|
15294
|
-
}, {
|
|
15295
|
-
ifMatch: existingLock._etag
|
|
15296
|
-
})
|
|
15297
|
-
);
|
|
15298
|
-
return ok && result.success;
|
|
15299
|
-
}
|
|
15300
|
-
const [okCreate, errCreate] = await tryFn(
|
|
15301
|
-
() => this.lockResource.insert({
|
|
15302
|
-
id: lockId,
|
|
15303
|
-
workerId: this.workerId,
|
|
15304
|
-
timestamp: now,
|
|
15305
|
-
ttl: 5e3
|
|
15306
|
-
})
|
|
15307
|
-
);
|
|
15308
|
-
return okCreate;
|
|
15647
|
+
const lock = await storage.acquireLock(lockKey, {
|
|
15648
|
+
ttl: 5,
|
|
15649
|
+
// 5 seconds
|
|
15650
|
+
timeout: 0,
|
|
15651
|
+
// Don't wait if locked
|
|
15652
|
+
workerId: this.workerId
|
|
15653
|
+
});
|
|
15654
|
+
return lock !== null;
|
|
15309
15655
|
} catch (error) {
|
|
15310
15656
|
if (this.config.verbose) {
|
|
15311
15657
|
console.log(`[acquireLock] Error: ${error.message}`);
|
|
@@ -15314,15 +15660,13 @@ class S3QueuePlugin extends Plugin {
|
|
|
15314
15660
|
}
|
|
15315
15661
|
}
|
|
15316
15662
|
/**
|
|
15317
|
-
* Release a distributed lock
|
|
15663
|
+
* Release a distributed lock via PluginStorage
|
|
15318
15664
|
*/
|
|
15319
15665
|
async releaseLock(messageId) {
|
|
15320
|
-
|
|
15321
|
-
|
|
15322
|
-
}
|
|
15323
|
-
const lockId = `lock-${messageId}`;
|
|
15666
|
+
const storage = this.getStorage();
|
|
15667
|
+
const lockKey = `msg-${messageId}`;
|
|
15324
15668
|
try {
|
|
15325
|
-
await
|
|
15669
|
+
await storage.releaseLock(lockKey);
|
|
15326
15670
|
} catch (error) {
|
|
15327
15671
|
if (this.config.verbose) {
|
|
15328
15672
|
console.log(`[releaseLock] Failed to release lock for ${messageId}: ${error.message}`);
|
|
@@ -15330,30 +15674,11 @@ class S3QueuePlugin extends Plugin {
|
|
|
15330
15674
|
}
|
|
15331
15675
|
}
|
|
15332
15676
|
/**
|
|
15333
|
-
* Clean up stale locks
|
|
15334
|
-
*
|
|
15677
|
+
* Clean up stale locks - NO LONGER NEEDED
|
|
15678
|
+
* TTL handles automatic expiration, no manual cleanup required
|
|
15335
15679
|
*/
|
|
15336
15680
|
async cleanupStaleLocks() {
|
|
15337
|
-
|
|
15338
|
-
return;
|
|
15339
|
-
}
|
|
15340
|
-
const now = Date.now();
|
|
15341
|
-
try {
|
|
15342
|
-
const locks = await this.lockResource.list();
|
|
15343
|
-
for (const lock of locks) {
|
|
15344
|
-
const lockAge = now - lock.timestamp;
|
|
15345
|
-
if (lockAge > lock.ttl) {
|
|
15346
|
-
await this.lockResource.delete(lock.id);
|
|
15347
|
-
if (this.config.verbose) {
|
|
15348
|
-
console.log(`[cleanupStaleLocks] Removed expired lock: ${lock.id}`);
|
|
15349
|
-
}
|
|
15350
|
-
}
|
|
15351
|
-
}
|
|
15352
|
-
} catch (error) {
|
|
15353
|
-
if (this.config.verbose) {
|
|
15354
|
-
console.log(`[cleanupStaleLocks] Error during cleanup: ${error.message}`);
|
|
15355
|
-
}
|
|
15356
|
-
}
|
|
15681
|
+
return;
|
|
15357
15682
|
}
|
|
15358
15683
|
async attemptClaim(msg) {
|
|
15359
15684
|
const now = Date.now();
|
|
@@ -15577,7 +15902,6 @@ class SchedulerPlugin extends Plugin {
|
|
|
15577
15902
|
...options
|
|
15578
15903
|
};
|
|
15579
15904
|
this.database = null;
|
|
15580
|
-
this.lockResource = null;
|
|
15581
15905
|
this.jobs = /* @__PURE__ */ new Map();
|
|
15582
15906
|
this.activeJobs = /* @__PURE__ */ new Map();
|
|
15583
15907
|
this.timers = /* @__PURE__ */ new Map();
|
|
@@ -15616,7 +15940,6 @@ class SchedulerPlugin extends Plugin {
|
|
|
15616
15940
|
return true;
|
|
15617
15941
|
}
|
|
15618
15942
|
async onInstall() {
|
|
15619
|
-
await this._createLockResource();
|
|
15620
15943
|
if (this.config.persistJobs) {
|
|
15621
15944
|
await this._createJobHistoryResource();
|
|
15622
15945
|
}
|
|
@@ -15645,25 +15968,6 @@ class SchedulerPlugin extends Plugin {
|
|
|
15645
15968
|
await this._startScheduling();
|
|
15646
15969
|
this.emit("initialized", { jobs: this.jobs.size });
|
|
15647
15970
|
}
|
|
15648
|
-
async _createLockResource() {
|
|
15649
|
-
const [ok, err, lockResource] = await tryFn(
|
|
15650
|
-
() => this.database.createResource({
|
|
15651
|
-
name: "plg_scheduler_job_locks",
|
|
15652
|
-
attributes: {
|
|
15653
|
-
id: "string|required",
|
|
15654
|
-
jobName: "string|required",
|
|
15655
|
-
lockedAt: "number|required",
|
|
15656
|
-
instanceId: "string|optional"
|
|
15657
|
-
},
|
|
15658
|
-
behavior: "body-only",
|
|
15659
|
-
timestamps: false
|
|
15660
|
-
})
|
|
15661
|
-
);
|
|
15662
|
-
if (!ok && !this.database.resources.plg_scheduler_job_locks) {
|
|
15663
|
-
throw new Error(`Failed to create lock resource: ${err?.message}`);
|
|
15664
|
-
}
|
|
15665
|
-
this.lockResource = ok ? lockResource : this.database.resources.plg_scheduler_job_locks;
|
|
15666
|
-
}
|
|
15667
15971
|
async _createJobHistoryResource() {
|
|
15668
15972
|
const [ok] = await tryFn(() => this.database.createResource({
|
|
15669
15973
|
name: this.config.jobHistoryResource,
|
|
@@ -15771,16 +16075,16 @@ class SchedulerPlugin extends Plugin {
|
|
|
15771
16075
|
return;
|
|
15772
16076
|
}
|
|
15773
16077
|
this.activeJobs.set(jobName, "acquiring-lock");
|
|
15774
|
-
const
|
|
15775
|
-
const
|
|
15776
|
-
|
|
15777
|
-
|
|
15778
|
-
|
|
15779
|
-
|
|
15780
|
-
|
|
15781
|
-
|
|
15782
|
-
);
|
|
15783
|
-
if (!
|
|
16078
|
+
const storage = this.getStorage();
|
|
16079
|
+
const lockKey = `job-${jobName}`;
|
|
16080
|
+
const lock = await storage.acquireLock(lockKey, {
|
|
16081
|
+
ttl: Math.ceil(job.timeout / 1e3) + 60,
|
|
16082
|
+
// Job timeout + 60 seconds buffer
|
|
16083
|
+
timeout: 0,
|
|
16084
|
+
// Don't wait if locked
|
|
16085
|
+
workerId: process.pid ? String(process.pid) : "unknown"
|
|
16086
|
+
});
|
|
16087
|
+
if (!lock) {
|
|
15784
16088
|
if (this.config.verbose) {
|
|
15785
16089
|
console.log(`[SchedulerPlugin] Job '${jobName}' already running on another instance`);
|
|
15786
16090
|
}
|
|
@@ -15884,7 +16188,7 @@ class SchedulerPlugin extends Plugin {
|
|
|
15884
16188
|
throw lastError;
|
|
15885
16189
|
}
|
|
15886
16190
|
} finally {
|
|
15887
|
-
await tryFn(() =>
|
|
16191
|
+
await tryFn(() => storage.releaseLock(lockKey));
|
|
15888
16192
|
}
|
|
15889
16193
|
}
|
|
15890
16194
|
async _persistJobExecution(jobName, executionId, startTime, endTime, duration, status, result, error, retryCount) {
|